diff --git a/README.md b/README.md index d996e9f..c333d4b 100644 --- a/README.md +++ b/README.md @@ -83,3 +83,40 @@ This will give you a URL you can visit to add the bot to your server. 2. Download release binary, rename to `.exe` 3. Place at `C:/wrench.exe` or some other location. Put on PATH if desired. 4. In admin terminal run `wrench.exe setup` + +## Run your own ziglang.org/download mirror + +If you want to mirror https://ziglang.org/download on-demand, similar to what https://pkg.machengine.org does, +you may do so using e.g. this `config.toml` with `Mode = "zig"` which disables all other Wrench functionality so it only mirrors Zig downloads: + +```toml +# Note: data will be written in a directory relative to this config file. +Mode = "zig" + +# HTTP configuration +ExternalURL = "http://foobar.com" +Address = ":80" + +# HTTPS configuration (optional, uses LetsEncrypt) +#ExternalURL = "https://foobar.com" +#Address = ":443" +#LetsEncryptEmail = "foo@bar.com" +``` + +Wrench will save data relative to that config file, so generally you should put that `config.toml` into e.g. a `wrench/` directory somewhere. + +Running `wrench svc run` will start the server. Then you can fetch e.g.: + +* http://localhost/ +* http://localhost/zig/zig-linux-x86_64-0.13.0.tar.xz +* http://localhost/zig/index.json - a strict superset of https://ziglang.org/download/index.json + +Downloads like http://localhost/zig/zig-linux-x86_64-0.13.0.tar.xz will be fetched on-demand from ziglang.org and then cached on the local filesystem forever after that. + +http://localhost/zig/index.json is like https://ziglang.org/download/index.json with some small differences: + +* It is fetched from ziglang.org once every 15 minutes and cached in-memory. +* Entries from https://machengine.org/zig/index.json are added so the index.json _additionally_ contains Mach [nominated Zig versions](https://machengine.org/about/nominated-zig/) +* `tarball` fields are rewritten to point to the configured `ExternalURL` + +If you want to run Wrench as a system service, have it auto-start after reboot, etc. then you can e.g. put the config file in `/root/wrench/config.toml`, run `wrench svc install` as root to install the systemd service, use `wrench svc start` to start the service, and `wrench svc status` to see the status and log file locations. \ No newline at end of file diff --git a/internal/wrench/bot.go b/internal/wrench/bot.go index e1040fd..4bc065a 100644 --- a/internal/wrench/bot.go +++ b/internal/wrench/bot.go @@ -45,6 +45,9 @@ func (b *Bot) loadConfig() error { return errors.New("expected Config or ConfigFile to be specified") } b.Config = &Config{} + if absPath, err := filepath.Abs(b.ConfigFile); err == nil { + b.logf("loading config file: %s", absPath) + } return LoadConfig(b.ConfigFile, b.Config) } return nil @@ -120,7 +123,7 @@ func (b *Bot) run(s service.Service) error { } if b.Config.Runner == "" { - if !b.Config.PkgProxy { + if b.Config.ModeType() == ModeWrench { b.store, err = OpenStore(filepath.Join(b.Config.WrenchDir, "wrench.db") + "?_pragma=busy_timeout%3d10000") if err != nil { return errors.Wrap(err, "OpenStore") @@ -135,7 +138,7 @@ func (b *Bot) run(s service.Service) error { if err := b.httpStart(); err != nil { return errors.Wrap(err, "http") } - if !b.Config.PkgProxy { + if b.Config.ModeType() == ModeWrench { if err := b.schedulerStart(); err != nil { return errors.Wrap(err, "scheduler") } diff --git a/internal/wrench/config.go b/internal/wrench/config.go index 4fbf34d..650d463 100644 --- a/internal/wrench/config.go +++ b/internal/wrench/config.go @@ -9,6 +9,14 @@ import ( "github.com/hexops/wrench/internal/wrench/api" ) +type ModeType string + +const ( + ModeWrench ModeType = "wrench" + ModePkg ModeType = "pkg" + ModeZig ModeType = "zig" +) + type Config struct { // ExternalURL where Wrench is hosted, if any. ExternalURL string @@ -21,40 +29,64 @@ type Config struct { // Act as a Zig package proxy like pkg.machengine.org, instead of as a regular wrench server. PkgProxy bool `toml:"PkgProxy,omitempty"` + // Mode to operate in, one of: + // + // * "wrench" -> https://wrench.machengine.org - custom CI system, etc. + // * "pkg" -> https://pkg.machengine.org - package mirror and Zig download mirror + // * "zig" -> Zig download mirror only (subset of pkg.machengine.org) + Mode string `toml:"Mode,omitempty"` + + // (optional) Directory for caching LetsEncrypt certificates + LetsEncryptCacheDir string `toml:"LetsEncryptCacheDir,omitempty"` + + // (optional) Email to use for LetsEncrypt notifications + LetsEncryptEmail string `toml:"LetsEncryptEmail,omitempty"` + + // Where Wrench should store its data, cofiguration, etc. Defaults to the directory containing + // this config file. + WrenchDir string `toml:"WrenchDir,omitempty"` + + // All options below here are only used in "wrench" mode. + // (optional) Discord bot token. See README.md for details on how to create this. // // Disabled if an empty string. + // + // Only used in "wrench" mode. DiscordBotToken string `toml:"DiscordBotToken,omitempty"` // (required if DiscordBotToken is set) Discord guild/server ID to operate in. // // Find this via User Settings -> Advanced -> Enabled developer mode, then right-click on any // server and Copy ID) + // + // Only used in "wrench" mode. DiscordGuildID string `toml:"DiscordGuildID,omitempty"` // (optional) Discord channel name for Wrench to send messages in. Defaults to "wrench" + // + // Only used in "wrench" mode. DiscordChannel string `toml:"DiscordChannel,omitempty"` // (optional) Discord channel name for Wrench to relay all Discord messages to. Defaults to "disabled" + // + // Only used in "wrench" mode. ActivityChannel string `toml:"ActivityChannel,omitempty"` - // (optional) Directory for caching LetsEncrypt certificates - LetsEncryptCacheDir string `toml:"LetsEncryptCacheDir,omitempty"` - - // (optional) Email to use for LetsEncrypt notifications - LetsEncryptEmail string `toml:"LetsEncryptEmail,omitempty"` - // (optional) When specified, this is an arbitrary secret of your choosing which can be used to // send GitHub webhook events from the github.com/hexops/wrench repository itself to Wrench. It // will respond to these by recompiling and launching itself: // // The webhook URL should be: /webhook/github/self // + // Only used in "wrench" mode. GitHubWebHookSecret string `toml:"GitHubWebHookSecret,omitempty"` // (optional) When specified Wrench can send PRs and assist with GitHub. // // Only applicable if running as the Wrench server. + // + // Only used in "wrench" mode. GitHubAccessToken string `toml:"GitHubAccessToken,omitempty"` // (optional) When specified wrench runners can push to Git using this configuration. @@ -62,20 +94,32 @@ type Config struct { // be distributed to all runners. // // Only applicable if running as the Wrench server. + // + // Only used in "wrench" mode. GitPushUsername string `toml:"GitPushUsername,omitempty"` GitPushPassword string `toml:"GitPushPassword,omitempty"` GitConfigUserName string `toml:"GitConfigUserName,omitempty"` GitConfigUserEmail string `toml:"GitConfigUserEmail,omitempty"` // (optional) Generic secret used to authenticate with this server. Any arbitrary string. + // + // Only used in "wrench" mode. Secret string `toml:"Secret,omitempty"` // (optional) Act as a runner, connecting to the root Wrench server specified in ExternalURL. + // + // Only used in "wrench" mode. Runner string `toml:"Runner,omitempty"` +} - // Where Wrench should store its data, cofiguration, etc. Defaults to the directory containing - // this config file. - WrenchDir string `toml:"WrenchDir,omitempty"` +func (c *Config) ModeType() ModeType { + if c.Mode != "" { + return ModeType(c.Mode) + } + if c.PkgProxy { + return ModePkg + } + return ModeWrench } func (c *Config) LogFilePath() string { diff --git a/internal/wrench/http.go b/internal/wrench/http.go index 076ba29..c38c21a 100644 --- a/internal/wrench/http.go +++ b/internal/wrench/http.go @@ -46,11 +46,17 @@ func (b *Bot) httpStart() error { } var mux http.Handler - if b.Config.PkgProxy { - b.logf("http: PkgProxy mode enabled") + if b.Config.ModeType() == ModePkg { + b.logf("http: pkg mirror mode enabled") mux = b.httpMuxPkgProxy(handler) - } else { + } else if b.Config.ModeType() == ModeZig { + b.logf("http: zig mirror mode enabled") + mux = b.httpMuxPkgProxy(handler) + } else if b.Config.ModeType() == ModeWrench { + b.logf("http: wrench mode enabled") mux = b.httpMuxDefault(handler) + } else { + b.logf("invalid config mode=%q", b.Config.ModeType()) } b.logf("http: listening on %v - %v", b.Config.Address, b.Config.ExternalURL) diff --git a/internal/wrench/http_pkg.go b/internal/wrench/http_pkg.go index 5ac2750..c5834db 100644 --- a/internal/wrench/http_pkg.go +++ b/internal/wrench/http_pkg.go @@ -1,6 +1,7 @@ package wrench import ( + "encoding/json" "fmt" "net/http" "net/url" @@ -8,13 +9,31 @@ import ( "path" "regexp" "strings" + "sync" + "time" + "github.com/hexops/wrench/internal/errors" "github.com/hexops/wrench/internal/wrench/scripts" + orderedmap "github.com/wk8/go-ordered-map/v2" ) func (b *Bot) httpMuxPkgProxy(handler func(prefix string, handle handlerFunc) http.Handler) http.Handler { mux := http.NewServeMux() mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if b.Config.ModeType() == ModeZig { + if r.URL.Path == "/" { + handler("general", b.httpPkgZigRoot).ServeHTTP(w, r) + return + } + if strings.HasPrefix(r.URL.Path, "/zig") { + handler("zig", b.httpPkgZig).ServeHTTP(w, r) + return + } + w.WriteHeader(http.StatusNotFound) + return + } + + // pkg mode if r.URL.Path == "/" { handler("general", b.httpPkgRoot).ServeHTTP(w, r) return @@ -39,13 +58,36 @@ func (b *Bot) httpMuxPkgProxy(handler func(prefix string, handle handlerFunc) ht return mux } +func (b *Bot) httpPkgZigRoot(w http.ResponseWriter, r *http.Request) error { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + + fmt.Fprintf(w, "%s", ` +
This site acts as a mirror of ziglang.org/download
+The rewrite logic is as follows:
++https://ziglang.org/builds/$FILE -> %s/zig/$FILE ++
Note: .tar.gz, .zip, and .minisig signatures are available for download. Signatures can also be downloaded from ziglang.org for verification purposes.
+ +~ Wrench.
+`, b.Config.ExternalURL) + return nil +} func (b *Bot) httpPkgRoot(w http.ResponseWriter, r *http.Request) error { w.Header().Set("Content-Type", "text/html; charset=utf-8") fmt.Fprintf(w, "%s", `This site acts as a mirror of ziglang.org/download
@@ -86,7 +128,7 @@ func (b *Bot) httpPkgRoot(w http.ResponseWriter, r *http.Request) error { } var ( - zigVersionRegexp = regexp.MustCompile(`(\d\.?)+-[[:alnum:]]+.\d+\+[[:alnum:]]+`) + zigVersionRegexp = regexp.MustCompile(`^zig-(\w+-)*\d+\.\d+\.\d+(-dev\.\d+\+[[:alnum:]]+)?$`) // From semver.org semverRegexp = regexp.MustCompile(`^(?P