diff --git a/README.md b/README.md index d996e9f..c333d4b 100644 --- a/README.md +++ b/README.md @@ -83,3 +83,40 @@ This will give you a URL you can visit to add the bot to your server. 2. Download release binary, rename to `.exe` 3. Place at `C:/wrench.exe` or some other location. Put on PATH if desired. 4. In admin terminal run `wrench.exe setup` + +## Run your own ziglang.org/download mirror + +If you want to mirror https://ziglang.org/download on-demand, similar to what https://pkg.machengine.org does, +you may do so using e.g. this `config.toml` with `Mode = "zig"` which disables all other Wrench functionality so it only mirrors Zig downloads: + +```toml +# Note: data will be written in a directory relative to this config file. +Mode = "zig" + +# HTTP configuration +ExternalURL = "http://foobar.com" +Address = ":80" + +# HTTPS configuration (optional, uses LetsEncrypt) +#ExternalURL = "https://foobar.com" +#Address = ":443" +#LetsEncryptEmail = "foo@bar.com" +``` + +Wrench will save data relative to that config file, so generally you should put that `config.toml` into e.g. a `wrench/` directory somewhere. + +Running `wrench svc run` will start the server. Then you can fetch e.g.: + +* http://localhost/ +* http://localhost/zig/zig-linux-x86_64-0.13.0.tar.xz +* http://localhost/zig/index.json - a strict superset of https://ziglang.org/download/index.json + +Downloads like http://localhost/zig/zig-linux-x86_64-0.13.0.tar.xz will be fetched on-demand from ziglang.org and then cached on the local filesystem forever after that. + +http://localhost/zig/index.json is like https://ziglang.org/download/index.json with some small differences: + +* It is fetched from ziglang.org once every 15 minutes and cached in-memory. +* Entries from https://machengine.org/zig/index.json are added so the index.json _additionally_ contains Mach [nominated Zig versions](https://machengine.org/about/nominated-zig/) +* `tarball` fields are rewritten to point to the configured `ExternalURL` + +If you want to run Wrench as a system service, have it auto-start after reboot, etc. then you can e.g. put the config file in `/root/wrench/config.toml`, run `wrench svc install` as root to install the systemd service, use `wrench svc start` to start the service, and `wrench svc status` to see the status and log file locations. \ No newline at end of file diff --git a/internal/wrench/bot.go b/internal/wrench/bot.go index e1040fd..4bc065a 100644 --- a/internal/wrench/bot.go +++ b/internal/wrench/bot.go @@ -45,6 +45,9 @@ func (b *Bot) loadConfig() error { return errors.New("expected Config or ConfigFile to be specified") } b.Config = &Config{} + if absPath, err := filepath.Abs(b.ConfigFile); err == nil { + b.logf("loading config file: %s", absPath) + } return LoadConfig(b.ConfigFile, b.Config) } return nil @@ -120,7 +123,7 @@ func (b *Bot) run(s service.Service) error { } if b.Config.Runner == "" { - if !b.Config.PkgProxy { + if b.Config.ModeType() == ModeWrench { b.store, err = OpenStore(filepath.Join(b.Config.WrenchDir, "wrench.db") + "?_pragma=busy_timeout%3d10000") if err != nil { return errors.Wrap(err, "OpenStore") @@ -135,7 +138,7 @@ func (b *Bot) run(s service.Service) error { if err := b.httpStart(); err != nil { return errors.Wrap(err, "http") } - if !b.Config.PkgProxy { + if b.Config.ModeType() == ModeWrench { if err := b.schedulerStart(); err != nil { return errors.Wrap(err, "scheduler") } diff --git a/internal/wrench/config.go b/internal/wrench/config.go index 4fbf34d..650d463 100644 --- a/internal/wrench/config.go +++ b/internal/wrench/config.go @@ -9,6 +9,14 @@ import ( "github.com/hexops/wrench/internal/wrench/api" ) +type ModeType string + +const ( + ModeWrench ModeType = "wrench" + ModePkg ModeType = "pkg" + ModeZig ModeType = "zig" +) + type Config struct { // ExternalURL where Wrench is hosted, if any. ExternalURL string @@ -21,40 +29,64 @@ type Config struct { // Act as a Zig package proxy like pkg.machengine.org, instead of as a regular wrench server. PkgProxy bool `toml:"PkgProxy,omitempty"` + // Mode to operate in, one of: + // + // * "wrench" -> https://wrench.machengine.org - custom CI system, etc. + // * "pkg" -> https://pkg.machengine.org - package mirror and Zig download mirror + // * "zig" -> Zig download mirror only (subset of pkg.machengine.org) + Mode string `toml:"Mode,omitempty"` + + // (optional) Directory for caching LetsEncrypt certificates + LetsEncryptCacheDir string `toml:"LetsEncryptCacheDir,omitempty"` + + // (optional) Email to use for LetsEncrypt notifications + LetsEncryptEmail string `toml:"LetsEncryptEmail,omitempty"` + + // Where Wrench should store its data, cofiguration, etc. Defaults to the directory containing + // this config file. + WrenchDir string `toml:"WrenchDir,omitempty"` + + // All options below here are only used in "wrench" mode. + // (optional) Discord bot token. See README.md for details on how to create this. // // Disabled if an empty string. + // + // Only used in "wrench" mode. DiscordBotToken string `toml:"DiscordBotToken,omitempty"` // (required if DiscordBotToken is set) Discord guild/server ID to operate in. // // Find this via User Settings -> Advanced -> Enabled developer mode, then right-click on any // server and Copy ID) + // + // Only used in "wrench" mode. DiscordGuildID string `toml:"DiscordGuildID,omitempty"` // (optional) Discord channel name for Wrench to send messages in. Defaults to "wrench" + // + // Only used in "wrench" mode. DiscordChannel string `toml:"DiscordChannel,omitempty"` // (optional) Discord channel name for Wrench to relay all Discord messages to. Defaults to "disabled" + // + // Only used in "wrench" mode. ActivityChannel string `toml:"ActivityChannel,omitempty"` - // (optional) Directory for caching LetsEncrypt certificates - LetsEncryptCacheDir string `toml:"LetsEncryptCacheDir,omitempty"` - - // (optional) Email to use for LetsEncrypt notifications - LetsEncryptEmail string `toml:"LetsEncryptEmail,omitempty"` - // (optional) When specified, this is an arbitrary secret of your choosing which can be used to // send GitHub webhook events from the github.com/hexops/wrench repository itself to Wrench. It // will respond to these by recompiling and launching itself: // // The webhook URL should be: /webhook/github/self // + // Only used in "wrench" mode. GitHubWebHookSecret string `toml:"GitHubWebHookSecret,omitempty"` // (optional) When specified Wrench can send PRs and assist with GitHub. // // Only applicable if running as the Wrench server. + // + // Only used in "wrench" mode. GitHubAccessToken string `toml:"GitHubAccessToken,omitempty"` // (optional) When specified wrench runners can push to Git using this configuration. @@ -62,20 +94,32 @@ type Config struct { // be distributed to all runners. // // Only applicable if running as the Wrench server. + // + // Only used in "wrench" mode. GitPushUsername string `toml:"GitPushUsername,omitempty"` GitPushPassword string `toml:"GitPushPassword,omitempty"` GitConfigUserName string `toml:"GitConfigUserName,omitempty"` GitConfigUserEmail string `toml:"GitConfigUserEmail,omitempty"` // (optional) Generic secret used to authenticate with this server. Any arbitrary string. + // + // Only used in "wrench" mode. Secret string `toml:"Secret,omitempty"` // (optional) Act as a runner, connecting to the root Wrench server specified in ExternalURL. + // + // Only used in "wrench" mode. Runner string `toml:"Runner,omitempty"` +} - // Where Wrench should store its data, cofiguration, etc. Defaults to the directory containing - // this config file. - WrenchDir string `toml:"WrenchDir,omitempty"` +func (c *Config) ModeType() ModeType { + if c.Mode != "" { + return ModeType(c.Mode) + } + if c.PkgProxy { + return ModePkg + } + return ModeWrench } func (c *Config) LogFilePath() string { diff --git a/internal/wrench/http.go b/internal/wrench/http.go index 076ba29..c38c21a 100644 --- a/internal/wrench/http.go +++ b/internal/wrench/http.go @@ -46,11 +46,17 @@ func (b *Bot) httpStart() error { } var mux http.Handler - if b.Config.PkgProxy { - b.logf("http: PkgProxy mode enabled") + if b.Config.ModeType() == ModePkg { + b.logf("http: pkg mirror mode enabled") mux = b.httpMuxPkgProxy(handler) - } else { + } else if b.Config.ModeType() == ModeZig { + b.logf("http: zig mirror mode enabled") + mux = b.httpMuxPkgProxy(handler) + } else if b.Config.ModeType() == ModeWrench { + b.logf("http: wrench mode enabled") mux = b.httpMuxDefault(handler) + } else { + b.logf("invalid config mode=%q", b.Config.ModeType()) } b.logf("http: listening on %v - %v", b.Config.Address, b.Config.ExternalURL) diff --git a/internal/wrench/http_pkg.go b/internal/wrench/http_pkg.go index 5ac2750..c5834db 100644 --- a/internal/wrench/http_pkg.go +++ b/internal/wrench/http_pkg.go @@ -1,6 +1,7 @@ package wrench import ( + "encoding/json" "fmt" "net/http" "net/url" @@ -8,13 +9,31 @@ import ( "path" "regexp" "strings" + "sync" + "time" + "github.com/hexops/wrench/internal/errors" "github.com/hexops/wrench/internal/wrench/scripts" + orderedmap "github.com/wk8/go-ordered-map/v2" ) func (b *Bot) httpMuxPkgProxy(handler func(prefix string, handle handlerFunc) http.Handler) http.Handler { mux := http.NewServeMux() mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if b.Config.ModeType() == ModeZig { + if r.URL.Path == "/" { + handler("general", b.httpPkgZigRoot).ServeHTTP(w, r) + return + } + if strings.HasPrefix(r.URL.Path, "/zig") { + handler("zig", b.httpPkgZig).ServeHTTP(w, r) + return + } + w.WriteHeader(http.StatusNotFound) + return + } + + // pkg mode if r.URL.Path == "/" { handler("general", b.httpPkgRoot).ServeHTTP(w, r) return @@ -39,13 +58,36 @@ func (b *Bot) httpMuxPkgProxy(handler func(prefix string, handle handlerFunc) ht return mux } +func (b *Bot) httpPkgZigRoot(w http.ResponseWriter, r *http.Request) error { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + + fmt.Fprintf(w, "%s", ` +

Zig download mirror

+ +
+ +

This site acts as a mirror of ziglang.org/download

+

The rewrite logic is as follows:

+
+
+https://ziglang.org/builds/$FILE -> %s/zig/$FILE
+
+
+

Note: .tar.gz, .zip, and .minisig signatures are available for download. Signatures can also be downloaded from ziglang.org for verification purposes.

+ +
+ +

~ Wrench.

+`, b.Config.ExternalURL) + return nil +} func (b *Bot) httpPkgRoot(w http.ResponseWriter, r *http.Request) error { w.Header().Set("Content-Type", "text/html; charset=utf-8") fmt.Fprintf(w, "%s", `

pkg.machengine.org

The Mach package download server -
+

Zig downloads

This site acts as a mirror of ziglang.org/download

@@ -86,7 +128,7 @@ func (b *Bot) httpPkgRoot(w http.ResponseWriter, r *http.Request) error { } var ( - zigVersionRegexp = regexp.MustCompile(`(\d\.?)+-[[:alnum:]]+.\d+\+[[:alnum:]]+`) + zigVersionRegexp = regexp.MustCompile(`^zig-(\w+-)*\d+\.\d+\.\d+(-dev\.\d+\+[[:alnum:]]+)?$`) // From semver.org semverRegexp = regexp.MustCompile(`^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`) @@ -102,6 +144,9 @@ func (b *Bot) httpPkgZig(w http.ResponseWriter, r *http.Request) error { return nil } fname := split[len(split)-1] + if fname == "index.json" { + return b.httpPkgZigIndex(w, r) + } // Validate this is an allowed file validate := fname @@ -288,3 +333,118 @@ func (b *Bot) httpPkgArtifact(w http.ResponseWriter, r *http.Request) error { } return serveCacheHit() } + +var ( + httpPkgZigIndexMu sync.RWMutex + httpPkgZigIndexFetchedAt time.Time + httpPkgZigIndexCached []byte +) + +// https://pkg.machengine.org/zig/index.json - a strict superset of https://ziglang.org/builds/index.json +// updated every 15 minutes. +// +// Serves a memory-cached version of https://ziglang.org/builds/index.json (updated every 15 minutes) +// with any keys not present in that file from https://machengine.org/zig/index.json added at the end. +func (b *Bot) httpPkgZigIndex(w http.ResponseWriter, r *http.Request) error { + httpPkgZigIndexMu.RLock() + if time.Since(httpPkgZigIndexFetchedAt) < 15*time.Minute { + defer httpPkgZigIndexMu.RUnlock() + + w.Header().Set("Content-Type", "application/json; charset=utf-8") + fmt.Fprintf(w, "%s", httpPkgZigIndexCached) + return nil + } + + // Cache needs updating + httpPkgZigIndexMu.RUnlock() + httpPkgZigIndexMu.Lock() + defer httpPkgZigIndexMu.Unlock() + + if time.Since(httpPkgZigIndexFetchedAt) < 15*time.Minute { + // Someone else beat us to the update. + w.Header().Set("Content-Type", "application/json; charset=utf-8") + fmt.Fprintf(w, "%s", httpPkgZigIndexCached) + return nil + } + + // Fetch the latest upstream Zig index.json + resp, err := http.Get("https://ziglang.org/download/index.json") + if err != nil { + return errors.Wrap(err, "fetching upstream https://ziglang.org/download/index.json") + } + defer resp.Body.Close() + latestIndex := orderedmap.New[string, *orderedmap.OrderedMap[string, any]]() + if err := json.NewDecoder(resp.Body).Decode(&latestIndex); err != nil { + return errors.Wrap(err, "parsing upstream https://ziglang.org/builds/index.json") + } + + // Fetch the Mach index.json which contains Mach nominated versions, but is otherwise not as + // up-to-date as ziglang.org's version. + resp, err = http.Get("https://machengine.org/zig/index.json") + if err != nil { + return errors.Wrap(err, "fetching mach https://machengine.org/zig/index.json") + } + defer resp.Body.Close() + machIndex := orderedmap.New[string, *orderedmap.OrderedMap[string, any]]() + if err := json.NewDecoder(resp.Body).Decode(&machIndex); err != nil { + return errors.Wrap(err, "parsing mach https://machengine.org/zig/index.json") + } + + // "master", "0.13.0", etc. + for version := latestIndex.Oldest(); version != nil; version = version.Next() { + // "src", "x86_64-macos", etc. + for versionField := version.Value.Oldest(); versionField != nil; versionField = versionField.Next() { + // "version", "date", "src", "x86_64-macos", etc. + download, ok := versionField.Value.(map[string]any) + if ok { + newDownload := map[string]any{} + for key, value := range download { + newDownload[key] = value + if key == "tarball" { + newDownload["zigTarball"] = value.(string) + + newTarball := strings.Replace(value.(string), "https://ziglang.org/builds/", b.Config.ExternalURL+"/zig/", 1) + newTarball = strings.Replace(newTarball, "https://ziglang.org/download/", b.Config.ExternalURL+"/zig/", 1) + newDownload["tarball"] = newTarball + } + } + version.Value.Set(versionField.Key, newDownload) + } + } + } + + // "master", "0.13.0", etc. + for version := machIndex.Oldest(); version != nil; version = version.Next() { + if _, present := latestIndex.Get(version.Key); present { + // Always use the upstream index.json in the event of a collision + continue + } + + // "src", "x86_64-macos", etc. + for versionField := version.Value.Oldest(); versionField != nil; versionField = versionField.Next() { + // "version", "date", "src", "x86_64-macos", etc. + download, ok := versionField.Value.(map[string]any) + if ok { + newDownload := map[string]any{} + for key, value := range download { + newDownload[key] = value + if key == "tarball" { + newDownload["tarball"] = strings.Replace(value.(string), "https://pkg.machengine.org/zig/", b.Config.ExternalURL+"/zig/", 1) + } + } + version.Value.Set(versionField.Key, newDownload) + } + } + latestIndex.Set(version.Key, version.Value) + } + + httpPkgZigIndexCached, err = json.MarshalIndent(latestIndex, "", " ") + if err != nil { + return errors.Wrap(err, "marshalling index.json") + } + httpPkgZigIndexFetchedAt = time.Now() + + w.Header().Set("Content-Type", "application/json; charset=utf-8") + fmt.Fprintf(w, "%s", httpPkgZigIndexCached) + return nil +}