-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
generate a more complete website, with json, tsv, and more
this centralises all mirror information to one page, and allows for more easily setting up new mirrors, because the mirror information, xmirror data files, and prometheus configurationo can be generated at onces.
- Loading branch information
1 parent
fd9e60c
commit 93b9100
Showing
5 changed files
with
360 additions
and
51 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,191 @@ | ||
| #!/usr/bin/python3 | ||
|
|
||
| import csv | ||
| import json | ||
| import re | ||
| from pathlib import Path | ||
| from shutil import rmtree | ||
| from sys import argv, stderr | ||
|
|
||
| import yaml | ||
|
|
||
|
|
||
| html = """ | ||
| <h1>Void Linux Mirrors</h1> | ||
| <p>Void Linux maintains mirrors in several geographic regions for users. A fresh | ||
| install will default to | ||
| <a href="https://repo-default.voidlinux.org">repo-default.voidlinux.org</a>, | ||
| which may map to any Tier 1 mirror, but you may have a better experience by | ||
| <a href="https://docs.voidlinux.org/xbps/repositories/mirrors/changing.html">selecting | ||
| a different mirror</a>.</p> | ||
| <p>The status of all listed mirrors can be viewed on Void's | ||
| <a href="https://grafana.voidlinux.org/d/cLraC-XMk/mirrors-status">Grafana dashboard</a>.</p> | ||
| <h2 id="tier1">Tier 1 mirrors</h2> | ||
| <p>Tier 1 mirrors are maintained by the Void Linux Infrastructure Team. These | ||
| mirrors sync directly from the build servers and will always have the latest | ||
| packages available.</p> | ||
| {tier1} | ||
| <h2 id="tier2">Tier 2 mirrors</h2> | ||
| <p>Tier 2 mirrors sync from a nearby Tier 1 mirror when possible. These mirrors are | ||
| not managed by Void and do not have any guarantees of freshness or completeness | ||
| of packages, nor are they required to sync every available architecture or | ||
| sub-repository.</p> | ||
| {tier2} | ||
| <h2 id="tor">Tor Mirrors</h2> | ||
| <p>Void Linux is also mirrored on the Tor network. See | ||
| <a href="https://docs.voidlinux.org/xbps/repositories/mirrors/tor.html">Using Tor Mirrors</a> | ||
| for more information.</p> | ||
| <h2 id="creating">Creating a Mirror</h2> | ||
| <p>If you'd like to set up a mirror, and are confident you can keep it reasonably | ||
| up-to-date, follow one of the many guides available for mirroring with | ||
| <a href="https://man.voidlinux.org/rsync.1">rsync(1)</a>. You should be syncing from | ||
| <code>rsync://repo-sync.voidlinux.org/voidlinux/</code>. To list your mirror on this site, | ||
| submit a pull request to | ||
| <a href="https://github.com/void-linux/xmirror">the xmirror repository</a> that adds your | ||
| mirror to <code>mirrors.yaml</code>.</p> | ||
| <p>A full mirror requires around 1TB of storage. It is also possible to mirror only | ||
| part of the repositories. Excluding debug packages is one way of decreasing the | ||
| load on the Tier 1 mirrors, with low impact on users.</p> | ||
| <p>Please keep in mind that we pay bandwidth for all data sent out from the Tier 1 | ||
| mirrors. You can respect this by only mirroring if your use case for your mirror | ||
| will offset the network throughput consumed by your mirror syncing.</p> | ||
| """ | ||
|
|
||
| tabletmpl = """ | ||
| <table> | ||
| <tr> | ||
| <th>Mirror</th> | ||
| <th>Region</th> | ||
| <th>Location</th> | ||
| </tr> | ||
| {rows} | ||
| </table> | ||
| """ | ||
|
|
||
| rowtmpl = """<tr> | ||
| <td><a href="{url}">{url}</a></td> | ||
| <td>{region}</td> | ||
| <td>{location}</td> | ||
| </tr> | ||
| """ | ||
|
|
||
|
|
||
| regions = { | ||
| "AF": "Africa", | ||
| "AN": "Antarctica", | ||
| "AS": "Asia", | ||
| "EU": "Europe", | ||
| "NA": "North America", | ||
| "OC": "Oceania", | ||
| "SA": "South and Central America", | ||
| "World": "Globally Available", | ||
| } | ||
|
|
||
|
|
||
| def argparse() -> tuple[Path, Path]: | ||
| if len(argv) != 3: | ||
| print(f"usage: {argv[0]} <sourcefile> <destdir>", file=stderr) | ||
| raise SystemExit(1) | ||
|
|
||
| srcfile = Path(argv[1]) | ||
| if not srcfile.is_file(): | ||
| print(f"{argv[0]}: {srcfile} does not exist or is not a file", file=stderr) | ||
| print(f"usage: {argv[0]} <sourcefile> <destdir>", file=stderr) | ||
| raise SystemExit(1) | ||
|
|
||
| destdir = Path(argv[2]) | ||
| if destdir.exists() and not destdir.is_dir(): | ||
| print(f"{argv[0]}: {destdir} exists but is not a directory", file=stderr) | ||
| print(f"usage: {argv[0]} <sourcefile> <destdir>", file=stderr) | ||
| raise SystemExit(1) | ||
|
|
||
| return (srcfile, destdir) | ||
|
|
||
|
|
||
| def write_api(destdir: Path, data: list[dict]): | ||
| apidir = destdir / "v0" | ||
| apidir.mkdir(parents=True) | ||
| with (apidir / "mirrors.json").open("w") as f: | ||
| json.dump(data, f) | ||
|
|
||
|
|
||
| def write_metrics(destdir: Path, data: list[dict]): | ||
| """ | ||
| generate files for assessing mirror status with prometheus | ||
| """ | ||
| def prom_targets(data: list[dict]) -> list[dict]: | ||
| """ | ||
| transforms the mirror list into a prometheus target list format | ||
| https://prometheus.io/docs/prometheus/latest/http_sd/ | ||
| """ | ||
| tgts = sorted(re.sub(r"^(?:http|ftp)s?://(.*?)/?$", r"\1/current", m["base_url"]) for m in data) | ||
| return [{"targets": tgts}] | ||
|
|
||
| metricsdir = destdir / "metrics" | ||
| metricsdir.mkdir(parents=True) | ||
| promdata = prom_targets(data) | ||
| with (metricsdir / "prometheus.json").open("w") as f: | ||
| json.dump(promdata, f) | ||
|
|
||
| def write_raw(destdir: Path, data: list[dict]): | ||
| """ | ||
| generate the tab-separated value file for bash xmirror | ||
| """ | ||
| rawdir = destdir / "raw" | ||
| rawdir.mkdir(parents=True) | ||
| with (rawdir / "mirrors.lst").open("w") as f: | ||
| wr = csv.writer(f, dialect="excel-tab") | ||
| wr.writerows([[m["region"], m["base_url"], m["location"], m["tier"]] for m in data if m["enabled"]]) | ||
|
|
||
|
|
||
| def write_html(destdir: Path, data: list[dict]): | ||
| """ | ||
| write the list of mirrors into 2 tables, one for each tier, and insert them in the html document | ||
| """ | ||
| def reg(r: str) -> str: | ||
| """get a pretty name for a region""" | ||
| if r in regions.keys(): | ||
| return regions[r] | ||
| return r | ||
|
|
||
| tier1 = tabletmpl.format(rows="\n".join( | ||
| [rowtmpl.format(url=m["base_url"], region=reg(m["region"]), location=m["location"]) | ||
| for m in data if m["tier"] == 1 and m["enabled"]] | ||
| )) | ||
| tier2 = tabletmpl.format(rows="\n".join( | ||
| [rowtmpl.format(url=m["base_url"], region=reg(m["region"]), location=m["location"]) | ||
| for m in data if m["tier"] == 2 and m["enabled"]] | ||
| )) | ||
| with (destdir / "index.html").open("w") as f: | ||
| f.write(html.format(tier1=tier1, tier2=tier2)) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| srcfile, destdir = argparse() | ||
|
|
||
| # clean up | ||
| if destdir.exists(): | ||
| rmtree(destdir) | ||
|
|
||
| # load data | ||
| with srcfile.open() as f: | ||
| data = yaml.safe_load(f.read()) | ||
|
|
||
| # write v0 api json | ||
| write_api(destdir, data) | ||
|
|
||
| # write prometheus json | ||
| write_metrics(destdir, data) | ||
|
|
||
| # write raw tsv data | ||
| write_raw(destdir, data) | ||
|
|
||
| # write html table | ||
| write_html(destdir, data) |
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.