From d64a46dc887296fa6515f0dccc730a6e22c269d6 Mon Sep 17 00:00:00 2001 From: crop Date: Mon, 22 Jul 2024 00:17:46 +0200 Subject: [PATCH 1/7] Add support for Forgejo --- README.md | 1 + src/cli.rs | 59 +++++++++++++++++++++++++++++++++++++++++ src/git.rs | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+) diff --git a/README.md b/README.md index cb09684..c93ce74 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,7 @@ OPTIONS: SUBCOMMANDS: channel Track a Nix channel + forgejo Track a Forgejo repository git Track a git repository github Track a GitHub repository gitlab Track a GitLab repository diff --git a/src/cli.rs b/src/cli.rs index ac1ab76..855fa1f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -140,6 +140,61 @@ impl GitHubAddOpts { } } +#[derive(Debug, StructOpt)] +pub struct ForgejoAddOpts { + pub server: String, + pub owner: String, + pub repository: String, + + #[structopt(flatten)] + pub more: GenericGitAddOpts, +} +impl ForgejoAddOpts { + pub fn add(&self) -> Result<(String, Pin)> { + let server_url = match Url::parse(&self.server) { + Ok(url) => url, + Err(firsterror) => match Url::parse(&("https://".to_string() + self.server.as_str())) { + Ok(url) => url, + Err(_seconderror) => return Err(firsterror.into()), + }, + }; + + Ok(( + self.repository.clone(), + match &self.more.branch { + Some(branch) => { + let pin = git::GitPin::forgejo( + server_url, + &self.owner, + &self.repository, + branch.clone(), + self.more.submodules, + ); + let version = self.more.at.as_ref().map(|at| git::GitRevision { + revision: at.clone(), + }); + (pin, version).into() + }, + None => { + let pin = git::GitReleasePin::forgejo( + server_url, + &self.owner, + &self.repository, + self.more.pre_releases, + self.more.version_upper_bound.clone(), + self.more.release_prefix.clone(), + self.more.submodules, + ); + let version = self.more.at.as_ref().map(|at| GenericVersion { + version: at.clone(), + }); + (pin, version).into() + }, + }, + )) + } +} + #[derive(Debug, StructOpt)] pub struct GitLabAddOpts { /// Usually just `"owner" "repository"`, but GitLab allows arbitrary folder-like structures. @@ -308,6 +363,9 @@ pub enum AddCommands { /// Track a GitHub repository #[structopt(name = "github")] GitHub(GitHubAddOpts), + /// Track a Forgejo repository + #[structopt(name = "forgejo")] + Forgejo(ForgejoAddOpts), /// Track a GitLab repository #[structopt(name = "gitlab")] GitLab(GitLabAddOpts), @@ -338,6 +396,7 @@ impl AddOpts { AddCommands::Channel(c) => c.add()?, AddCommands::Git(g) => g.add()?, AddCommands::GitHub(gh) => gh.add()?, + AddCommands::Forgejo(fg) => fg.add()?, AddCommands::GitLab(gl) => gl.add()?, AddCommands::PyPi(p) => p.add()?, }; diff --git a/src/git.rs b/src/git.rs index 41e688c..8588645 100644 --- a/src/git.rs +++ b/src/git.rs @@ -84,6 +84,11 @@ pub enum Repository { /// URL to the Git repository url: Url, }, + Forgejo { + server: Url, + owner: String, + repo: String, + }, GitHub { /// "owner/repo" owner: String, @@ -111,6 +116,11 @@ impl Repository { Repository::GitHub { owner, repo } => { format!("{}/{}/{}.git", get_github_url(), owner, repo).parse()? }, + Repository::Forgejo { + server, + owner, + repo, + } => format!("{}/{}/{}.git", server, owner, repo).parse()?, Repository::GitLab { repo_path, server, @@ -143,6 +153,20 @@ impl Repository { ) .parse()?, ), + Repository::Forgejo { + server, + owner, + repo, + } => Some( + format!( + "{server}/{owner}/{repo}/archive/{revision}.tar.gz", + server = server, + owner = owner, + repo = repo, + revision = revision, + ) + .parse()?, + ), Repository::GitLab { repo_path, server, @@ -185,6 +209,20 @@ impl Repository { ) .parse()?, ), + Repository::Forgejo { + server, + owner, + repo, + } => Some( + format!( + "{server}/api/v1/repos/{owner}/{repo}/archive/{tag}.tar.gz", + server = server, + owner = owner, + repo = repo, + tag = tag, + ) + .parse()?, + ), Repository::GitLab { repo_path, server, @@ -262,6 +300,24 @@ impl GitPin { } } + pub fn forgejo( + server: Url, + owner: impl Into, + repo: impl Into, + branch: String, + submodules: bool, + ) -> Self { + Self { + repository: Repository::Forgejo { + server, + owner: owner.into(), + repo: repo.into(), + }, + branch, + submodules, + } + } + pub fn gitlab( repo_path: String, branch: String, @@ -413,6 +469,28 @@ impl GitReleasePin { } } + pub fn forgejo( + server: Url, + owner: impl Into, + repo: impl Into, + pre_releases: bool, + version_upper_bound: Option, + release_prefix: Option, + submodules: bool, + ) -> Self { + Self { + repository: Repository::Forgejo { + server, + owner: owner.into(), + repo: repo.into(), + }, + pre_releases, + version_upper_bound, + release_prefix, + submodules, + } + } + pub fn gitlab( repo_path: String, server: Option, From 7bf32efd8770cbc50931066450e860c1dda0b979 Mon Sep 17 00:00:00 2001 From: crop Date: Fri, 26 Jul 2024 17:59:00 +0200 Subject: [PATCH 2/7] add nix tests for forgejo --- test.nix | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/test.nix b/test.nix index 20ffe63..af4bda8 100644 --- a/test.nix +++ b/test.nix @@ -132,6 +132,78 @@ let touch $out ''; + mkForgejoTest = + { + name, + commands, + # Repositories to host. key = repo path, value = repo derivation + repositories, + # For simplicity, all fake releases will be added to all repositories, + # and both as "archive" (for refs) and as "tarball" (for releases) + apiTarballs ? [ ], + }: + pkgs.runCommand name + { + nativeBuildInputs = with pkgs; [ + npins + python3 + netcat + nix + gitMinimal + jq + ]; + } + '' + set -euo pipefail + export HOME=$TMPDIR + export NIX_STATE_DIR=$TMPDIR + export NIX_DATA_DIR=$TMPDIR + export NIX_STORE_DIR=$TMPDIR + export NIX_LOG_DIR=$TMPDIR + source ${prelude} + + echo "Running test ${name}" + cd $(mktemp -d) + + # Mock the repositories + ${lib.pipe repositories [ + (lib.mapAttrsToList ( + repoPath: gitRepo: '' + mkdir -p $(dirname ${repoPath}) + ln -s ${gitRepo} "${repoPath}.git" + + # Mock the releases + tarballPath="api/v1/repos/${repoPath}/archive" + mkdir -p $tarballPath + archivePath="${repoPath}/archive" + mkdir -p $archivePath + ${lib.concatMapStringsSep "\n" (path: '' + ln -s ${testTarball} $tarballPath/${path}.tar.gz + ln -s ${testTarball} $archivePath/${path} + '') apiTarballs} + + chmod -R +rw $archivePath + chmod -R +rw $tarballPath + pwd + ls -la $tarballPath + # For each of the commits in the repo create the tarballs + git config --global --add safe.directory ${gitRepo} + echo $(git -C ${gitRepo} log --oneline --format="format:%H") + git -C ${gitRepo} log --oneline --format="format:%H" | xargs -I XX -n1 git -C ${gitRepo} archive -o $PWD/$tarballPath/XX.tar.gz XX + git -C ${gitRepo} log --oneline --format="format:%H" | xargs -I XX -n1 git -C ${gitRepo} archive -o $PWD/$archivePath/XX.tar.gz XX + '' + )) + (lib.concatStringsSep "\n") + ]} + + python -m http.server 8000 & + timeout 30 sh -c 'set -e; until nc -z 127.0.0.1 8000; do sleep 1; done' || exit 1 + + ${commands} + + touch $out + ''; + mkGithubTest = { name, @@ -264,6 +336,75 @@ in ''; }; + # maybe test using forgejo? https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/forgejo.nix + forgejoRelease = mkForgejoTest { + name = "forgejo-release"; + repositories."foo/bar" = gitRepo; + apiTarballs = [ "v0.2" ]; + commands = '' + npins init --bare + npins add forgejo http://localhost:8000 foo bar + nix-instantiate --eval npins -A bar.outPath + + # Check version and url + eq "$(jq -r .pins.bar.version npins/sources.json)" "v0.2" + eq "$(jq -r .pins.bar.revision npins/sources.json)" "$(resolveGitCommit ${gitRepo} v0.2)" + eq "$(jq -r .pins.bar.url npins/sources.json)" "http://localhost:8000/api/v1/repos/foo/bar/archive/v0.2.tar.gz" + ''; + }; + + forgejoBranch = mkForgejoTest { + name = "forgejo-branch"; + repositories."foo/bar" = gitRepo; + apiTarballs = [ "v0.2" ]; + commands = '' + npins init --bare + npins add forgejo http://localhost:8000 foo bar --branch test-branch + nix-instantiate --eval npins -A bar.outPath + + # Check version and url + eq "$(jq -r .pins.bar.version npins/sources.json)" "null" + eq "$(jq -r .pins.bar.revision npins/sources.json)" "$(resolveGitCommit ${gitRepo} test-branch)" + eq "$(jq -r .pins.bar.url npins/sources.json)" "http://localhost:8000/foo/bar/archive/$(resolveGitCommit ${gitRepo} test-branch).tar.gz" + ''; + }; + + forgejoSubmodule = mkForgejoTest rec { + name = "forgejo-submodule"; + apiTarballs = [ "cbbbea814edccc7bf23af61bd620647ed7c0a436" ]; + repositories."owner/bar" = gitRepo; + repositories."owner/foo" = mkGitRepo { + name = "repo-with-submodules"; + extraCommands = '' + git submodule init + + # In order to be able to add the submodule, we need to fake host it + cd .. + ${pkgs.python3}/bin/python -m http.server 8000 & + timeout 30 sh -c 'set -e; until ${pkgs.netcat}/bin/nc -z 127.0.0.1 8000; do sleep 1; done' || exit 1 + mkdir owner + ln -s ${repositories."owner/bar"} "owner/bar.git" + cd tmp + + git submodule add "http://localhost:8000/owner/bar.git" + ''; + }; + + commands = '' + npins init --bare + npins add forgejo http://localhost:8000 owner foo --branch main + npins add --name foo2 forgejo http://localhost:8000 owner foo --branch main --submodules + + # Both have the same revision, but only foo has an URL + eq "$(jq -r .pins.foo.version npins/sources.json)" "null" + eq "$(jq -r .pins.foo2.version npins/sources.json)" "null" + eq "$(jq -r .pins.foo.revision npins/sources.json)" "$(resolveGitCommit ${repositories."owner/foo"})" + eq "$(jq -r .pins.foo2.revision npins/sources.json)" "$(resolveGitCommit ${repositories."owner/foo"})" + eq "$(jq -r .pins.foo.url npins/sources.json)" "http://localhost:8000/owner/foo/archive/$(resolveGitCommit ${repositories."owner/foo"}).tar.gz" + eq "$(jq -r .pins.foo2.url npins/sources.json)" "null" + ''; + }; + githubRelease = mkGithubTest { name = "github-release"; repositories."foo/bar" = gitRepo; From 96d0ee966927b7f7fbf791e6fe32629092597e68 Mon Sep 17 00:00:00 2001 From: crop Date: Fri, 26 Jul 2024 17:59:11 +0200 Subject: [PATCH 3/7] minor fix for forgejo --- src/git.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/git.rs b/src/git.rs index 8588645..a89e151 100644 --- a/src/git.rs +++ b/src/git.rs @@ -159,7 +159,7 @@ impl Repository { repo, } => Some( format!( - "{server}/{owner}/{repo}/archive/{revision}.tar.gz", + "{server}{owner}/{repo}/archive/{revision}.tar.gz", server = server, owner = owner, repo = repo, @@ -215,7 +215,7 @@ impl Repository { repo, } => Some( format!( - "{server}/api/v1/repos/{owner}/{repo}/archive/{tag}.tar.gz", + "{server}api/v1/repos/{owner}/{repo}/archive/{tag}.tar.gz", server = server, owner = owner, repo = repo, From 5ce63a1792eedbb4427808a33f559fef7e2b01a8 Mon Sep 17 00:00:00 2001 From: crop Date: Fri, 26 Jul 2024 18:50:49 +0200 Subject: [PATCH 4/7] add rust tests for forgejo --- src/git.rs | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/src/git.rs b/src/git.rs index a89e151..fce3314 100644 --- a/src/git.rs +++ b/src/git.rs @@ -1021,6 +1021,69 @@ mod test { Ok(()) } + #[tokio::test] + async fn test_forgejo_update() -> Result<()> { + let pin = GitPin { + repository: Repository::Forgejo { + server: "https://git.lix.systems".parse().unwrap(), + owner: "lix-project".into(), + repo: "lix".into(), + }, + branch: "release-2.90".into(), + submodules: false, + }; + let version = pin.update(None).await?; + assert_eq!( + version, + GitRevision { + revision: "4bbdb2f5564b9b42bcaf0e1eec28325300f31c72".into(), + } + ); + assert_eq!( + pin.fetch(&version).await?, + OptionalUrlHashes { + url: Some("https://git.lix.systems/lix-project/lix/archive/4bbdb2f5564b9b42bcaf0e1eec28325300f31c72.tar.gz".parse().unwrap()), + hash: "03rygh7i9wzl6mhha6cv5q26iyzwy8l59d5cq4r6j5kpss9l1hn3".into(), + } + ); + Ok(()) + } + + #[tokio::test] + async fn test_forgejo_release_update() -> Result<()> { + let pin = GitReleasePin { + repository: Repository::Forgejo { + server: "https://git.lix.systems".parse().unwrap(), + owner: "lix-project".into(), + repo: "lix".into(), + }, + pre_releases: false, + version_upper_bound: None, + release_prefix: None, + submodules: false, + }; + let version = pin.update(None).await?; + assert_eq!( + version, + GenericVersion { + version: "2.90.0".into(), + } + ); + assert_eq!( + pin.fetch(&version).await?, + ReleasePinHashes { + revision: "2a4376be20d70feaa2b0e640c5041fb66ddc67ed".into(), + url: Some( + "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/2.90.0.tar.gz" + .parse() + .unwrap() + ), + hash: "1iyylsiv1n6mf6rbi4k4fm5nv24a940cwfz92gk9fx6axh2kxjbz".into(), + } + ); + Ok(()) + } + #[tokio::test] async fn test_gitlab_update() -> Result<()> { let pin = GitPin { From f07f8db8bbe415683767ef7526de93b200c6e633 Mon Sep 17 00:00:00 2001 From: crop Date: Mon, 12 Aug 2024 00:58:22 +0200 Subject: [PATCH 5/7] Apply suggestions from code review Co-authored-by: piegames --- src/cli.rs | 12 +++++------- src/git.rs | 8 -------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 855fa1f..7a54421 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -151,13 +151,11 @@ pub struct ForgejoAddOpts { } impl ForgejoAddOpts { pub fn add(&self) -> Result<(String, Pin)> { - let server_url = match Url::parse(&self.server) { - Ok(url) => url, - Err(firsterror) => match Url::parse(&("https://".to_string() + self.server.as_str())) { - Ok(url) => url, - Err(_seconderror) => return Err(firsterror.into()), - }, - }; + let server_url = Url::parse(&self.server) + .or_else(|err| + Url::parse(&("https://".to_string() + self.server.as_str()) + .map_err(|_| err) + ) Ok(( self.repository.clone(), diff --git a/src/git.rs b/src/git.rs index fce3314..05da194 100644 --- a/src/git.rs +++ b/src/git.rs @@ -160,10 +160,6 @@ impl Repository { } => Some( format!( "{server}{owner}/{repo}/archive/{revision}.tar.gz", - server = server, - owner = owner, - repo = repo, - revision = revision, ) .parse()?, ), @@ -216,10 +212,6 @@ impl Repository { } => Some( format!( "{server}api/v1/repos/{owner}/{repo}/archive/{tag}.tar.gz", - server = server, - owner = owner, - repo = repo, - tag = tag, ) .parse()?, ), From 6b62650a4fe8247e0a75b6f9ccfb6e9142138e83 Mon Sep 17 00:00:00 2001 From: crop Date: Mon, 12 Aug 2024 02:14:31 +0200 Subject: [PATCH 6/7] fix error msg --- src/cli.rs | 13 +++++++------ src/git.rs | 16 ++++------------ test.nix | 21 ++++++--------------- 3 files changed, 17 insertions(+), 33 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 7a54421..adcc629 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -7,7 +7,7 @@ use std::io::Write; use anyhow::{Context, Result}; use structopt::StructOpt; -use url::Url; +use url::{ParseError, Url}; const DEFAULT_NIX: &'static str = include_str!("default.nix"); @@ -151,11 +151,12 @@ pub struct ForgejoAddOpts { } impl ForgejoAddOpts { pub fn add(&self) -> Result<(String, Pin)> { - let server_url = Url::parse(&self.server) - .or_else(|err| - Url::parse(&("https://".to_string() + self.server.as_str()) - .map_err(|_| err) - ) + let server_url = Url::parse(&self.server).or_else(|err| match err { + ParseError::RelativeUrlWithoutBase => { + Url::parse(&("https://".to_string() + self.server.as_str())) + }, + _ => Err(err), + })?; Ok(( self.repository.clone(), diff --git a/src/git.rs b/src/git.rs index 05da194..ade9300 100644 --- a/src/git.rs +++ b/src/git.rs @@ -157,12 +157,7 @@ impl Repository { server, owner, repo, - } => Some( - format!( - "{server}{owner}/{repo}/archive/{revision}.tar.gz", - ) - .parse()?, - ), + } => Some(format!("{server}{owner}/{repo}/archive/{revision}.tar.gz",).parse()?), Repository::GitLab { repo_path, server, @@ -209,12 +204,9 @@ impl Repository { server, owner, repo, - } => Some( - format!( - "{server}api/v1/repos/{owner}/{repo}/archive/{tag}.tar.gz", - ) - .parse()?, - ), + } => { + Some(format!("{server}api/v1/repos/{owner}/{repo}/archive/{tag}.tar.gz",).parse()?) + }, Repository::GitLab { repo_path, server, diff --git a/test.nix b/test.nix index af4bda8..df75e7c 100644 --- a/test.nix +++ b/test.nix @@ -7,6 +7,12 @@ let # utility bash functions used throught the tests prelude = pkgs.writeShellScript "prelude" '' + export HOME=$TMPDIR + export NIX_STATE_DIR=$TMPDIR + export NIX_DATA_DIR=$TMPDIR + export NIX_STORE_DIR=$TMPDIR + export NIX_LOG_DIR=$TMPDIR + function eq() { local a=$1 local b=$2 @@ -98,11 +104,6 @@ let } '' set -euo pipefail - export HOME=$TMPDIR - export NIX_STATE_DIR=$TMPDIR - export NIX_DATA_DIR=$TMPDIR - export NIX_STORE_DIR=$TMPDIR - export NIX_LOG_DIR=$TMPDIR source ${prelude} echo -e "\n\nRunning test ${name}\n" @@ -155,11 +156,6 @@ let } '' set -euo pipefail - export HOME=$TMPDIR - export NIX_STATE_DIR=$TMPDIR - export NIX_DATA_DIR=$TMPDIR - export NIX_STORE_DIR=$TMPDIR - export NIX_LOG_DIR=$TMPDIR source ${prelude} echo "Running test ${name}" @@ -227,11 +223,6 @@ let } '' set -euo pipefail - export HOME=$TMPDIR - export NIX_STATE_DIR=$TMPDIR - export NIX_DATA_DIR=$TMPDIR - export NIX_STORE_DIR=$TMPDIR - export NIX_LOG_DIR=$TMPDIR export NPINS_GITHUB_HOST=http://localhost:8000 export NPINS_GITHUB_API_HOST=http://localhost:8000/api source ${prelude} From 722389031204ec22f407c154dfbac61b01899756 Mon Sep 17 00:00:00 2001 From: crop Date: Sun, 8 Sep 2024 23:24:16 +0200 Subject: [PATCH 7/7] fix forgejo rust test by setting upper version bound --- src/git.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/git.rs b/src/git.rs index ade9300..fb9d8a9 100644 --- a/src/git.rs +++ b/src/git.rs @@ -1042,7 +1042,7 @@ mod test { repo: "lix".into(), }, pre_releases: false, - version_upper_bound: None, + version_upper_bound: Some("2.90.1".to_string()), release_prefix: None, submodules: false, };