Skip to content
This repository has been archived by the owner on Jan 14, 2025. It is now read-only.

Commit

Permalink
first commit for rclone
Browse files Browse the repository at this point in the history
  • Loading branch information
funatsufumiya committed Sep 2, 2023
1 parent 6089e66 commit 074dcc4
Show file tree
Hide file tree
Showing 7 changed files with 34 additions and 33 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
git-lfs-agent-scp
git-lfs-agent-rclone
tmp/
logs/
*.log
Expand Down
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[package]
name = "git-lfs-agent-scp"
name = "git-lfs-agent-rclone"
version = "0.1.0"
edition = "2021"

Expand All @@ -11,5 +11,5 @@ serde_json = "1.0.105"
tempfile = "3.8.0"

[[bin]]
name = "git-lfs-agent-scp"
name = "git-lfs-agent-rclone"
path = "main.rs"
19 changes: 9 additions & 10 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,24 +1,23 @@
# `git-lfs-agent-scp`
# `git-lfs-agent-rclone`

A custom transfer agent for [`git-lfs`](https://git-lfs.github.com/) that uses [`scp`](https://www.openssh.com/) to transfer files.
This transfer agent makes it possible to use `git-lfs` in situations where the remote only speaks `ssh`.
A custom transfer agent for [`git-lfs`](https://git-lfs.github.com/) that uses [`rclone`](https://www.openssh.com/) to transfer files.
This is useful if you do not want to install a `git-lfs` server.

## Usage

Configure your local git repository as follows

```sh
$ git config lfs.standalonetransferagent scp <1>
$ git config lfs.customtransfer.scp.path git-lfs-agent-scp <2>
$ git config lfs.customtransfer.scp.args $DESTINATION <3>
$ git config lfs.standalonetransferagent rclone <1>
$ git config lfs.customtransfer.rclone.path git-lfs-agent-rclone <2>
$ git config lfs.customtransfer.rclone.args $DESTINATION <3>
```
1. tell `git-lfs` to use the transfer agent named "scp"
1. tell `git-lfs` to use the transfer agent named "rclone"
2. tell `git-lfs` what the name of the program is of the transfer agent
3. `$DESTINATION` is the destination to which `scp` will copy files tracked by `git-lfs` when running `$ git pull` and the place it will store files when running `$ git push`
3. `$DESTINATION` is the destination to which `rclone` will copy files tracked by `git-lfs` when running `$ git pull` and the place it will store files when running `$ git push`

- NOTE: `$DESTINATION` can be set to anything `scp` understands.
As an example, `server.example.com:/home/git/my-lfs-files` ships files to a remote server over `ssh`.
- NOTE: `$DESTINATION` can be set to anything `rclone` understands.
As an example, `source:/home/git/my-lfs-files` ships files to a remote server over `rclone`.

## Install

Expand Down
8 changes: 4 additions & 4 deletions integration-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,11 @@ $git add --all
$git commit --message 'commit message'
# Add $DEST_GIT as a remote to the source repository
$git remote add origin $DEST_GIT
# Configure the source repository to use git-lfs-agent-scp to transfer files
$git config lfs.standalonetransferagent scp
# Configure the source repository to use git-lfs-agent-rclone to transfer files
$git config lfs.standalonetransferagent rclone
$git config lfs.concurrenttransfers 1
$git config lfs.customtransfer.scp.path $mydir/git-lfs-agent-scp
$git config lfs.customtransfer.scp.args $DEST_GIT_LFS_FILES
$git config lfs.customtransfer.rclone.path $mydir/git-lfs-agent-rclone
$git config lfs.customtransfer.rclone.args $DEST_GIT_LFS_FILES

#
# Test uploading.
Expand Down
22 changes: 12 additions & 10 deletions main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,8 @@ fn get_event (ev: &Event) -> String {
fn main() {
// get args
let args: Vec<String> = env::args().collect();
let scp_args = &args[1..];
let scp_arg_str = scp_args.join(" ");
let rclone_args = &args[1..];
let rclone_arg_str = rclone_args.join(" ");

// read stdin
let stdin = io::stdin();
Expand All @@ -167,12 +167,13 @@ fn main() {
Event::Upload(ref e) => e,
_ => panic!("invalid event"),
};
// call scp to upload file
// call rclone to upload file
let src_path = &ev.path;
// let dst_path = Path::new(&scp_arg_str).join(&ev.oid);
// let dst_path = Path::new(&rclone_arg_str).join(&ev.oid);
let sep = "/";
let dst_path = format!("{}{}{}", &scp_arg_str, sep, &ev.oid);
let cmd = Command::new("scp")
let dst_path = format!("{}{}{}", &rclone_arg_str, sep, &ev.oid);
let cmd = Command::new("rclone")
.arg("copy")
.arg(src_path)
.arg(dst_path)
.output()
Expand Down Expand Up @@ -201,14 +202,15 @@ fn main() {
Event::Download(e) => e,
_ => panic!("invalid event"),
};
// call scp to download file
// call rclone to download file
let (_tmp_file, tmp_path_buf) = tempfile::NamedTempFile::new().unwrap().keep().unwrap();
let tmp_path = tmp_path_buf.to_str().unwrap();

let sep = "/";
// let src_path = Path::new(&scp_arg_str).join(&ev.oid);
let src_path = format!("{}{}{}", &scp_arg_str, sep, &ev.oid);
let cmd = Command::new("scp")
// let src_path = Path::new(&rclone_arg_str).join(&ev.oid);
let src_path = format!("{}{}{}", &rclone_arg_str, sep, &ev.oid);
let cmd = Command::new("rclone")
.arg("copy")
.arg(src_path)
.arg(tmp_path)
.output()
Expand Down
10 changes: 5 additions & 5 deletions test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@ touch $SRC/0000000000000000000000000000000000000000000000000000000000000000 \
$DEST/0000000000000000000000000000000000000000000000000000000000000006 \
$DEST/0000000000000000000000000000000000000000000000000000000000000007 \

rm -f /tmp/git-lfs-agent-scp-*
rm -f /tmp/git-lfs-agent-rclone-*

cat <<EOF | ./git-lfs-agent-scp 2>$output 1>&2 $DEST
cat <<EOF | ./git-lfs-agent-rclone 2>$output 1>&2 $DEST
{"event":"init"}
{"event":"upload","oid":"0000000000000000000000000000000000000000000000000000000000000000","path":"$SRC/0000000000000000000000000000000000000000000000000000000000000000"}
{"event":"download","oid":"0000000000000000000000000000000000000000000000000000000000000004"}
Expand All @@ -27,7 +27,7 @@ cat <<EOF | ./git-lfs-agent-scp 2>$output 1>&2 $DEST
{"event":"terminate"}
EOF

cat <<EOF | ./git-lfs-agent-scp 2>$output 1>&2 $DEST/
cat <<EOF | ./git-lfs-agent-rclone 2>$output 1>&2 $DEST/
{"event":"init"}
{"event":"upload","oid":"0000000000000000000000000000000000000000000000000000000000000002","path":"$SRC/0000000000000000000000000000000000000000000000000000000000000002"}
{"event":"download","oid":"0000000000000000000000000000000000000000000000000000000000000006"}
Expand All @@ -37,8 +37,8 @@ cat <<EOF | ./git-lfs-agent-scp 2>$output 1>&2 $DEST/
EOF

# Move the tmp files (git-lfs takes care of this in practice)
for file in /tmp/git-lfs-agent-scp-*; do
mv $file $SRC/${file#/tmp/git-lfs-agent-scp-}
for file in /tmp/git-lfs-agent-rclone-*; do
mv $file $SRC/${file#/tmp/git-lfs-agent-rclone-}
done

diff $SRC $DEST
Expand Down

0 comments on commit 074dcc4

Please sign in to comment.