add e2e test script and docker support

Signed-off-by: Dweb Fan <[email protected]>

.github/workflows/main.yml

@@ -44,4 +44,4 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: 'true'
-      - run: make test
+      - run: make unit-tests

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test/data"]
+	path = test/data
+	url = https://github.com/lomorage/homepage

Makefile

@@ -2,6 +2,10 @@
 
 SHELL=/bin/bash # Use bash syntax
 GOPATH=/go
+USER_ID := $(shell id -u)
+USER_NAME := $(shell whoami)
+GROUP_ID := $(shell id -g)
+GROUP_NAME := $(shell id -gn)
 
 vendor:
 	go mod vendor
@@ -12,18 +16,32 @@ lint:
 install:
 	go install -v ./cmd/...
 
+localstack-tarball:
+	docker pull localstack/localstack:3.4.0
+	docker save -o localstack_3.4.0.tar localstack/localstack:3.4.0
+	gzip localstack_3.4.0.tar
+
 dev-container:
-	docker build --tag "lomorage/lomo-backup:1.0" -f dockerfiles/dev-image .
+	docker build --tag "lomorage/lomo-backup:build-stage1" -f dockerfiles/dev-image .
+	docker build --tag "lomorage/lomo-backup:build-stage2" -f dockerfiles/dev-image-load .
+	docker rm -f lomo-backup-build
+	docker run --name lomo-backup-build --privileged --net host -it "lomorage/lomo-backup:build-stage2" hostname
+	docker commit lomo-backup-build lomorage/lomo-backup:1.0
 
 dev:
-	docker build --tag "lomorage/lomo-backup" -f dockerfiles/dev-run .
+	docker build \
+	    --build-arg USER_ID=$(USER_ID) --build-arg USER_NAME=$(USER_NAME) \
+	    --build-arg GROUP_ID=$(GROUP_ID) --build-arg GROUP_NAME=$(GROUP_NAME) \
+		--tag "lomorage/lomo-backup" -f dockerfiles/dev-run .
 	docker rm -f lomo-backup
-	docker run \
-		--name lomo-backup --hostname lomo-backup \
-		--privileged --cap-add=ALL -v /dev:/dev -v /lib/modules:/lib/modules \
+	docker run --rm \
+		--name lomo-backup --hostname lomo-backup -v ./dockerfiles/hosts:/etc/hosts \
+		--privileged --net host --dns-search local \
 		-v "${PWD}:/go/src/github.com/lomorage/lomo-backup" \
-		--net host --dns-search local \
 		-it "lomorage/lomo-backup" -d bash
 
-test:
-	go test -v ./common/
+unit-tests:
+	go list ./common/... | xargs -I {} go test -v {}
+
+e2e-tests:
+	cd test/scripts; ./e2e.sh

clients/upload.go

@@ -3,6 +3,7 @@ package clients
 import (
 	"context"
 	"io"
+	"os"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -70,6 +71,10 @@ func NewAWSClient(keyID, key, region string) (*AWSClient, error) {
 	if err != nil {
 		return nil, err
 	}
+	if os.Getenv("LOCALSTACK_ENDPOINT") != "" {
+		cfg.S3ForcePathStyle = aws.Bool(true)
+		cfg.Endpoint = aws.String(os.Getenv("LOCALSTACK_ENDPOINT"))
+	}
 	return &AWSClient{region: region, svc: s3.New(sess, cfg)}, nil
 }
 

cmd/lomob/upload-files.go

@@ -224,6 +224,7 @@ func uploadFilesToS3(ctx *cli.Context) error {
 	}
 
 	for _, name := range ctx.Args() {
+		name = filepath.Clean(name)
 		fmt.Printf("Uploading file %s\n", name)
 
 		if masterKey == "" {

cmd/lomob/upload-iso.go

@@ -567,7 +567,7 @@ func uploadISOs(ctx *cli.Context) error {
 
 	for _, isoFilename := range ctx.Args() {
 		err = uploadISO(accessKeyID, secretAccessKey, region, bucket, storageClass,
-			isoFilename, masterKey, partSize, saveParts, force)
+			filepath.Clean(isoFilename), masterKey, partSize, saveParts, force)
 		if err != nil {
 			return err
 		}

dockerfiles/dev-image

@@ -4,8 +4,7 @@ ENV DEBIAN_FRONTEND noninteractive
 ENV GOPATH=/go
 
 RUN apt-get -yqq update && apt-get install -yq --no-install-recommends man-db vim build-essential \
-  sqlite3 curl apt-transport-https ca-certificates software-properties-common \
-  && apt-get install -yq sqlite3 \
+  sqlite3 curl apt-transport-https ca-certificates software-properties-common sqlite3 sudo \
   && apt-get clean autoclean && apt-get autoremove -y && rm -rf /var/lib/{apt,dpkg,cache,log}/
 
 # install docker
@@ -17,3 +16,5 @@ RUN apt-get -yqq update && apt-get install -yq docker-ce && apt-get clean autocl
 RUN curl -fsSL https://go.dev/dl/go1.21.0.linux-amd64.tar.gz | tar xz -C /usr/local
 
 RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.55.2
+
+COPY ./localstack_3.4.0.tar.gz /root/

dockerfiles/dev-image-load

@@ -0,0 +1,6 @@
+FROM "lomorage/lomo-backup:build-stage1"
+
+COPY ./dockerfiles/entrypoint-build.sh /entrypoint.sh
+RUN chmod 755 /entrypoint.sh
+
+ENTRYPOINT /entrypoint.sh "$@"

dockerfiles/dev-run

@@ -4,6 +4,26 @@ ENV GOROOT=/usr/local/go
 ENV PATH=$PATH:$GOROOT/bin:$GOPATH/bin
 WORKDIR /go/src/github.com/lomorage/lomo-backup
 
+ARG USER_ID
+ARG USER_NAME
+ARG GROUP_ID
+ARG GROUP_NAME
+
+RUN addgroup --gid $GROUP_ID $GROUP_NAME && \
+    adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID $USER_NAME && \
+    usermod -aG docker $USER_NAME
+
+RUN mkdir $GOPATH/bin && chown -R $USER_ID:$GROUP_ID $GOPATH/bin
+RUN mkdir $GOPATH/pkg && chown -R $USER_ID:$GROUP_ID $GOPATH/pkg
+
+# Add user to sudo group and configure sudo without password
+RUN echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+#COPY ./dockerfiles/localstack.service /etc/systemd/system/multi-user.target.wants
+
 COPY ./dockerfiles/entrypoint.sh /entrypoint.sh
 RUN chmod 755 /entrypoint.sh
+
+USER $USER_NAME
+
 ENTRYPOINT /entrypoint.sh "$@"

dockerfiles/entrypoint-build.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -x
+
+dockerd -s vfs &
+
+# Check if `docker ps` command works, otherwise retry
+while true; do
+    if docker ps; then
+        echo "Docker is running!"
+        break
+    else
+        echo "Docker is not yet ready. Retrying in 5 seconds..."
+        sleep 5
+    fi
+done
+
+cd /root; gunzip localstack_3.4.0.tar.gz; docker load -i localstack_3.4.0.tar
+
+pkill -9 dockerd
+
+rm /var/run/docker.pid
+
+$@

dockerfiles/entrypoint.sh

@@ -2,7 +2,31 @@
 
 set -x
 
-echo user: $USER
-echo uid: $UID
+sudo rm -rf /var/run/docker*
+sudo service docker start
 
-$@
+# Check if `docker ps` command works, otherwise retry
+while true; do
+    if docker ps; then
+        echo "Docker is running!"
+        break
+    else
+        echo "Docker is not yet ready. Retrying in 5 seconds..."
+        sudo service docker start
+        sleep 5
+    fi
+done
+
+docker run --rm -d -p 4566:4566 -p 4510-4559:4510-4559 localstack/localstack:3.4.0 &
+
+while true; do
+    if curl localhost:4566; then
+        echo "Localstack is running!"
+        break
+    else
+        echo "Localstack is not yet ready. Retrying in 5 seconds..."
+        sleep 5
+    fi
+done
+
+$@

dockerfiles/hosts

@@ -0,0 +1 @@
+127.0.0.1 lomo-backup

dockerfiles/localstack.service

@@ -0,0 +1,20 @@
+[Unit]
+Description=Localstack service
+After=network-online.target docker.socket firewalld.service containerd.service time-set.target docker.service
+Wants=network-online.target containerd.service docker.service
+Requires=docker.service
+
+[Service]
+Type=simple
+# the default is not to use systemd for cgroups because the delegate issues still
+# exists and systemd currently does not support the cgroup feature set required
+# for containers run by docker
+ExecStart=docker run --name localstack -p 4566:4566 -p 4510-4559:4510-4559 localstack/localstack:3.4.0
+ExecReload=docker rm -f localstack
+RestartSec=2
+Restart=always
+
+WillMode = process
+
+[Install]
+WantedBy=multi-user.target

test/scripts/e2e.sh

@@ -0,0 +1,20 @@
+sqlite3 ./lomob.db < ../../common/dbx/schema/1.sql
+
+lomob scan -t 1 ../data
+
+expectTotalDirs=58
+expectTotalFiles=201
+
+data=`sqlite3 ./lomob.db "select count(*) from dirs"`
+if [ "$data" = "$expectTotalDirs" ]; then
+  echo "Number of scanned dirs pass"
+else
+  echo "Number of scanned dirs is $data while expect $expectTotalDirs, failed!"
+fi
+
+data=`sqlite3 ./lomob.db "select count(*) from files"`
+if [ "$data" = "$expectTotalFiles" ]; then
+  echo "Number of scanned files pass"
+else
+  echo "Number of scanned file is $data while expect $expectTotalFiles, failed!"
+fi