From 8118c8956667d3d4b11ca7619ad39cf64756bc7e Mon Sep 17 00:00:00 2001 From: l3uddz Date: Thu, 18 Mar 2021 21:23:47 +0000 Subject: [PATCH] feat: web interface, scan-stats and jellyfin and autoscan targets (#103) * build: new build process * refactor: use a cgo-less sqlite * fix(processor): do not process scans when no targets are set * feat(processor): database migrations * feat(targets): autoscan * feat(triggers): manual trigger web interface * feat(targets): jellyfin * feat: scan-stats to display remaining scans --- .github/workflows/build.yml | 124 +++++++++++++++-------- .github/workflows/cleanup.yml | 4 +- .github/workflows/test.yml | 16 --- .golangci.yml | 22 ---- .goreleaser.yml | 33 ++---- Makefile | 41 ++++---- README.md | 61 +++++++++-- cmd/autoscan/main.go | 87 +++++++++++++--- cmd/autoscan/stats.go | 38 +++++++ docker/Dockerfile | 6 +- go.mod | 20 ++-- go.sum | 97 +++++++++++++++--- migrate/migrator.go | 79 +++++++++++++++ migrate/util.go | 119 ++++++++++++++++++++++ processor/datastore.go | 43 +++++--- processor/datastore_test.go | 12 ++- processor/migrations/1_init.sql | 6 ++ processor/processor.go | 22 +++- targets/autoscan/api.go | 101 +++++++++++++++++++ targets/autoscan/autoscan.go | 67 +++++++++++++ targets/emby/api.go | 9 +- targets/jellyfin/api.go | 173 ++++++++++++++++++++++++++++++++ targets/jellyfin/jellyfin.go | 102 +++++++++++++++++++ triggers/bernard/bernard.go | 6 +- triggers/bernard/datastore.go | 4 +- triggers/bernard/paths.go | 6 +- triggers/bernard/postprocess.go | 6 +- triggers/manual/manual.go | 18 +++- triggers/manual/template.html | 80 +++++++++++++++ triggers/middleware.go | 15 ++- 30 files changed, 1211 insertions(+), 206 deletions(-) delete mode 100644 .github/workflows/test.yml delete mode 100644 .golangci.yml create mode 100644 cmd/autoscan/stats.go create mode 100644 migrate/migrator.go create mode 100644 migrate/util.go create mode 100644 processor/migrations/1_init.sql create mode 100644 targets/autoscan/api.go create mode 100644 targets/autoscan/autoscan.go create mode 100644 targets/jellyfin/api.go create mode 100644 targets/jellyfin/jellyfin.go create mode 100644 triggers/manual/template.html diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7ba5a391..6f18acc3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,6 +11,17 @@ jobs: build: runs-on: ubuntu-latest steps: + # dependencies + - name: goreleaser + run: | + curl -sfL https://install.goreleaser.com/github.com/goreleaser/goreleaser.sh | sudo sh -s -- -b /usr/local/bin + + - name: qemu + uses: docker/setup-qemu-action@v1 + + - name: buildx + uses: docker/setup-buildx-action@v1 + # checkout - name: checkout uses: actions/checkout@v2 @@ -21,9 +32,12 @@ jobs: - name: go uses: actions/setup-go@v1 with: - go-version: 1.14 - - run: go version - - run: go env + go-version: 1.16 + + - name: go info + run: | + go version + go env # cache - name: cache @@ -39,24 +53,26 @@ jobs: run: | make vendor + # test + - name: tests + run: | + make test + + # git status + - name: git status + run: git status + # build - name: build if: startsWith(github.ref, 'refs/tags/') == false run: | make snapshot - # get tag name - - name: tag_name - if: startsWith(github.ref, 'refs/tags/') - uses: little-core-labs/get-git-tag@v3.0.2 - with: - tagRegex: "v?(.+)" - # publish - name: publish if: startsWith(github.ref, 'refs/tags/') env: - TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REF: ${{ github.ref }} run: | make publish @@ -74,42 +90,64 @@ jobs: name: build_darwin path: dist/*darwin* + # docker login + - name: docker login + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin + # docker build (latest & tag) - - name: docker - build latest + - name: release tag if: startsWith(github.ref, 'refs/tags/') == true - uses: docker/build-push-action@v1 + uses: little-core-labs/get-git-tag@v3.0.2 + id: releasetag with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - repository: cloudb0x/autoscan - dockerfile: docker/Dockerfile - tags: latest - tag_with_ref: true - tag_with_sha: true - always_pull: true - - # docker build (master) - - name: docker - build master - if: github.ref == 'refs/heads/master' - uses: docker/build-push-action@v1 + tagRegex: "v?(.+)" + + - name: docker - build release + if: startsWith(github.ref, 'refs/tags/') == true + uses: docker/build-push-action@v2 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - repository: cloudb0x/autoscan - dockerfile: docker/Dockerfile - tags: master - tag_with_sha: true - always_pull: true + context: . + file: ./docker/Dockerfile + platforms: linux/amd64,linux/arm64,linux/arm/v7 + pull: true + push: true + tags: | + cloudb0x/autoscan:${{ steps.releasetag.outputs.tag }} + cloudb0x/autoscan:latest # docker build (branch) - - name: docker - build other - if: startsWith(github.ref, 'refs/heads/master') == false - uses: docker/build-push-action@v1 + - name: branch name + if: startsWith(github.ref, 'refs/tags/') == false + id: branch-name + uses: tj-actions/branch-names@v2.2 + + - name: docker tag + if: startsWith(github.ref, 'refs/tags/') == false + uses: frabert/replace-string-action@master + id: dockertag with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - repository: cloudb0x/autoscan - dockerfile: docker/Dockerfile - tag_with_ref: true - tag_with_sha: false - always_pull: true \ No newline at end of file + pattern: '[:\.\/]+' + string: "${{ steps.branch-name.outputs.current_branch }}" + replace-with: '-' + flags: 'g' + + - name: docker - build branch + if: startsWith(github.ref, 'refs/tags/') == false + uses: docker/build-push-action@v2 + with: + context: . + file: ./docker/Dockerfile + platforms: linux/amd64,linux/arm64,linux/arm/v7 + pull: true + push: true + tags: | + cloudb0x/autoscan:${{ steps.dockertag.outputs.replaced }} + + # cleanup + - name: cleanup + run: | + rm -f ${HOME}/.docker/config.json \ No newline at end of file diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index 260d68c6..9901b20d 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -7,7 +7,7 @@ jobs: if: startsWith(github.event.ref_type, 'branch') == true runs-on: ubuntu-latest steps: - - name: Sanitize branch docker tag + - name: docker tag uses: frabert/replace-string-action@master id: dockertag with: @@ -16,7 +16,7 @@ jobs: replace-with: '-' flags: 'g' - - name: Remove branch docker tag + - name: remove docker tag shell: bash env: username: ${{ secrets.DOCKER_USERNAME }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 063c6d5f..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Test - -on: [push, pull_request] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Set up Golang - uses: actions/setup-go@v2 - with: - go-version: '1.x' - - name: Run tests - run: go test ./... \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml deleted file mode 100644 index 06be7d5a..00000000 --- a/.golangci.yml +++ /dev/null @@ -1,22 +0,0 @@ -run: - timeout: 10m - -linters: - enable: - - bodyclose -# - goimports - - unconvert -# - unparam - - scopelint - - dupl - - interfacer - - stylecheck - -issues: - exclude-rules: - - linters: - - stylecheck - text: "ST1003:" - - linters: - - scopelint - text: 'Using the variable on range scope `tc` in function literal' diff --git a/.goreleaser.yml b/.goreleaser.yml index da512007..9c766f1e 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,50 +1,33 @@ # https://goreleaser.com project_name: autoscan -env: - - GO111MODULE=on - - CGO_ENABLED=1 # Build builds: - - id: build_darwin + - env: - - CC=o64-clang - - CXX=o64-clang++ - main: ./cmd/autoscan + - CGO_ENABLED=0 goos: + - linux - darwin - goarch: - - amd64 - ldflags: - - -s -w - - -X "main.Version={{ .Version }}" - - -X "main.GitCommit={{ .ShortCommit }}" - - -X "main.Timestamp={{ .Timestamp }}" - flags: - - -trimpath - - - id: build_linux main: ./cmd/autoscan - goos: - - linux goarch: - amd64 + - arm64 + - arm + goarm: + - 7 ldflags: - - -linkmode external - - -extldflags -static - -s -w - -X "main.Version={{ .Version }}" - -X "main.GitCommit={{ .ShortCommit }}" - -X "main.Timestamp={{ .Timestamp }}" flags: - -trimpath - - -tags=netgo - - -v # Archive archives: - - name_template: "{{ .ProjectName }}_v{{ .Version }}_{{ .Os }}_{{ .Arch }}" + name_template: "{{ .ProjectName }}_v{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" format: "binary" # Checksum diff --git a/Makefile b/Makefile index d1a969cf..c11e97d8 100644 --- a/Makefile +++ b/Makefile @@ -4,12 +4,22 @@ TARGET := $(shell go env GOOS)_$(shell go env GOARCH) DIST_PATH := dist BUILD_PATH := ${DIST_PATH}/${CMD}_${TARGET} GO_FILES := $(shell find . -path ./vendor -prune -or -type f -name '*.go' -print) +HTML_FILES := $(shell find . -path ./vendor -prune -or -type f -name '*.html' -print) +SQL_FILES := $(shell find . -path ./vendor -prune -or -type f -name '*.sql' -print) GIT_COMMIT := $(shell git rev-parse --short HEAD) TIMESTAMP := $(shell date +%s) VERSION ?= 0.0.0-dev -CGO := 1 +CGO := 0 # Deps +.PHONY: check_goreleaser +check_goreleaser: + @command -v goreleaser >/dev/null || (echo "goreleaser is required."; exit 1) + +.PHONY: test +test: ## Run tests + go test ./... -cover -v -race ${GO_PACKAGES} + .PHONY: vendor vendor: ## Vendor files and tidy go.mod go mod vendor @@ -24,7 +34,7 @@ vendor_update: ## Update vendor dependencies build: vendor ${BUILD_PATH}/${CMD} ## Build application # Binary -${BUILD_PATH}/${CMD}: ${GO_FILES} go.sum +${BUILD_PATH}/${CMD}: ${GO_FILES} ${HTML_FILES} ${SQL_FILES} go.sum @echo "Building for ${TARGET}..." && \ mkdir -p ${BUILD_PATH} && \ CGO_ENABLED=${CGO} go build \ @@ -34,25 +44,14 @@ ${BUILD_PATH}/${CMD}: ${GO_FILES} go.sum -o ${BUILD_PATH}/${CMD} \ ./cmd/autoscan +.PHONY: release +release: check_goreleaser ## Generate a release, but don't publish + goreleaser --skip-validate --skip-publish --rm-dist + .PHONY: publish -publish: ## Generate a release, and publish - docker run --rm --privileged \ - -e GITHUB_TOKEN="${TOKEN}" \ - -e VERSION="${GIT_TAG_NAME}" \ - -e GIT_COMMIT="${GIT_COMMIT}" \ - -e TIMESTAMP="${TIMESTAMP}" \ - -v `pwd`:/go/src/github.com/Cloudbox/autoscan \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -w /go/src/github.com/Cloudbox/autoscan \ - neilotoole/xcgo:latest goreleaser --rm-dist +publish: check_goreleaser ## Generate a release, and publish + goreleaser --rm-dist .PHONY: snapshot -snapshot: ## Generate a snapshot release - docker run --rm --privileged \ - -e VERSION="${VERSION}" \ - -e GIT_COMMIT="${GIT_COMMIT}" \ - -e TIMESTAMP="${TIMESTAMP}" \ - -v `pwd`:/go/src/github.com/Cloudbox/autoscan \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -w /go/src/github.com/Cloudbox/autoscan \ - neilotoole/xcgo:latest goreleaser --snapshot --skip-validate --skip-publish --rm-dist \ No newline at end of file +snapshot: check_goreleaser ## Generate a snapshot release + goreleaser --snapshot --skip-publish --rm-dist diff --git a/README.md b/README.md index 23a10f53..6562c5c6 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,9 @@ Alternatively, you can build the Autoscan binary yourself. To build the autoscan CLI on your system, make sure: 1. Your machine runs Linux, macOS or WSL2 -2. You have [Go](https://golang.org/doc/install) installed (1.14 or later preferred) -3. You have a GCC compiler present \ - *Yup, we need to link to C because of SQLite >:(* -4. Clone this repository and cd into it from the terminal -5. Run `go build -o autoscan ./cmd/autoscan` from the terminal +2. You have [Go](https://golang.org/doc/install) installed (1.16 or later) +3. Clone this repository and cd into it from the terminal +4. Run `go build -o autoscan ./cmd/autoscan` from the terminal You should now have a binary with the name `autoscan` in the root directory of the project. To start autoscan, simply run `./autoscan`. If you want autoscan to be globally available, move it to `/bin` or `/usr/local/bin`. @@ -169,6 +167,8 @@ curl --request POST \ --header 'Authorization: Basic aGVsbG8gdGhlcmU6Z2VuZXJhbCBrZW5vYmk=' ``` +**Note: You can visit `/triggers/manual` within a browser to manually submit requests** + #### Configuration A snippet of the `config.yml` file showcasing what is possible. @@ -319,13 +319,17 @@ minimum-age: 30m # defaults to 5 seconds scan-delay: 15s +# override the interval scan stats are displayed: +# defaults to 1 hour / 0s to disable +scan-stats: 1m + # set multiple anchor files anchors: - /mnt/unionfs/drive1.anchor - /mnt/unionfs/drive2.anchor ``` -The `minimum-age` and `scan-delay` fields should be given a string in the following format: +The `minimum-age`, `scan-delay` and `scan-stats` fields should be given a string in the following format: - `1s` if the min-age should be set at 1 second. - `5m` if the min-age should be set at 5 minutes. @@ -334,15 +338,22 @@ The `minimum-age` and `scan-delay` fields should be given a string in the follow *Please do not forget the `s`, `m` or `h` suffix, otherwise the time unit defaults to nanoseconds.* +Scan stats will print the following information at a configured interval: + +- Scans processed +- Scans remaining + ### Targets While collecting Scans is fun and all, they need to have a final destination. Targets are these final destinations and are given Scans from the processor, one batch at a time. -Autoscan currently supports two targets: +Autoscan currently supports the following targets: - Plex - Emby +- Jellyfin +- Autoscan #### Plex @@ -388,6 +399,42 @@ targets: *It's a bit out of date, but I'm sure you will manage!* - Rewrite. If Emby is not running on the host OS, but in a Docker container (or Autoscan is running in a Docker container), then you need to rewrite paths accordingly. Check out our [rewriting section](#rewriting-paths) for more info. +#### Jellyfin + +While Jellyfin provides much better behaviour out of the box than Plex, it still might be useful to use Autoscan for even better performance. + +You can setup one or multiple Jellyfin targets in the config: + +```yaml +targets: + jellyfin: + - url: https://jellyfin.domain.tld # URL of your Jellyfin server + token: XXXX # Jellyfin API Token + rewrite: + - from: /mnt/unionfs/Media/ # local file system + to: /data/ # path accessible by the Jellyfin docker container (if applicable) +``` + +- URL. The URL can link to the docker container directly, the localhost or a reverse proxy sitting in front of Jellyfin. +- Token. We need a Jellyfin API Token to make requests on your behalf. [This article](https://github.com/MediaBrowser/Emby/wiki/Api-Key-Authentication) should help you out. \ + *It's a bit out of date, but I'm sure you will manage!* +- Rewrite. If Jellyfin is not running on the host OS, but in a Docker container (or Autoscan is running in a Docker container), then you need to rewrite paths accordingly. Check out our [rewriting section](#rewriting-paths) for more info. + +#### Autoscan + +You can also send scan requests to other instances of autoscan! + +```yaml +targets: + autoscan: + - url: https://autoscan.domain.tld # URL of Autoscan + username: XXXX # Username for remote autoscan instance + password: XXXX # Password for remote autoscan instance + rewrite: + - from: /mnt/unionfs/Media/ # local file system + to: /mnt/nfs/Media/ # path accessible by the remote autoscan instance (if applicable) +``` + ### Full config file With the examples given in the [triggers](#triggers), [processor](#processor) and [targets](#targets) sections, here is what your full config file *could* look like: diff --git a/cmd/autoscan/main.go b/cmd/autoscan/main.go index e79f82da..ab8986f1 100644 --- a/cmd/autoscan/main.go +++ b/cmd/autoscan/main.go @@ -17,8 +17,11 @@ import ( "gopkg.in/yaml.v2" "github.com/cloudbox/autoscan" + "github.com/cloudbox/autoscan/migrate" "github.com/cloudbox/autoscan/processor" + ast "github.com/cloudbox/autoscan/targets/autoscan" "github.com/cloudbox/autoscan/targets/emby" + "github.com/cloudbox/autoscan/targets/jellyfin" "github.com/cloudbox/autoscan/targets/plex" "github.com/cloudbox/autoscan/triggers" "github.com/cloudbox/autoscan/triggers/bernard" @@ -29,7 +32,7 @@ import ( "github.com/cloudbox/autoscan/triggers/sonarr" // sqlite3 driver - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" ) type config struct { @@ -37,6 +40,7 @@ type config struct { Port int `yaml:"port"` MinimumAge time.Duration `yaml:"minimum-age"` ScanDelay time.Duration `yaml:"scan-delay"` + ScanStats time.Duration `yaml:"scan-stats"` Anchors []string `yaml:"anchors"` // Authentication for autoscan.HTTPTrigger @@ -57,13 +61,15 @@ type config struct { // autoscan.Target Targets struct { - Plex []plex.Config `yaml:"plex"` - Emby []emby.Config `yaml:"emby"` + Autoscan []ast.Config `yaml:"autoscan"` + Emby []emby.Config `yaml:"emby"` + Jellyfin []jellyfin.Config `yaml:"jellyfin"` + Plex []plex.Config `yaml:"plex"` } `yaml:"targets"` } var ( - // Release variables + // release variables Version string Timestamp string GitCommit string @@ -117,6 +123,7 @@ func main() { os.Exit(1) } + // logger logger := log.Output(io.MultiWriter(zerolog.ConsoleWriter{ TimeFormat: time.Stamp, Out: os.Stderr, @@ -141,7 +148,7 @@ func main() { } // datastore - db, err := sql.Open("sqlite3", cli.Database) + db, err := sql.Open("sqlite", cli.Database) if err != nil { log.Fatal(). Err(err). @@ -149,9 +156,7 @@ func main() { } db.SetMaxOpenConns(1) - // run - mux := http.NewServeMux() - + // config file, err := os.Open(cli.Config) if err != nil { log.Fatal(). @@ -164,6 +169,7 @@ func main() { c := config{ MinimumAge: 10 * time.Minute, ScanDelay: 5 * time.Second, + ScanStats: 1 * time.Hour, Port: 3030, } @@ -176,10 +182,21 @@ func main() { Msg("Failed decoding config") } + // migrator + mg, err := migrate.New(db, "migrations") + if err != nil { + log.Fatal(). + Err(err). + Msg("Failed initialising migrator") + } + + // processor proc, err := processor.New(processor.Config{ Anchors: c.Anchors, MinimumAge: c.MinimumAge, - }, db) + Db: db, + Mg: mg, + }) if err != nil { log.Fatal(). @@ -194,12 +211,11 @@ func main() { // Set authentication. If none and running at least one webhook -> warn user. authHandler := triggers.WithAuth(c.Auth.Username, c.Auth.Password) - if (c.Auth.Username == "" || c.Auth.Password == "") && - len(c.Triggers.Radarr)+len(c.Triggers.Sonarr) > 0 { + if c.Auth.Username == "" || c.Auth.Password == "" { log.Warn().Msg("Webhooks running without authentication") } - // Daemon Triggers + // daemon triggers for _, t := range c.Triggers.Bernard { trigger, err := bernard.New(t, db) if err != nil { @@ -224,7 +240,9 @@ func main() { go trigger(proc.Add) } - // HTTP Triggers + // http triggers + mux := http.NewServeMux() + manualTrigger, err := manual.New(c.Triggers.Manual) if err != nil { log.Fatal(). @@ -296,6 +314,19 @@ func main() { // targets targets := make([]autoscan.Target, 0) + for _, t := range c.Targets.Autoscan { + tp, err := ast.New(t) + if err != nil { + log.Fatal(). + Err(err). + Str("target", "autoscan"). + Str("target_url", t.URL). + Msg("Failed initialising target") + } + + targets = append(targets, tp) + } + for _, t := range c.Targets.Plex { tp, err := plex.New(t) if err != nil { @@ -322,17 +353,44 @@ func main() { targets = append(targets, tp) } + for _, t := range c.Targets.Jellyfin { + tp, err := jellyfin.New(t) + if err != nil { + log.Fatal(). + Err(err). + Str("target", "jellyfin"). + Str("target_url", t.URL). + Msg("Failed initialising target") + } + + targets = append(targets, tp) + } + log.Info(). + Int("autoscan", len(c.Targets.Autoscan)). Int("plex", len(c.Targets.Plex)). Int("emby", len(c.Targets.Emby)). + Int("jellyfin", len(c.Targets.Jellyfin)). Msg("Initialised targets") + // scan stats + if c.ScanStats.Seconds() > 0 { + go scanStats(proc, c.ScanStats) + } + // processor log.Info().Msg("Processor started") targetsAvailable := false - + targetsSize := len(targets) for { + // sleep indefinitely when no targets setup + if targetsSize == 0 { + log.Warn().Msg("No targets initialised, processor stopped, triggers will continue...") + select {} + } + + // target availability checker if !targetsAvailable { err = proc.CheckAvailability(targets) switch { @@ -355,6 +413,7 @@ func main() { } } + // process scans err = proc.Process(targets) switch { case err == nil: diff --git a/cmd/autoscan/stats.go b/cmd/autoscan/stats.go new file mode 100644 index 00000000..62abcc58 --- /dev/null +++ b/cmd/autoscan/stats.go @@ -0,0 +1,38 @@ +package main + +import ( + "errors" + "github.com/cloudbox/autoscan" + "github.com/cloudbox/autoscan/processor" + "github.com/rs/zerolog/log" + "time" +) + +func scanStats(proc *processor.Processor, interval time.Duration) { + st := time.NewTicker(interval) + for { + select { + case _ = <-st.C: + // retrieve amount of scans remaining + sm, err := proc.ScansRemaining() + switch { + case err == nil: + log.Info(). + Int("remaining", sm). + Int64("processed", proc.ScansProcessed()). + Msg("Scan stats") + case errors.Is(err, autoscan.ErrFatal): + log.Error(). + Err(err). + Msg("Fatal error determining amount of remaining scans, scan stats stopped...") + st.Stop() + return + default: + // ErrNoScans should never occur as COUNT should always at-least return 0 + log.Error(). + Err(err). + Msg("Failed determining amount of remaining scans") + } + } + } +} diff --git a/docker/Dockerfile b/docker/Dockerfile index 80df5d8a..6509f47d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,9 @@ FROM sc4h/alpine-s6overlay:3.12 +ARG TARGETOS +ARG TARGETARCH +ARG TARGETVARIANT + ENV \ PATH="/app/autoscan:${PATH}" \ AUTOSCAN_CONFIG="/config/config.yml" \ @@ -8,7 +12,7 @@ ENV \ AUTOSCAN_VERBOSITY="0" # Binary -COPY ["dist/build_linux_linux_amd64/autoscan", "/app/autoscan/autoscan"] +COPY ["dist/autoscan_${TARGETOS}_${TARGETARCH}${TARGETVARIANT:+_7}/autoscan", "/app/autoscan/autoscan"] # Add root files COPY ["docker/run", "/etc/services.d/autoscan/run"] diff --git a/go.mod b/go.mod index 381d5960..d2bba713 100644 --- a/go.mod +++ b/go.mod @@ -1,23 +1,29 @@ module github.com/cloudbox/autoscan -go 1.14 +go 1.16 require ( github.com/BurntSushi/toml v0.3.1 // indirect - github.com/alecthomas/kong v0.2.12 + github.com/alecthomas/kong v0.2.16 github.com/fsnotify/fsnotify v1.4.9 github.com/justinas/alice v1.2.0 github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f - github.com/m-rots/bernard v0.3.4 + github.com/l3uddz/bernard v0.5.1 github.com/m-rots/stubbs v1.1.0 - github.com/mattn/go-sqlite3 v2.0.3+incompatible + github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect github.com/natefinch/lumberjack v2.0.0+incompatible + github.com/oriser/regroup v0.0.0-20201024192559-010c434ff8f3 github.com/pkg/errors v0.9.1 // indirect github.com/robfig/cron/v3 v3.0.1 github.com/rs/zerolog v1.20.0 - golang.org/x/sync v0.0.0-20201207232520-09787c993a3a - golang.org/x/sys v0.0.0-20210112091331-59c308dcf3cc - golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 + golang.org/x/mod v0.4.2 // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + golang.org/x/sys v0.0.0-20210313110737-8e9fff1a3a18 + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba + golang.org/x/tools v0.1.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 + modernc.org/cc/v3 v3.32.0 // indirect + modernc.org/sqlite v1.10.0 + modernc.org/strutil v1.1.1 // indirect ) diff --git a/go.sum b/go.sum index 7706736a..8e3cf73b 100644 --- a/go.sum +++ b/go.sum @@ -1,60 +1,129 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/alecthomas/kong v0.2.9/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE= -github.com/alecthomas/kong v0.2.11 h1:RKeJXXWfg9N47RYfMm0+igkxBCTF4bzbneAxaqid0c4= github.com/alecthomas/kong v0.2.11/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE= -github.com/alecthomas/kong v0.2.12 h1:X3kkCOXGUNzLmiu+nQtoxWqj4U2a39MpSJR3QdQXOwI= -github.com/alecthomas/kong v0.2.12/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE= +github.com/alecthomas/kong v0.2.16 h1:F232CiYSn54Tnl1sJGTeHmx4vJDNLVP2b9yCVMOQwHQ= +github.com/alecthomas/kong v0.2.16/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/google/go-cmp v0.5.3 h1:x95R7cp+rSeeqAMI2knLtQ0DKlaBhv2NrtrOvafPHRo= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/justinas/alice v1.2.0 h1:+MHSA/vccVCF4Uq37S42jwlkvI2Xzl7zTPCN5BnZNVo= github.com/justinas/alice v1.2.0/go.mod h1:fN5HRH/reO/zrUflLfTN43t3vXvKzvZIENsNEe7i7qA= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f h1:dKccXx7xA56UNqOcFIbuqFjAWPVtP688j5QMgmo6OHU= github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f/go.mod h1:4rEELDSfUAlBSyUjPG0JnaNGjf13JySHFeRdD/3dLP0= -github.com/m-rots/bernard v0.3.4 h1:QTip7rcepstmNFJs86RCbeymHgaJS1BJdExZUct5fEg= -github.com/m-rots/bernard v0.3.4/go.mod h1:yDQffALXQDh6sTXdFCbI2rJtYuXgx41MyJM6Sf/j7Sc= -github.com/m-rots/stubbs v1.0.0 h1:lBrjn27J32/iGHp7eKPYGcphuqDIg5UIs/YI4q1m63Q= +github.com/l3uddz/bernard v0.5.1 h1:PdkmJn44q4dmix1riBkrvnpb2LVvhyRLwIRhWoc05bo= +github.com/l3uddz/bernard v0.5.1/go.mod h1:J2ad7LeQl+6Nxc8sGJ8HtQIsv9c9bk2jHxsBt9OIHpo= github.com/m-rots/stubbs v1.0.0/go.mod h1:iDS6z2oonw2UMo2l0S1WTPJ9git7FWU4YEo6fq7F2WU= github.com/m-rots/stubbs v1.1.0 h1:QR1LHxFYPasju/sEO0KLmI5/RADF70CW3ZtisCs7XrQ= github.com/m-rots/stubbs v1.1.0/go.mod h1:Ive+DY/P1EikQ644M3tuyvsO/7ohPLnmEru2L+6hbVw= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/oriser/regroup v0.0.0-20201024192559-010c434ff8f3 h1:SIHa1eb8CJDqFu8potgn0n7grPG2cf2WEKekk/nSz5g= +github.com/oriser/regroup v0.0.0-20201024192559-010c434ff8f3/go.mod h1:odkMeLkWS8G6+WP2z3Pn2vkzhPSvBtFhAUYTKXAtZMQ= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs= github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112091331-59c308dcf3cc h1:y0Og6AYdwus7SIAnKnDxjc4gJetRiYEWOx4AKbOeyEI= -golang.org/x/sys v0.0.0-20210112091331-59c308dcf3cc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210313110737-8e9fff1a3a18 h1:jxr7/dEo+rR29uEBoLSWJ1tRHCFAMwFbGUU9nRqzpds= +golang.org/x/sys v0.0.0-20210313110737-8e9fff1a3a18/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v3 v3.31.5-0.20210308123301-7a3e9dab9009/go.mod h1:0R6jl1aZlIl2avnYfbfHBS1QB6/f+16mihBObaBC878= +modernc.org/cc/v3 v3.32.0 h1:f7CzEixf9/9Rv7KksLyFHgOT1BfpbPLpwD/SPAHu3pg= +modernc.org/cc/v3 v3.32.0/go.mod h1:0R6jl1aZlIl2avnYfbfHBS1QB6/f+16mihBObaBC878= +modernc.org/ccgo/v3 v3.9.0 h1:JbcEIqjw4Agf+0g3Tc85YvfYqkkFOv6xBwS4zkfqSoA= +modernc.org/ccgo/v3 v3.9.0/go.mod h1:nQbgkn8mwzPdp4mm6BT6+p85ugQ7FrGgIcYaE7nSrpY= +modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/libc v1.7.13-0.20210308123627-12f642a52bb8/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w= +modernc.org/libc v1.8.0 h1:Pp4uv9g0csgBMpGPABKtkieF6O5MGhfGo6ZiOdlYfR8= +modernc.org/libc v1.8.0/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w= +modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.2.2 h1:+yFk8hBprV+4c0U9GjFtL+dV3N8hOJ8JCituQcMShFY= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.0.4 h1:utMBrFcpnQDdNsmM6asmyH/FM9TqLPS7XF7otpJmrwM= +modernc.org/memory v1.0.4/go.mod h1:nV2OApxradM3/OVbs2/0OsP6nPfakXpi50C7dcoHXlc= +modernc.org/opt v0.1.1 h1:/0RX92k9vwVeDXj+Xn23DKp2VJubL7k8qNffND6qn3A= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.10.0 h1:0QNqx4EzfZzNEG13sFbS/L+egh0X5WXSckHrxHkySX8= +modernc.org/sqlite v1.10.0/go.mod h1:PGzq6qlhyYjL6uVbSgS6WoF7ZopTW/sI7+7p+mb4ZVU= +modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/strutil v1.1.1 h1:xv+J1BXY3Opl2ALrBwyfEikFAj8pmqcpnfmuwUwcozs= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/tcl v1.5.0 h1:euZSUNfE0Fd4W8VqXI1Ly1v7fqDJoBuAV88Ea+SnaSs= +modernc.org/tcl v1.5.0/go.mod h1:gb57hj4pO8fRrK54zveIfFXBaMHK3SKJNWcmRw1cRzc= +modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.0.1-0.20210308123920-1f282aa71362/go.mod h1:8/SRk5C/HgiQWCgXdfpb+1RvhORdkz5sw72d3jjtyqA= +modernc.org/z v1.0.1 h1:WyIDpEpAIx4Hel6q/Pcgj/VhaQV5XPJ2I6ryIYbjnpc= +modernc.org/z v1.0.1/go.mod h1:8/SRk5C/HgiQWCgXdfpb+1RvhORdkz5sw72d3jjtyqA= diff --git a/migrate/migrator.go b/migrate/migrator.go new file mode 100644 index 00000000..ac61c65c --- /dev/null +++ b/migrate/migrator.go @@ -0,0 +1,79 @@ +package migrate + +import ( + "database/sql" + "embed" + "errors" + "fmt" + "github.com/oriser/regroup" + "modernc.org/sqlite" +) + +type Migrator struct { + db *sql.DB + dir string + + re *regroup.ReGroup +} + +/* Credits to https://github.com/Boostport/migration */ + +func New(db *sql.DB, dir string) (*Migrator, error) { + var err error + + m := &Migrator{ + db: db, + dir: dir, + } + + // validate supported driver + if _, ok := db.Driver().(*sqlite.Driver); !ok { + return nil, errors.New("database instance is not using the sqlite driver") + } + + // verify schema + if err = m.verify(); err != nil { + return nil, fmt.Errorf("verify: %w", err) + } + + // compile migration regexp + m.re, err = regroup.Compile(`(?P\d+)\w?(?P.+)?\.sql`) + if err != nil { + return nil, fmt.Errorf("regexp: %w", err) + } + + return m, nil +} + +func (m *Migrator) Migrate(fs *embed.FS, component string) error { + // parse migrations + migrations, err := m.parse(fs) + if err != nil { + return fmt.Errorf("parse: %w", err) + } + + if len(migrations) == 0 { + return nil + } + + // get current migration versions + versions, err := m.versions(component) + if err != nil { + return fmt.Errorf("versions: %v: %w", component, err) + } + + // migrate + for _, migration := range migrations { + // already have this version? + if _, exists := versions[migration.Version]; exists { + continue + } + + // migrate + if err := m.exec(component, migration); err != nil { + return fmt.Errorf("migrate: %v: %w", migration.Filename, err) + } + } + + return nil +} diff --git a/migrate/util.go b/migrate/util.go new file mode 100644 index 00000000..2cec6711 --- /dev/null +++ b/migrate/util.go @@ -0,0 +1,119 @@ +package migrate + +import ( + "database/sql" + "embed" + "fmt" + "path/filepath" +) + +type migration struct { + Version int `regroup:"Version"` + Name string `regroup:"Name"` + Filename string + Schema string +} + +func (m *Migrator) verify() error { + if _, err := m.db.Exec(sqlSchema); err != nil { + return fmt.Errorf("schema: %w", err) + } + return nil +} + +func (m *Migrator) versions(component string) (map[int]bool, error) { + rows, err := m.db.Query(sqlVersions, component) + if err != nil { + return nil, fmt.Errorf("query: %w", err) + } + defer rows.Close() + + versions := make(map[int]bool, 0) + for rows.Next() { + var version int + if err := rows.Scan(&version); err != nil { + return nil, fmt.Errorf("scan: %w", err) + } + + versions[version] = true + } + + return versions, nil +} + +func (m *Migrator) exec(component string, migration *migration) (err error) { + // begin tx + tx, err := m.db.Begin() + if err != nil { + return fmt.Errorf("begin: %w", err) + } + + // commit - rollback + defer func(tx *sql.Tx) { + // roll back + if err != nil { + if errRb := tx.Rollback(); errRb != nil { + err = fmt.Errorf("rollback: %v: %w", errRb, err) + } + return + } + + // commit + if errCm := tx.Commit(); err != nil { + err = fmt.Errorf("commit: %w", errCm) + } + }(tx) + + // exec migration + if _, err := tx.Exec(migration.Schema); err != nil { + return fmt.Errorf("exec: %w", err) + } + + // insert migration version + if _, err := tx.Exec(sqlInsertVersion, component, migration.Version); err != nil { + return fmt.Errorf("schema_migration: %w", err) + } + + return nil +} + +func (m *Migrator) parse(fs *embed.FS) (map[int]*migration, error) { + // parse migrations from filesystem + files, err := fs.ReadDir(m.dir) + if err != nil { + return nil, fmt.Errorf("read migrations: %w", err) + } + + // parse migrations + migrations := make(map[int]*migration) + for _, f := range files { + // skip dirs + if f.IsDir() { + continue + } + + // parse migration + md := new(migration) + if err := m.re.MatchToTarget(f.Name(), md); err != nil { + return nil, fmt.Errorf("parse migration: %w", err) + } + + b, err := fs.ReadFile(filepath.Join(m.dir, f.Name())) + if err != nil { + return nil, fmt.Errorf("read migration: %w", err) + } + md.Schema = string(b) + md.Filename = f.Name() + + // set migration + migrations[md.Version] = md + } + + return migrations, nil +} + +const ( + sqlSchema = `CREATE TABLE IF NOT EXISTS schema_migration (component VARCHAR(255) NOT NULL, version INTEGER NOT NULL, PRIMARY KEY (component, version))` + sqlVersions = `SELECT version FROM schema_migration WHERE component = ?` + sqlInsertVersion = `INSERT INTO schema_migration (component, version) VALUES (?, ?)` +) diff --git a/processor/datastore.go b/processor/datastore.go index 1b55771c..7f8feaac 100644 --- a/processor/datastore.go +++ b/processor/datastore.go @@ -2,38 +2,34 @@ package processor import ( "database/sql" + "embed" "errors" "fmt" "time" "github.com/cloudbox/autoscan" + "github.com/cloudbox/autoscan/migrate" // sqlite3 driver - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" ) type datastore struct { *sql.DB } -const sqlSchema = ` -CREATE TABLE IF NOT EXISTS scan ( - "folder" TEXT NOT NULL, - "priority" INTEGER NOT NULL, - "time" DATETIME NOT NULL, - PRIMARY KEY(folder) +var ( + //go:embed migrations + migrations embed.FS ) -` -func newDatastore(db *sql.DB) (*datastore, error) { - _, err := db.Exec(sqlSchema) - if err != nil { - return nil, fmt.Errorf("exec schema: %w", err) +func newDatastore(db *sql.DB, mg *migrate.Migrator) (*datastore, error) { + // migrations + if err := mg.Migrate(&migrations, "processor"); err != nil { + return nil, fmt.Errorf("migrate: %w", err) } - store := &datastore{db} - - return store, nil + return &datastore{db}, nil } const sqlUpsert = ` @@ -68,6 +64,23 @@ func (store *datastore) Upsert(scans []autoscan.Scan) error { return tx.Commit() } +const sqlGetScansRemaining = `SELECT COUNT(folder) FROM scan` + +func (store *datastore) GetScansRemaining() (int, error) { + row := store.QueryRow(sqlGetScansRemaining) + + remaining := 0 + err := row.Scan(&remaining) + switch { + case errors.Is(err, sql.ErrNoRows): + return remaining, nil + case err != nil: + return remaining, fmt.Errorf("get remaining scans: %v: %w", err, autoscan.ErrFatal) + } + + return remaining, nil +} + const sqlGetAvailableScan = ` SELECT folder, priority, time FROM scan WHERE time < ? diff --git a/processor/datastore_test.go b/processor/datastore_test.go index d02d78bb..f68a329a 100644 --- a/processor/datastore_test.go +++ b/processor/datastore_test.go @@ -3,6 +3,7 @@ package processor import ( "database/sql" "errors" + "github.com/cloudbox/autoscan/migrate" "reflect" "testing" "time" @@ -10,7 +11,7 @@ import ( "github.com/cloudbox/autoscan" // sqlite3 driver - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" ) const sqlGetScan = ` @@ -28,12 +29,17 @@ func (store *datastore) GetScan(folder string) (autoscan.Scan, error) { } func getDatastore(t *testing.T) *datastore { - db, err := sql.Open("sqlite3", ":memory:") + db, err := sql.Open("sqlite", ":memory:") if err != nil { t.Fatal(err) } - ds, err := newDatastore(db) + mg, err := migrate.New(db, "migrations") + if err != nil { + t.Fatal(err) + } + + ds, err := newDatastore(db, mg) if err != nil { t.Fatal(err) } diff --git a/processor/migrations/1_init.sql b/processor/migrations/1_init.sql new file mode 100644 index 00000000..d46a0c10 --- /dev/null +++ b/processor/migrations/1_init.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS scan ( + "folder" TEXT NOT NULL, + "priority" INTEGER NOT NULL, + "time" DATETIME NOT NULL, + PRIMARY KEY(folder) +) \ No newline at end of file diff --git a/processor/processor.go b/processor/processor.go index 86d83fb3..dc266f73 100644 --- a/processor/processor.go +++ b/processor/processor.go @@ -4,19 +4,25 @@ import ( "database/sql" "fmt" "os" + "sync/atomic" "time" "github.com/cloudbox/autoscan" + "github.com/cloudbox/autoscan/migrate" + "golang.org/x/sync/errgroup" ) type Config struct { Anchors []string MinimumAge time.Duration + + Db *sql.DB + Mg *migrate.Migrator } -func New(c Config, db *sql.DB) (*Processor, error) { - store, err := newDatastore(db) +func New(c Config) (*Processor, error) { + store, err := newDatastore(c.Db, c.Mg) if err != nil { return nil, err } @@ -33,12 +39,23 @@ type Processor struct { anchors []string minimumAge time.Duration store *datastore + processed int64 } func (p *Processor) Add(scans ...autoscan.Scan) error { return p.store.Upsert(scans) } +// ScansRemaining returns the amount of scans remaining +func (p *Processor) ScansRemaining() (int, error) { + return p.store.GetScansRemaining() +} + +// ScansProcessed returns the amount of scans processed +func (p *Processor) ScansProcessed() int64 { + return atomic.LoadInt64(&p.processed) +} + // CheckAvailability checks whether all targets are available. // If one target is not available, the error will return. func (p *Processor) CheckAvailability(targets []autoscan.Target) error { @@ -91,6 +108,7 @@ func (p *Processor) Process(targets []autoscan.Target) error { return err } + atomic.AddInt64(&p.processed, 1) return nil } diff --git a/targets/autoscan/api.go b/targets/autoscan/api.go new file mode 100644 index 00000000..fc4bdc1f --- /dev/null +++ b/targets/autoscan/api.go @@ -0,0 +1,101 @@ +package autoscan + +import ( + "fmt" + "github.com/cloudbox/autoscan" + "github.com/rs/zerolog" + "net/http" + "net/url" +) + +type apiClient struct { + client *http.Client + log zerolog.Logger + baseURL string + user string + pass string +} + +func newAPIClient(baseURL string, user string, pass string, log zerolog.Logger) apiClient { + return apiClient{ + client: &http.Client{}, + log: log, + baseURL: baseURL, + user: user, + pass: pass, + } +} + +func (c apiClient) do(req *http.Request) (*http.Response, error) { + res, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("%v: %w", err, autoscan.ErrTargetUnavailable) + } + + if res.StatusCode >= 200 && res.StatusCode < 300 { + return res, nil + } + + c.log.Trace(). + Stringer("request_url", res.Request.URL). + Int("response_status", res.StatusCode). + Msg("Request failed") + + // statusCode not in the 2xx range, close response + res.Body.Close() + + switch res.StatusCode { + case 401: + return nil, fmt.Errorf("invalid basic auth: %s: %w", res.Status, autoscan.ErrFatal) + case 404, 500, 502, 503, 504: + return nil, fmt.Errorf("%s: %w", res.Status, autoscan.ErrTargetUnavailable) + default: + return nil, fmt.Errorf("%s: %w", res.Status, autoscan.ErrFatal) + } +} + +func (c apiClient) Available() error { + // create request + req, err := http.NewRequest("HEAD", autoscan.JoinURL(c.baseURL, "triggers", "manual"), nil) + if err != nil { + return fmt.Errorf("failed creating head request: %v: %w", err, autoscan.ErrFatal) + } + + if c.user != "" && c.pass != "" { + req.SetBasicAuth(c.user, c.pass) + } + + // send request + res, err := c.do(req) + if err != nil { + return fmt.Errorf("availability: %w", err) + } + + defer res.Body.Close() + return nil +} + +func (c apiClient) Scan(path string) error { + // create request + req, err := http.NewRequest("POST", autoscan.JoinURL(c.baseURL, "triggers", "manual"), nil) + if err != nil { + return fmt.Errorf("failed creating scan request: %v: %w", err, autoscan.ErrFatal) + } + + if c.user != "" && c.pass != "" { + req.SetBasicAuth(c.user, c.pass) + } + + q := url.Values{} + q.Add("dir", path) + req.URL.RawQuery = q.Encode() + + // send request + res, err := c.do(req) + if err != nil { + return fmt.Errorf("scan: %w", err) + } + + defer res.Body.Close() + return nil +} diff --git a/targets/autoscan/autoscan.go b/targets/autoscan/autoscan.go new file mode 100644 index 00000000..d57afd7f --- /dev/null +++ b/targets/autoscan/autoscan.go @@ -0,0 +1,67 @@ +package autoscan + +import ( + "github.com/cloudbox/autoscan" + "github.com/rs/zerolog" +) + +type Config struct { + URL string `yaml:"url"` + User string `yaml:"username"` + Pass string `yaml:"password"` + Rewrite []autoscan.Rewrite `yaml:"rewrite"` + Verbosity string `yaml:"verbosity"` +} + +type target struct { + url string + user string + pass string + + log zerolog.Logger + rewrite autoscan.Rewriter + api apiClient +} + +func New(c Config) (autoscan.Target, error) { + l := autoscan.GetLogger(c.Verbosity).With(). + Str("target", "autoscan"). + Str("url", c.URL).Logger() + + rewriter, err := autoscan.NewRewriter(c.Rewrite) + if err != nil { + return nil, err + } + + return &target{ + url: c.URL, + user: c.User, + pass: c.Pass, + + log: l, + rewrite: rewriter, + api: newAPIClient(c.URL, c.User, c.Pass, l), + }, nil +} + +func (t target) Scan(scan autoscan.Scan) error { + scanFolder := t.rewrite(scan.Folder) + + // send scan request + l := t.log.With(). + Str("path", scanFolder). + Logger() + + l.Trace().Msg("Sending scan request") + + if err := t.api.Scan(scanFolder); err != nil { + return err + } + + l.Info().Msg("Scan moved to target") + return nil +} + +func (t target) Available() error { + return t.api.Available() +} diff --git a/targets/emby/api.go b/targets/emby/api.go index c75ed192..3d3d0d8d 100644 --- a/targets/emby/api.go +++ b/targets/emby/api.go @@ -113,9 +113,16 @@ func (c apiClient) Libraries() ([]library, error) { libraries := make([]library, 0) for _, lib := range resp { for _, folder := range lib.Folders { + libPath := folder.Path + + // Add trailing slash if there is none. + if len(libPath) > 0 && libPath[len(libPath)-1] != '/' { + libPath += "/" + } + libraries = append(libraries, library{ Name: lib.Name, - Path: folder.Path, + Path: libPath, }) } } diff --git a/targets/jellyfin/api.go b/targets/jellyfin/api.go new file mode 100644 index 00000000..ac581f25 --- /dev/null +++ b/targets/jellyfin/api.go @@ -0,0 +1,173 @@ +package jellyfin + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "github.com/cloudbox/autoscan" + "github.com/rs/zerolog" +) + +type apiClient struct { + client *http.Client + log zerolog.Logger + baseURL string + token string +} + +func newAPIClient(baseURL string, token string, log zerolog.Logger) apiClient { + return apiClient{ + client: &http.Client{}, + log: log, + baseURL: baseURL, + token: token, + } +} + +func (c apiClient) do(req *http.Request) (*http.Response, error) { + req.Header.Set("X-Emby-Token", c.token) + req.Header.Set("Accept", "application/json") // Force JSON Response. + + res, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("%v: %w", err, autoscan.ErrTargetUnavailable) + } + + if res.StatusCode >= 200 && res.StatusCode < 300 { + return res, nil + } + + c.log.Trace(). + Stringer("request_url", res.Request.URL). + Int("response_status", res.StatusCode). + Msg("Request failed") + + // statusCode not in the 2xx range, close response + res.Body.Close() + + switch res.StatusCode { + case 401: + return nil, fmt.Errorf("invalid jellyfin token: %s: %w", res.Status, autoscan.ErrFatal) + case 404, 500, 502, 503, 504: + return nil, fmt.Errorf("%s: %w", res.Status, autoscan.ErrTargetUnavailable) + default: + return nil, fmt.Errorf("%s: %w", res.Status, autoscan.ErrFatal) + } +} + +func (c apiClient) Available() error { + // create request + reqURL := autoscan.JoinURL(c.baseURL, "System", "Info") + req, err := http.NewRequest("GET", reqURL, nil) + if err != nil { + return fmt.Errorf("failed creating availability request: %v: %w", err, autoscan.ErrFatal) + } + + // send request + res, err := c.do(req) + if err != nil { + return fmt.Errorf("availability: %w", err) + } + + defer res.Body.Close() + return nil +} + +type library struct { + Name string + Path string +} + +func (c apiClient) Libraries() ([]library, error) { + // create request + reqURL := autoscan.JoinURL(c.baseURL, "Library", "VirtualFolders") + req, err := http.NewRequest("GET", reqURL, nil) + if err != nil { + return nil, fmt.Errorf("failed creating libraries request: %v: %w", err, autoscan.ErrFatal) + } + + // send request + res, err := c.do(req) + if err != nil { + return nil, fmt.Errorf("libraries: %w", err) + } + + defer res.Body.Close() + + // decode response + type Response struct { + Name string `json:"Name"` + Locations []string `json:"Locations"` + } + + resp := make([]Response, 0) + if err := json.NewDecoder(res.Body).Decode(&resp); err != nil { + return nil, fmt.Errorf("failed decoding libraries request response: %v: %w", err, autoscan.ErrFatal) + } + + // process response + libraries := make([]library, 0) + for _, lib := range resp { + for _, folder := range lib.Locations { + libPath := folder + + // Add trailing slash if there is none. + if len(libPath) > 0 && libPath[len(libPath)-1] != '/' { + libPath += "/" + } + + libraries = append(libraries, library{ + Name: lib.Name, + Path: libPath, + }) + } + } + + return libraries, nil +} + +type scanRequest struct { + Path string `json:"path"` + UpdateType string `json:"updateType"` +} + +func (c apiClient) Scan(path string) error { + // create request payload + type Payload struct { + Updates []scanRequest `json:"Updates"` + } + + payload := &Payload{ + Updates: []scanRequest{ + { + Path: path, + UpdateType: "Modified", + }, + }, + } + + b, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed encoding scan request payload: %v: %w", err, autoscan.ErrFatal) + } + + // create request + reqURL := autoscan.JoinURL(c.baseURL, "Library", "Media", "Updated") + req, err := http.NewRequest("POST", reqURL, bytes.NewBuffer(b)) + if err != nil { + return fmt.Errorf("failed creating scan request: %v: %w", err, autoscan.ErrFatal) + } + + req.Header.Set("Content-Type", "application/json") + + // send request + res, err := c.do(req) + if err != nil { + return fmt.Errorf("scan: %w", err) + } + + defer res.Body.Close() + return nil +} diff --git a/targets/jellyfin/jellyfin.go b/targets/jellyfin/jellyfin.go new file mode 100644 index 00000000..f8f88d0c --- /dev/null +++ b/targets/jellyfin/jellyfin.go @@ -0,0 +1,102 @@ +package jellyfin + +import ( + "fmt" + "strings" + + "github.com/cloudbox/autoscan" + "github.com/rs/zerolog" +) + +type Config struct { + URL string `yaml:"url"` + Token string `yaml:"token"` + Rewrite []autoscan.Rewrite `yaml:"rewrite"` + Verbosity string `yaml:"verbosity"` +} + +type target struct { + url string + token string + libraries []library + + log zerolog.Logger + rewrite autoscan.Rewriter + api apiClient +} + +func New(c Config) (autoscan.Target, error) { + l := autoscan.GetLogger(c.Verbosity).With(). + Str("target", "jellyfin"). + Str("url", c.URL). + Logger() + + rewriter, err := autoscan.NewRewriter(c.Rewrite) + if err != nil { + return nil, err + } + + api := newAPIClient(c.URL, c.Token, l) + + libraries, err := api.Libraries() + if err != nil { + return nil, err + } + + l.Debug(). + Interface("libraries", libraries). + Msg("Retrieved libraries") + + return &target{ + url: c.URL, + token: c.Token, + libraries: libraries, + + log: l, + rewrite: rewriter, + api: api, + }, nil +} + +func (t target) Available() error { + return t.api.Available() +} + +func (t target) Scan(scan autoscan.Scan) error { + // determine library for this scan + scanFolder := t.rewrite(scan.Folder) + + lib, err := t.getScanLibrary(scanFolder) + if err != nil { + t.log.Warn(). + Err(err). + Msg("No target libraries found") + + return nil + } + + l := t.log.With(). + Str("path", scanFolder). + Str("library", lib.Name). + Logger() + + // send scan request + l.Trace().Msg("Sending scan request") + + if err := t.api.Scan(scanFolder); err != nil { + return err + } + + l.Info().Msg("Scan moved to target") + return nil +} + +func (t target) getScanLibrary(folder string) (*library, error) { + for _, l := range t.libraries { + if strings.HasPrefix(folder, l.Path) { + return &l, nil + } + } + + return nil, fmt.Errorf("%v: failed determining library", folder) +} diff --git a/triggers/bernard/bernard.go b/triggers/bernard/bernard.go index 53c4a113..d40d1bf4 100644 --- a/triggers/bernard/bernard.go +++ b/triggers/bernard/bernard.go @@ -7,9 +7,9 @@ import ( "path/filepath" "time" - lowe "github.com/m-rots/bernard" - ds "github.com/m-rots/bernard/datastore" - "github.com/m-rots/bernard/datastore/sqlite" + lowe "github.com/l3uddz/bernard" + ds "github.com/l3uddz/bernard/datastore" + "github.com/l3uddz/bernard/datastore/sqlite" "github.com/m-rots/stubbs" "github.com/robfig/cron/v3" "github.com/rs/zerolog" diff --git a/triggers/bernard/datastore.go b/triggers/bernard/datastore.go index 7fb2ab92..f9f89c98 100644 --- a/triggers/bernard/datastore.go +++ b/triggers/bernard/datastore.go @@ -4,8 +4,8 @@ import ( "database/sql" "errors" "fmt" - "github.com/m-rots/bernard/datastore" - "github.com/m-rots/bernard/datastore/sqlite" + "github.com/l3uddz/bernard/datastore" + "github.com/l3uddz/bernard/datastore/sqlite" ) type bds struct { diff --git a/triggers/bernard/paths.go b/triggers/bernard/paths.go index 9be807f8..edbb5e90 100644 --- a/triggers/bernard/paths.go +++ b/triggers/bernard/paths.go @@ -2,9 +2,9 @@ package bernard import ( "fmt" - "github.com/m-rots/bernard" - "github.com/m-rots/bernard/datastore" - "github.com/m-rots/bernard/datastore/sqlite" + "github.com/l3uddz/bernard" + "github.com/l3uddz/bernard/datastore" + "github.com/l3uddz/bernard/datastore/sqlite" "path/filepath" ) diff --git a/triggers/bernard/postprocess.go b/triggers/bernard/postprocess.go index 913f46ad..b1a89ddc 100644 --- a/triggers/bernard/postprocess.go +++ b/triggers/bernard/postprocess.go @@ -2,9 +2,9 @@ package bernard import ( "fmt" - "github.com/m-rots/bernard" - "github.com/m-rots/bernard/datastore" - "github.com/m-rots/bernard/datastore/sqlite" + "github.com/l3uddz/bernard" + "github.com/l3uddz/bernard/datastore" + "github.com/l3uddz/bernard/datastore/sqlite" ) func NewPostProcessBernardDiff(driveID string, store *bds, diff *sqlite.Difference) bernard.Hook { diff --git a/triggers/manual/manual.go b/triggers/manual/manual.go index d80701c7..a3561017 100644 --- a/triggers/manual/manual.go +++ b/triggers/manual/manual.go @@ -1,6 +1,7 @@ package manual import ( + _ "embed" "net/http" "path" "time" @@ -15,6 +16,11 @@ type Config struct { Verbosity string `yaml:"verbosity"` } +var ( + //go:embed "template.html" + template []byte +) + // New creates an autoscan-compatible HTTP Trigger for manual webhooks. func New(c Config) (autoscan.HTTPTrigger, error) { rewriter, err := autoscan.NewRewriter(c.Rewrite) @@ -46,7 +52,15 @@ func (h handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { query := r.URL.Query() directories := query["dir"] - rlog.Trace().Interface("dirs", directories).Msg("Received directories") + switch r.Method { + case "GET": + rw.Header().Set("Content-Type", "text/html") + _, _ = rw.Write(template) + return + case "HEAD": + rw.WriteHeader(http.StatusOK) + return + } if len(directories) == 0 { rlog.Error().Msg("Manual webhook should receive at least one directory") @@ -54,6 +68,8 @@ func (h handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { return } + rlog.Trace().Interface("dirs", directories).Msg("Received directories") + scans := make([]autoscan.Scan, 0) for _, dir := range directories { diff --git a/triggers/manual/template.html b/triggers/manual/template.html new file mode 100644 index 00000000..8b014a9d --- /dev/null +++ b/triggers/manual/template.html @@ -0,0 +1,80 @@ + + + + autoscan + + + + + +
+
+
+

autoscan

+

Path to scan

+
+
+ +
+
+
+ +
+
+
+ + + + + + \ No newline at end of file diff --git a/triggers/middleware.go b/triggers/middleware.go index b1ca8d88..6beb17f5 100644 --- a/triggers/middleware.go +++ b/triggers/middleware.go @@ -1,6 +1,7 @@ package triggers import ( + "crypto/subtle" "net/http" "time" @@ -51,13 +52,25 @@ func WithAuth(username, password string) func(http.Handler) http.Handler { l := hlog.FromRequest(r) user, pass, ok := r.BasicAuth() - if ok && user == username && pass == password { + if !ok || user == "" || pass == "" { + // prompt auth dialog + rw.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`) + rw.WriteHeader(http.StatusUnauthorized) + return + } + + // validate credentials + if ok && + subtle.ConstantTimeCompare([]byte(user), []byte(username)) == 1 && + subtle.ConstantTimeCompare([]byte(pass), []byte(password)) == 1 { l.Trace().Msg("Successful authentication") next.ServeHTTP(rw, r) return } + // invalid credentials l.Warn().Msg("Invalid authentication") + rw.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`) rw.WriteHeader(http.StatusUnauthorized) }) }