refactor(imgs): use imgproxy instead of libwebp (#56)

---------

Co-authored-by: Antonio Mika <[email protected]>

.env.example

@@ -13,6 +13,17 @@ MINIO_PROMETHEUS_AUTH_TYPE=public
 MINIO_PROMETHEUS_URL=
 MINIO_PROMETHEUS_JOB_ID=minio
 
+USE_IMGPROXY=1
+IMGPROXY_URL=http://imgproxy:8080
+IMGPROXY_ALLOWED_SOURCES=s3://,local://
+IMGPROXY_LOCAL_FILESYSTEM_ROOT=/storage
+IMGPROXY_USE_S3=true
+IMGPROXY_S3_ENDPOINT=$MINIO_URL
+IMGPROXY_KEY=6465616462656566 # deadbeef
+IMGPROXY_SALT=6465616462656566 # deadbeef
+AWS_ACCESS_KEY_ID=$MINIO_ROOT_USER
+AWS_SECRET_ACCESS_KEY=$MINIO_ROOT_PASSWORD
+
 LISTS_CADDYFILE=./caddy/Caddyfile
 LISTS_V4=
 LISTS_V6=

.github/workflows/build.yml

@@ -21,9 +21,6 @@ jobs:
   test:
     runs-on: ubuntu-22.04
     steps:
-    - name: Install package
-      run: |
-        sudo apt-get -y install libwebp-dev
     - name: Set up Go
       uses: actions/setup-go@v3
       with:

.github/workflows/pgs.yml

@@ -8,9 +8,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Install package
-        run: |
-          sudo apt-get -y install libwebp-dev
       - name: Set outputs
         id: vars
         run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT

Dockerfile

@@ -3,12 +3,8 @@ LABEL maintainer="Pico Maintainers <[email protected]>"
 
 WORKDIR /app
 
-RUN dpkg --add-architecture arm64 && dpkg --add-architecture amd64
 RUN apt-get update
-RUN apt-get install -y git ca-certificates \
-    libwebp-dev:amd64 libwebp-dev:arm64 \
-    crossbuild-essential-amd64 crossbuild-essential-arm64 \
-    libc-dev:amd64 libc-dev:arm64
+RUN apt-get install -y git ca-certificates
 
 COPY go.* ./
 
@@ -22,17 +18,16 @@ ARG APP=lists
 ARG TARGETOS
 ARG TARGETARCH
 
-ENV CGO_ENABLED=1
-ENV LDFLAGS="-s -w -linkmode external -extldflags '-static -lm -pthread'"
-ENV CC=/app/scripts/gccwrap.sh
+ENV CGO_ENABLED=0
+ENV LDFLAGS="-s -w"
 
 ENV GOOS=${TARGETOS} GOARCH=${TARGETARCH}
 
-RUN go build -ldflags "$LDFLAGS" -tags "netgo osusergo" -o /go/bin/${APP}-web ./cmd/${APP}/web
+RUN go build -ldflags "$LDFLAGS" -o /go/bin/${APP}-web ./cmd/${APP}/web
 
 FROM builder-web as builder-ssh
 
-RUN go build -ldflags "$LDFLAGS" -tags "netgo osusergo" -o /go/bin/${APP}-ssh ./cmd/${APP}/ssh
+RUN go build -ldflags "$LDFLAGS" -o /go/bin/${APP}-ssh ./cmd/${APP}/ssh
 
 FROM scratch as release-web
 

Makefile

@@ -22,7 +22,7 @@ css:
 .PHONY: css
 
 lint:
-	$(DOCKER_CMD) run --rm -v $(shell pwd):/app -w /app golangci/golangci-lint:latest bash -c 'apt-get update > /dev/null 2>&1 && apt-get install -y libwebp-dev > /dev/null 2>&1; golangci-lint run -E goimports -E godot --timeout 10m'
+	$(DOCKER_CMD) run --rm -v $(shell pwd):/app -w /app golangci/golangci-lint:latest run -E goimports -E godot --timeout 10m
 .PHONY: lint
 
 lint-dev:

README.md

@@ -21,9 +21,6 @@ This repo hosts the following pico services:
 
 - `golang` >= 1.21.0
 - `direnv` to load environment vars
-- `webp` package dependency
-  - on mac can be installed with `brew install webp`
-  - on ubuntu can be installed with `sudo apt install libwebp-dev`
 
 ```bash
 cp ./.env.example .env

docker-compose.override.yml

@@ -11,6 +11,13 @@ services:
     ports:
       - "9000:9000"
       - "9001:9001"
+  imgproxy:
+    env_file:
+      - .env.example
+    volumes:
+      - ./data/storage/data:/storage
+    ports:
+      - "8080:8080"
   lists-web:
     build:
       args:
@@ -79,7 +86,7 @@ services:
     env_file:
       - .env.example
     volumes:
-      - ./data/imgs-storage/data:/app/.storage
+      - ./data/storage/data:/app/.storage
     ports:
       - "3003:3000"
   imgs-ssh:
@@ -90,7 +97,7 @@ services:
     env_file:
       - .env.example
     volumes:
-      - ./data/imgs-storage/data:/app/.storage
+      - ./data/storage/data:/app/.storage
       - ./data/imgs-ssh/data:/app/ssh_data
     ports:
       - "2223:2222"
@@ -102,7 +109,7 @@ services:
     env_file:
       - .env.example
     volumes:
-      - ./data/pgs-storage/data:/app/.storage
+      - ./data/storage/data:/app/.storage
     ports:
       - "3004:3000"
   pgs-ssh:
@@ -113,7 +120,7 @@ services:
     env_file:
       - .env.example
     volumes:
-      - ./data/pgs-storage/data:/app/.storage
+      - ./data/storage/data:/app/.storage
       - ./data/pgs-ssh/data:/app/ssh_data
     ports:
       - "2224:2222"
@@ -156,4 +163,4 @@ services:
       - ./data/certs:/certs
     ports:
       - "6697:6697"
-      - "8080:8080"
+      - "8081:8080"

docker-compose.prod.yml

@@ -31,9 +31,12 @@ services:
       - .env.prod
     volumes:
       - ./data/minio-data:/data
-    ports:
-      - "9000:9000"
-      - "9001:9001"
+  imgproxy:
+    env_file:
+      - .env.prod
+    volumes:
+      - ./data/imgs-storage/data:/storage/imgs
+      - ./data/pgs-storage/data:/storage/pgs
   lists-caddy:
     image: ghcr.io/picosh/pico/caddy:latest
     restart: always
@@ -191,7 +194,7 @@ services:
     env_file:
       - .env.prod
     volumes:
-      - ./data/imgs-storage/data:/app/.storage
+      - ./data/storage/data:/app/.storage
   imgs-ssh:
     networks:
       imgs:
@@ -200,7 +203,7 @@ services:
     env_file:
       - .env.prod
     volumes:
-      - ./data/imgs-storage/data:/app/.storage
+      - ./data/storage/data:/app/.storage
       - ./data/imgs-ssh/data:/app/ssh_data
     ports:
       - "${IMGS_SSH_V4:-22}:2222"
@@ -236,7 +239,7 @@ services:
     env_file:
       - .env.prod
     volumes:
-      - ./data/pgs-storage/data:/app/.storage
+      - ./data/storage/data:/app/.storage
   pgs-ssh:
     networks:
       pgs:
@@ -245,7 +248,7 @@ services:
     env_file:
       - .env.prod
     volumes:
-      - ./data/pgs-storage/data:/app/.storage
+      - ./data/storage/data:/app/.storage
       - ./data/pgs-ssh/data:/app/ssh_data
     ports:
       - "${PGS_SSH_V4:-22}:2222"

docker-compose.yml

@@ -13,6 +13,12 @@ services:
     profiles:
       - db
       - all
+  imgproxy:
+    image: darthsim/imgproxy:latest
+    restart: always
+    profiles:
+      - db
+      - all
   lists-web:
     image: ghcr.io/picosh/pico/lists-web:latest
     restart: always

feeds/config.go

@@ -22,6 +22,7 @@ func NewConfigSite() *shared.ConfigSite {
 	minioPass := shared.GetEnv("MINIO_ROOT_PASSWORD", "")
 	dbURL := shared.GetEnv("DATABASE_URL", "")
 	sendgridKey := shared.GetEnv("SENDGRID_API_KEY", "")
+	useImgProxy := shared.GetEnv("USE_IMGPROXY", "1")
 
 	intro := "To get started, enter a username and email.\n"
 	intro += "Then upload a file containing a list of rss feeds (e.g. ~/feeds.txt)\n"
@@ -32,6 +33,7 @@ func NewConfigSite() *shared.ConfigSite {
 		Debug:                debug == "1",
 		SubdomainsEnabled:    subdomains == "1",
 		CustomdomainsEnabled: customdomains == "1",
+		UseImgProxy:          useImgProxy == "1",
 		SendgridKey:          sendgridKey,
 		ConfigCms: config.ConfigCms{
 			Domain:        domain,