Improving Self-Hosting and Removing 3rd Party dependencies.

.eslintrc

@@ -11,6 +11,7 @@
   },
   "plugins": ["@typescript-eslint"],
   "rules": {
-    "semi": [2, "never"]
+    "semi": [2, "never"],
+    "@typescript-eslint/no-unnecessary-type-assertion": [0, "never"]
   }
 }

.github/workflows/build-docker-images.yml

@@ -13,11 +13,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: 'Login to GitHub container registry'
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{github.actor}}
@@ -30,3 +30,4 @@ jobs:
         run: |
           docker build --file packages/content-fetch/Dockerfile . --tag "ghcr.io/omnivore-app/content-fetch:${GITHUB_SHA}" --tag ghcr.io/omnivore-app/content-fetch:latest
           docker push ghcr.io/omnivore-app/content-fetch:${GITHUB_SHA}
+

.github/workflows/build-self-host-docker-images.yml

@@ -0,0 +1,64 @@
+name: Build Self-Hosting Docker Images
+on:
+  push:
+    branches:
+      - main
+      - self-host-updates
+    paths-ignore:
+      - 'apple/**'
+      - 'android/**'
+
+jobs:
+  build-self-hostdocker-images:
+    name: Build self-host docker images
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: 'Login to GitHub container registry'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
+      - name: Build the backend docker image
+        run: |
+          docker build . --file packages/api/Dockerfile --tag "ghcr.io/omnivore-app/sh-backend:${GITHUB_SHA}" --tag ghcr.io/omnivore-app/sh-backend:latest
+          docker push ghcr.io/omnivore-app/sh-backend:${GITHUB_SHA}
+          docker push ghcr.io/omnivore-app/sh-backend:latest
+      - name: Build the content-fetch docker image
+        run: |
+          docker build --file packages/content-fetch/Dockerfile . --tag "ghcr.io/omnivore-app/sh-content-fetch:${GITHUB_SHA}" --tag ghcr.io/omnivore-app/sh-content-fetch:latest
+          docker push ghcr.io/omnivore-app/sh-content-fetch:${GITHUB_SHA}
+          docker push ghcr.io/omnivore-app/sh-content-fetch:latest
+      - name: Build the queue-processor docker image.
+        run: |
+          docker build . --file packages/api/queue-processor/Dockerfile --tag "ghcr.io/omnivore-app/sh-queue-processor:${GITHUB_SHA}" --tag ghcr.io/omnivore-app/sh-queue-processor:latest
+          docker push ghcr.io/omnivore-app/sh-queue-processor:${GITHUB_SHA}
+          docker push ghcr.io/omnivore-app/sh-queue-processor:latest
+      - name: Build the migrate docker image
+        run: |
+          docker build --file packages/db/Dockerfile . --tag "ghcr.io/omnivore-app/sh-migrate:${GITHUB_SHA}" --tag ghcr.io/omnivore-app/sh-migrate:latest
+          docker push ghcr.io/omnivore-app/sh-migrate:${GITHUB_SHA}
+          docker push ghcr.io/omnivore-app/sh-migrate:latest
+      - name: Build the image-proxy docker image
+        run: |
+          cp imageproxy/start_imageproxy.sh .
+          chmod +x start_imageproxy.sh
+          docker build --file imageproxy/Dockerfile . --tag "ghcr.io/omnivore-app/sh-image-proxy:${GITHUB_SHA}" --tag ghcr.io/omnivore-app/sh-image-proxy:latest
+          docker push ghcr.io/omnivore-app/sh-image-proxy:${GITHUB_SHA}
+          docker push ghcr.io/omnivore-app/sh-image-proxy:latest
+      - name: Build the mail-watch-server docker image
+        run: |
+          docker build --file packages/local-mail-watcher/Dockerfile . --tag "ghcr.io/omnivore-app/sh-local-mail-watcher:${GITHUB_SHA}" --tag ghcr.io/omnivore-app/sh-local-mail-watcher:latest
+          docker push ghcr.io/omnivore-app/sh-local-mail-watcher:${GITHUB_SHA}
+          docker push ghcr.io/omnivore-app/sh-local-mail-watcher:latest
+
+

.github/workflows/lint-migrations.yml

@@ -9,7 +9,7 @@ jobs:
   lint_migrations:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
       - name: Fetch main branch
         run: git fetch origin main:main
       - name: Find modified migrations

.github/workflows/run-distiller.yml

@@ -14,12 +14,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: ${{ github.event.pull_request.head.repo.full_name }}
           ref: ${{ github.event.pull_request.head.ref }}
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
           go-version: 1.19
       - name: Generate distiller output for readability
@@ -34,7 +34,7 @@ jobs:
             go-domdistiller file -i $f/source.html -o $f/distiller.html
           done
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
       - name: Generate static html

.github/workflows/run-tests.yaml

@@ -45,7 +45,7 @@ jobs:
         ports:
           - 6379
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Initialize the database
@@ -55,9 +55,9 @@ jobs:
         env:
           PGPASSWORD: postgres
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v4
         with:
-          node-version: 18.16
+          node-version: 22.12.0
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(source ~/.nvm/nvm.sh && yarn cache dir)"
@@ -102,7 +102,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Build the API docker image

.github/workflows/static.yml

@@ -26,13 +26,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup Pages
-        uses: actions/configure-pages@v3
+        uses: actions/configure-pages@v5
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v1
+        uses: actions/upload-pages-artifact@v3
         with:
           path: 'packages/readabilityjs/test'
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v4

.node-version

@@ -1 +1 @@
-18.16
+22.12.0

README.md

@@ -151,24 +151,7 @@ is done fetching your content you will see it in your library.
 
 ## How to deploy to your own server
 
-Omnivore was originally designed to be deployed on GCP and takes advantage
-of some of GCP's PaaS features. We are working to make Omnivore more portable
-so you can easily run the service on your own infrastructure. You can track
-progress here: <https://github.com/omnivore-app/omnivore/issues/25>
-
-To deploy Omnivore on your own hardware you will need to deploy three
-dockerized services and configure access to a postgres service. To handle
-PDF documents you will need to configure access to a Google Cloud Storage
-bucket.
-
-- `packages/api` - the backend API service
-- `packages/web` - the web frontend (can easily be deployed to vercel)
-- `packages/puppeteer-parse` - the content fetching service (can easily
-  be deployed as an AWS lambda or GCP Cloud Function)
-
-Additionally, you will need to run our database migrations to initialize
-your database. These are dockerized and can be run with the
-`packages/db` service.
+A guide for running a self hosted server can be found [here](./self-hosting/GUIDE.md)
 
 ## License
 

android/Omnivore/README.md

@@ -1,5 +1,12 @@
 # Omnivore - Android
 
+## Setup with gradle
+
+In case you do not have Android Studio and you do not want to install it, you may want to use gradlew scripts to build the application.
+
+`./gradlew assembleDebug` should create `./app/build/outputs/apk/debug/app-debug.apk`
+
+
 ## Setup
 
 From the root directory run the following command:

android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/editinfo/EditInfoViewModel.kt

@@ -10,6 +10,7 @@ import app.omnivore.omnivore.R
 import app.omnivore.omnivore.core.data.DataService
 import app.omnivore.omnivore.core.datastore.DatastoreRepository
 import app.omnivore.omnivore.core.datastore.omnivoreAuthToken
+import app.omnivore.omnivore.core.datastore.omnivoreSelfHostedApiServer
 import app.omnivore.omnivore.graphql.generated.UpdatePageMutation
 import app.omnivore.omnivore.graphql.generated.type.UpdatePageInput
 import app.omnivore.omnivore.utils.Constants
@@ -48,6 +49,12 @@ class EditInfoViewModel @Inject constructor(
     datastoreRepo.getString(omnivoreAuthToken)
   }
 
+  fun baseUrl() = runBlocking {
+        datastoreRepo.getString(omnivoreSelfHostedApiServer) ?: Constants.apiURL
+  }
+
+  private fun serverUrl() = "${baseUrl()}/api/graphql"
+
   fun editInfo(itemId: String, title: String, author: String?, description: String?) {
     viewModelScope.launch {
       isLoading = true
@@ -62,7 +69,7 @@ class EditInfoViewModel @Inject constructor(
       }
 
       val apolloClient = ApolloClient.Builder()
-        .serverUrl("${Constants.apiURL}/api/graphql")
+        .serverUrl(serverUrl())
         .addHttpHeader("Authorization", value = authToken)
         .build()
 

android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/onboarding/OnboardingViewModel.kt

@@ -176,6 +176,12 @@ class OnboardingViewModel @Inject constructor(
         resetPendingEmailUserCreds()
     }
 
+    fun baseUrl() = runBlocking {
+        datastoreRepository.getString(omnivoreSelfHostedApiServer) ?: Constants.apiURL
+    }
+
+    private fun serverUrl() = "${baseUrl()}/api/graphql"
+
     fun validateUsername(potentialUsername: String) {
         validateUsernameJob?.cancel()
 
@@ -209,7 +215,7 @@ class OnboardingViewModel @Inject constructor(
             }
 
             val apolloClient =
-                ApolloClient.Builder().serverUrl("${Constants.apiURL}/api/graphql").build()
+                ApolloClient.Builder().serverUrl(serverUrl()).build()
 
             try {
                 val response = apolloClient.query(

android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/save/SaveURLWorker.kt

@@ -16,6 +16,7 @@ import androidx.work.WorkerParameters
 import app.omnivore.omnivore.R
 import app.omnivore.omnivore.core.datastore.DatastoreRepository
 import app.omnivore.omnivore.core.datastore.omnivoreAuthToken
+import app.omnivore.omnivore.core.datastore.omnivoreSelfHostedApiServer
 import app.omnivore.omnivore.graphql.generated.SaveUrlMutation
 import app.omnivore.omnivore.graphql.generated.type.SaveUrlInput
 import app.omnivore.omnivore.utils.Constants
@@ -59,11 +60,16 @@ class SaveURLWorker @AssistedInject constructor(
         }
     }
 
+    suspend fun baseUrl() =
+        datastoreRepository.getString(omnivoreSelfHostedApiServer) ?: Constants.apiURL
+
+    private suspend fun serverUrl() = "${baseUrl()}/api/graphql"
+
     private suspend fun saveURL(url: String): Boolean {
         val authToken = datastoreRepository.getString(omnivoreAuthToken) ?: return false
 
         val apolloClient = ApolloClient.Builder()
-            .serverUrl("${Constants.apiURL}/api/graphql")
+            .serverUrl(serverUrl())
             .addHttpHeader("Authorization", value = authToken)
             .build()
 

android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/save/SaveViewModel.kt

@@ -11,6 +11,7 @@ import androidx.lifecycle.viewModelScope
 import app.omnivore.omnivore.R
 import app.omnivore.omnivore.core.datastore.DatastoreRepository
 import app.omnivore.omnivore.core.datastore.omnivoreAuthToken
+import app.omnivore.omnivore.core.datastore.omnivoreSelfHostedApiServer
 import app.omnivore.omnivore.graphql.generated.SaveUrlMutation
 import app.omnivore.omnivore.graphql.generated.type.SaveUrlInput
 import app.omnivore.omnivore.utils.Constants
@@ -71,6 +72,12 @@ class SaveViewModel @Inject constructor(
     return null
   }
 
+  fun baseUrl() = runBlocking {
+        datastoreRepo.getString(omnivoreSelfHostedApiServer) ?: Constants.apiURL
+  }
+
+  private fun serverUrl() = "${baseUrl()}/api/graphql"
+
   fun saveURL(url: String) {
     viewModelScope.launch {
       isLoading = true
@@ -86,7 +93,7 @@ class SaveViewModel @Inject constructor(
       }
 
       val apolloClient = ApolloClient.Builder()
-        .serverUrl("${Constants.apiURL}/api/graphql")
+        .serverUrl(serverUrl())
         .addHttpHeader("Authorization", value = authToken)
         .build()
 

imageproxy/Dockerfile

@@ -1,4 +1,4 @@
-FROM willnorris/imageproxy:v0.10.0 as build
+FROM ghcr.io/willnorris/imageproxy:main as build
 
 # Above imageproxy image is built from scratch image and is barebones
 # Switching over to ubuntu base image to allow us to debug better.

package.json

@@ -33,7 +33,7 @@
     "graphql-tag": "^2.11.0",
     "lerna": "^7.4.1",
     "prettier": "^2.5.1",
-    "typescript": "4.5.2"
+    "typescript": "5.7.3"
   },
   "volta": {
     "node": "18.16.1",

packages/api/.eslintrc

@@ -4,13 +4,27 @@
     "project": "tsconfig.json"
   },
   "rules": {
-    "@typescript-eslint/no-unsafe-argument": 0
+    "@typescript-eslint/no-unsafe-argument": 0,
+    "@typescript-eslint/no-unsafe-member-access": 0,
+    "@typescript-eslint/no-unnecessary-type-assertion": 0,
+    "@typescript-eslint/no-unsafe-assignment": 0,
+    "@typescript-eslint/no-unsafe-return": ["warn"],
+    "@typescript-eslint/no-unsafe-call" : ["warn"],
+    "@typescript-eslint/restrict-template-expressions": ["warn"],
+    "@typescript-eslint/no-misused-promises": ["warn"],
+    "@typescript-eslint/ban-ts-comment": ["warn"],
+    "@typescript-eslint/no-unused-vars": 0
   },
   "overrides": [
     {
       "files": ["test/**/*.ts"],
       "rules": {
-        "@typescript-eslint/no-unsafe-member-access": 0
+        "@typescript-eslint/no-unsafe-member-access": 0,
+        "@typescript-eslint/no-unnecessary-type-assertion": 0,
+        "@typescript-eslint/no-unsafe-assignment": 0,
+        "@typescript-eslint/no-unsafe-return": ["warn"],
+        "@typescript-eslint/no-unsafe-call" : ["warn"],
+        "@typescript-eslint/no-unused-vars": 0
       }
     }
   ]