chore: resolve merge conflicts

.github/workflows/test.yml

@@ -23,7 +23,12 @@ jobs:
       COUCHDB_HOST: "couchdb"
       COUCHDB_PORT: 5984
       COUCHDB_SECURE: false
-      COUCHDB_SEQ: /tmp/couchdb/sequence_path.txt
+      REDIS_HOST: redis
+      REDIS_PORT: 6379
+      REDIS_KEY: medic
+      POSTGREST_ENDPOINT: postgrest:3000
+      REDIS_BATCH_SIZE: 100
+      POSTGRES_HOST: postgres
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
@@ -50,4 +55,18 @@ jobs:
       - name: Sleep for 60 seconds
         run: sleep 60s
       - name: Run e2e tests
+        run: npm run test:e2e
+  worker-tests:
+    name: Worker Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 16.x
+      - name: Install
+        working-directory: ./redis-worker
+        run: npm install
+      - name: Run worker tests
+        working-directory: ./redis-worker
         run: npm test

docker-compose.yml

@@ -14,6 +14,9 @@ services:
       - COUCHDB_SEQ=/tmp/couchdb/sequence_path.txt
       - COUCHDB_SECURE=${COUCHDB_SECURE:-true}
       - HTTP_ENDPOINT=postgrest:3000
+      - REDIS_HOST=${REDIS_HOST}
+      - REDIS_PORT=${REDIS_PORT}
+      - REDIS_KEY=${REDIS_KEY}
 
   dbt:
     platform: linux/amd64
@@ -31,3 +34,24 @@ services:
       - CHT_PIPELINE_BRANCH_URL=${CHT_PIPELINE_BRANCH_URL}
       - DATAEMON_INTERVAL=${DATAEMON_INTERVAL}
 
+  redis:
+    image: redis:latest
+    restart: unless-stopped
+    ports:
+      - 6379:6379
+    volumes:
+      - redis-data:/data
+
+  redis-worker:
+    build: ./redis-worker/
+    environment:
+      - REDIS_HOST=${REDIS_HOST}
+      - REDIS_PORT=${REDIS_PORT}
+      - REDIS_KEY=${REDIS_KEY}
+      - POSTGREST_ENDPOINT=postgrest:3000
+      - REDIS_BATCH_SIZE=${REDIS_BATCH_SIZE}
+    depends_on:
+      - redis
+    restart: unless-stopped
+volumes:
+  redis-data:

env.template

@@ -24,3 +24,9 @@ COUCHDB_DBS="medic" # space separated list of databases you want to sync e.g "me
 COUCHDB_HOST=couchdb
 COUCHDB_PORT=5984
 COUCHDB_SECURE=false
+
+# redis
+REDIS_HOST=redis
+REDIS_PORT=6379
+REDIS_KEY=couchdb
+REDIS_BATCH_SIZE=100

logstash/pipeline/pipeline.conf.template

@@ -18,17 +18,14 @@ filter {
     json{
         source => "message"
     }
-    mutate {
-       add_field => { "_id" => "%{[doc][_id]}" }
-       add_field => { "_rev" => "%{[doc][_rev]}" }
-    }
+
 }
 
 output {
-    http {
-        format => "json"
-        http_method => "post"
-        ignorable_codes => 409
-        url => "http://${HTTP_ENDPOINT}/${COUCHDB_DB}"
+    redis {
+        host => "${REDIS_HOST}"
+        port => "${REDIS_PORT}"
+        data_type => "list"
+        key => "${REDIS_KEY}"
     }
 }

package.json

@@ -4,7 +4,7 @@
   "description": "",
   "main": "./scripts/index.ts",
   "scripts": {
-    "test": "jest"
+    "test:e2e": "jest --testPathPattern=tests/.*\\.spec\\.ts$"
   },
   "keywords": [],
   "author": "",

redis-worker/.eslintrc

@@ -0,0 +1,54 @@
+{
+  "extends": "@medic",
+  "parserOptions": {
+    "ecmaVersion": 6
+  },
+  "ignorePatterns": [
+    "**/node_modules/**",
+    "dist/**"
+  ],
+  "plugins": ["promise", "node"],
+  "rules": {
+    "array-bracket-newline": ["error", "consistent"],
+    "array-callback-return": ["error", { "allowImplicit": true }],
+    "arrow-spacing": ["error", { "before": true, "after": true }],
+    "brace-style": ["error", "1tbs"],
+    "comma-spacing": ["error", { "before": false, "after": true }],
+    "comma-style": ["error", "last"],
+    "default-param-last": "error",
+    "dot-location": ["error", "property"],
+    "dot-notation": ["error", { "allowKeywords": true }],
+    "func-call-spacing": ["error", "never"],
+    "func-style": ["error", "expression"],
+    "function-call-argument-newline": ["error", "consistent"],
+    "function-paren-newline": ["error", "consistent"],
+    "implicit-arrow-linebreak": ["error", "beside"],
+    "key-spacing": ["error", { "beforeColon": false, "afterColon": true }],
+    "keyword-spacing": ["error", { "before": true, "after": true }],
+    "linebreak-style": ["error", "unix"],
+    "lines-between-class-members": ["error", "always", { "exceptAfterSingleLine": true }],
+    "new-parens": "error",
+    "no-alert": "error",
+    "no-else-return": "error",
+    "no-extra-bind": "error",
+    "no-lone-blocks": "error",
+    "no-nested-ternary": "error",
+    "no-undef-init": "error",
+    "no-useless-rename": "error",
+    "no-whitespace-before-property": "error",
+    "node/no-exports-assign": "error",
+    "rest-spread-spacing": ["error", "never"],
+    "semi-spacing": ["error", { "before": false, "after": true }],
+    "semi-style": ["error", "last"],
+    "template-curly-spacing": "error",
+    "unicode-bom": ["error", "never"]
+  },
+  "overrides": [
+    {
+      "files": [ "**/test/**", "**/tests/**" ],
+      "rules": {
+        "promise/catch-or-return": "error"
+      }
+    }
+  ]
+}

redis-worker/Dockerfile

@@ -0,0 +1,18 @@
+FROM node:20-alpine AS base_build
+
+RUN addgroup -S nonroot \
+    && adduser -S nonroot -G nonroot
+
+WORKDIR /
+
+COPY package*.json ./
+RUN npm install --ignore-scripts
+
+COPY package*.json ./
+COPY tsconfig.json ./
+COPY src ./src
+
+RUN npm run build
+
+USER nonroot
+CMD ["node", "dist/index.js"]

redis-worker/jest.config.ts

@@ -0,0 +1,201 @@
+/**
+ * For a detailed explanation regarding each configuration property, visit:
+ * https://jestjs.io/docs/configuration
+ */
+
+import type {Config} from 'jest';
+
+const config: Config = {
+  // All imported modules in your tests should be mocked automatically
+  // automock: false,
+
+  // Stop running tests after `n` failures
+  // bail: 0,
+
+  // The directory where Jest should store its cached dependency information
+  // cacheDirectory: "/private/var/folders/rm/ntw5nqk559g1ddj0w2ng32nc0000gn/T/jest_dx",
+
+  // Automatically clear mock calls, instances, contexts and results before every test
+  clearMocks: true,
+
+  // Indicates whether the coverage information should be collected while executing the test
+  collectCoverage: true,
+
+  // An array of glob patterns indicating a set of files for which coverage information should be collected
+  // collectCoverageFrom: undefined,
+
+  // The directory where Jest should output its coverage files
+  coverageDirectory: "coverage",
+
+  // An array of regexp pattern strings used to skip coverage collection
+  // coveragePathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // Indicates which provider should be used to instrument code for coverage
+  coverageProvider: "v8",
+
+  // A list of reporter names that Jest uses when writing coverage reports
+  // coverageReporters: [
+  //   "json",
+  //   "text",
+  //   "lcov",
+  //   "clover"
+  // ],
+
+  // An object that configures minimum threshold enforcement for coverage results
+  // coverageThreshold: undefined,
+
+  // A path to a custom dependency extractor
+  // dependencyExtractor: undefined,
+
+  // Make calling deprecated APIs throw helpful error messages
+  // errorOnDeprecated: false,
+
+  // The default configuration for fake timers
+  // fakeTimers: {
+  //   "enableGlobally": false
+  // },
+
+  // Force coverage collection from ignored files using an array of glob patterns
+  // forceCoverageMatch: [],
+
+  // A path to a module which exports an async function that is triggered once before all test suites
+  // globalSetup: undefined,
+
+  // A path to a module which exports an async function that is triggered once after all test suites
+  // globalTeardown: undefined,
+
+  // A set of global variables that need to be available in all test environments
+  // globals: {},
+
+  // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
+  // maxWorkers: "50%",
+
+  // An array of directory names to be searched recursively up from the requiring module's location
+  // moduleDirectories: [
+  //   "node_modules"
+  // ],
+
+  // An array of file extensions your modules use
+  // moduleFileExtensions: [
+  //   "js",
+  //   "mjs",
+  //   "cjs",
+  //   "jsx",
+  //   "ts",
+  //   "tsx",
+  //   "json",
+  //   "node"
+  // ],
+
+  // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
+  // moduleNameMapper: {},
+
+  // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
+  // modulePathIgnorePatterns: [],
+
+  // Activates notifications for test results
+  // notify: false,
+
+  // An enum that specifies notification mode. Requires { notify: true }
+  // notifyMode: "failure-change",
+
+  // A preset that is used as a base for Jest's configuration
+  preset: 'ts-jest',
+
+  // Run tests from one or more projects
+  // projects: undefined,
+
+  // Use this configuration option to add custom reporters to Jest
+  // reporters: undefined,
+
+  // Automatically reset mock state before every test
+  // resetMocks: false,
+
+  // Reset the module registry before running each individual test
+  // resetModules: false,
+
+  // A path to a custom resolver
+  // resolver: undefined,
+
+  // Automatically restore mock state and implementation before every test
+  // restoreMocks: false,
+
+  // The root directory that Jest should scan for tests and modules within
+  // rootDir: undefined,
+
+  // A list of paths to directories that Jest should use to search for files in
+  // roots: [
+  //   "<rootDir>"
+  // ],
+
+  // Allows you to use a custom runner instead of Jest's default test runner
+  // runner: "jest-runner",
+
+  // The paths to modules that run some code to configure or set up the testing environment before each test
+  // setupFiles: [],
+
+  // A list of paths to modules that run some code to configure or set up the testing framework before each test
+  setupFilesAfterEnv: ['./jest.setup.redis-mock.ts'],
+
+  // The number of seconds after which a test is considered as slow and reported as such in the results.
+  // slowTestThreshold: 5,
+
+  // A list of paths to snapshot serializer modules Jest should use for snapshot testing
+  // snapshotSerializers: [],
+
+  // The test environment that will be used for testing
+  // testEnvironment: "jest-environment-node",
+
+  // Options that will be passed to the testEnvironment
+  // testEnvironmentOptions: {},
+
+  // Adds a location field to test results
+  // testLocationInResults: false,
+
+  // The glob patterns Jest uses to detect test files
+  // testMatch: [
+  //   "**/__tests__/**/*.[jt]s?(x)",
+  //   "**/?(*.)+(spec|test).[tj]s?(x)"
+  // ],
+
+  // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
+  // testPathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // The regexp pattern or array of patterns that Jest uses to detect test files
+  // testRegex: [],
+
+  // This option allows the use of a custom results processor
+  // testResultsProcessor: undefined,
+
+  // This option allows use of a custom test runner
+  // testRunner: "jest-circus/runner",
+
+  // A map from regular expressions to paths to transformers
+  transform: {
+    '^.+\\.(ts|tsx)?$': 'ts-jest',
+  }
+
+  // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
+  // transformIgnorePatterns: [
+  //   "/node_modules/",
+  //   "\\.pnp\\.[^\\/]+$"
+  // ],
+
+  // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
+  // unmockedModulePathPatterns: undefined,
+
+  // Indicates whether each individual test should be reported during the run
+  // verbose: undefined,
+
+  // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
+  // watchPathIgnorePatterns: [],
+
+  // Whether to use watchman for file crawling
+  // watchman: true,
+};
+
+export default config;

redis-worker/jest.setup.redis-mock.ts

@@ -0,0 +1 @@
+jest.mock('redis', () => jest.requireActual('redis-mock'));