Remove gearman (#439)

Co-authored-by: Scott Aubrey <[email protected]>
Co-authored-by: Ali Amin <[email protected]>
Co-authored-by: Nathan Lisgo <[email protected]>

.github/workflows/ci.yaml

@@ -11,6 +11,7 @@ on:
     branches:
       - master
       - develop
+      - remove-gearman
     tags:
       - '*'
 
@@ -50,7 +51,7 @@ jobs:
   build-and-push:
     needs: [tests]
     runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop'
+    if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/remove-gearman'
     env:
       IMAGE_REPO: ghcr.io/elifesciences/search
       BRANCH_NAME: ${{ github.head_ref || github.ref_name }}

Dockerfile

@@ -5,8 +5,7 @@ FROM php:7.4-apache AS base
 
 USER root
 
-RUN apt-get update && apt-get install libgearman-dev zip unzip gearman-tools bc procps jq -y && rm -rf /var/lib/apt/lists/*
-RUN pecl install gearman && docker-php-ext-enable gearman
+RUN apt-get update && apt-get install zip unzip bc procps jq -y && rm -rf /var/lib/apt/lists/*
 RUN docker-php-ext-configure pcntl --enable-pcntl &&  docker-php-ext-install pcntl
 
 COPY .docker/php.ini /usr/local/etc/php/php.ini

README.md

@@ -18,7 +18,7 @@ To bring up all services, run:
 docker compose up
 ```
 
-Alternatively, you can run without the SQS queue watcher and gearman worker by just bring up the app service:
+Alternatively, you can run without the SQS queue watcher by just bring up the app service:
 ```shell
 docker compose up app
 ```
@@ -31,11 +31,12 @@ The `bin/console queue:import` command imports items from API (in dev this is th
 docker compose exec app bin/console queue:import all
 ```
 
-> **Note**: `all` here means all types of search content. Other possible values can be found in src/Search/Gearman/Command/ImportCommand.php
+> **Note**: `all` here means all types of search content. Other possible values can be found in src/Search/Queue/Command/ImportCommand.php
 
-If you are running the workers and queue watcher, you should now see the results by accessing the search API on http://localhost:8888/search
+If you are running the queue watcher, you should now see the results by accessing the search API on http://localhost:8888/search
+
+If you are not running the watcher, inspect the queue count via
 
-If you are not running the worker, inspect the queue count via
 ```shell
 docker compose exec app bin/console queue:count
 ```
@@ -49,11 +50,10 @@ docker compose exec app vendor/bin/phpunit
 
 To run all the project tests (inc above tests and integration tests)
 ```shell
-docker compose down gearman-worker
 docker compose down queue-watcher
 docker compose exec app bash project_tests.sh
 ```
-NOTE: these integration tests require the queue watcher and gearman worker to not be running so the tests can control when items are consumed. This is why we make sure to stop worker/watcher services.
+NOTE: these integration tests require the queue watcher  to not be running so the tests can control when items are consumed. This is why we make sure to stop watcher services.
 
 To run the smoke tests:
 ```shell

bin/ci-empty-queue-and-wait

@@ -20,24 +20,3 @@ for i in $(seq 60); do
 
   sleep 1
 done
-
-# Clean the Gearman Queue
-gearadmin -h ${GEARMAN_HOST:-localhost} --show-jobs | cut -f 1 | xargs -I % gearadmin -h ${GEARMAN_HOST:-localhost} --cancel-job % > /dev/null 2>&1
-
-echo -n "Waiting for job queue to empty..."
-for i in $(seq 60); do
-  echo -n "."
-
-  gearman_queue=$(gearadmin -h ${GEARMAN_HOST:-localhost} --show-jobs | cut -f 4 | paste -sd+ | bc || true)
-  if [ "$gearman_queue" -eq "0" ]; then
-    echo "OK"
-    break
-  fi
-
-  if [ "$i" -eq "60" ]; then
-    echo "FAILED"
-    exit 1
-  fi
-
-  sleep 1
-done

bin/ci-start-processes

@@ -1,23 +1,5 @@
 #!/bin/bash
 
-echo -n "Waiting for lingering workers to terminate..."
-for i in $(seq 60); do
-  echo -n "."
-
-  workers=$(pgrep -cf gearman:worker || true)
-  if [ "$workers" -eq "0" ]; then
-    echo "OK"
-    break
-  fi
-
-  if [ "$i" -eq "60" ]; then
-    echo "FAILED"
-    exit 1
-  fi
-
-  sleep 1
-done
-
 echo -n "Waiting for lingering watchers to terminate..."
 for i in $(seq 60); do
   echo -n "."
@@ -36,10 +18,6 @@ for i in $(seq 60); do
   sleep 1
 done
 
-echo "Starting a gearman:worker"
-./bin/console gearman:worker >> /tmp/gearman-worker.log 2>&1 &
-workerPid=$!
-echo "gearman:worker PID $workerPid"
 
 echo "Starting a queue:watch"
 ./bin/console queue:watch >> /tmp/queue-watch.log 2>&1 &

bin/ci-wait-steady-state

@@ -5,9 +5,7 @@ for i in $(seq 60); do
   echo -n "."
 
   sqs_queue=$(bin/console queue:count || true)
-  gearman_queue=$(gearadmin -h ${GEARMAN_HOST:-localhost} --show-jobs | wc -l || true)
-  # NOTE: gearadmin includes an additional line in it's output that needs to be ignored, hence the 1 rather than 0.
-  if [ "$sqs_queue" -eq "0" ] && [ "$gearman_queue" -eq "1" ]; then
+  if [ "$sqs_queue" -eq "0" ]; then
     echo "OK"
     break
   fi

bin/console

@@ -11,9 +11,6 @@ use Symfony\Component\Console\Output\ConsoleOutput;
 require_once __DIR__.'/../src/bootstrap.php';
 set_time_limit(0);
 
-if (!defined('GEARMAN_INSTALLED')) {
-    define('GEARMAN_INSTALLED', class_exists('GearmanClient'));
-}
 $app = new Kernel($config);
 $input = new ArgvInput();
 $output = new ConsoleOutput();

bin/reindex

@@ -10,19 +10,14 @@ fi
 write_index_backup=$(./bin/console index:read)
 index_name="$1"
 
-echo "Stopping gearman:worker instances"
-sudo systemctl stop search-gearman-worker-controller.target
-sudo systemctl stop search-queue-watch-controller.target
-
 echo "Creating new index $index_name"
 ./bin/console search:setup --index="$index_name" --delete
 
 echo "Switching writes to new index $index_name"
 ./bin/console index:switch:write "$index_name"
 
-echo "Restarting processes"
-sudo systemctl start search-gearman-worker-controller.target
-sudo systemctl start search-queue-watch-controller.target
+echo "Restarting services"
+sudo systemctl restart queuewatch.service
 
 echo "Importing all API content"
 ./bin/console queue:import all
@@ -38,10 +33,6 @@ if [ "$index_check" -eq "0" ]; then
     echo "Switching writes back to original write index $write_index_backup"
     ./bin/console index:switch:write "$write_index_backup"
 
-    echo "Restarting processes"
-    sudo systemctl start search-gearman-worker-controller.target
-    sudo systemctl start search-queue-watch-controller.target
-
     echo "Deleting unused indexes"
     ./bin/console index:delete:unused
 

bin/wait-for-empty-queues

@@ -5,10 +5,6 @@ set -e
 # with more workers this becomes less unlikely.
 # more workers also means more CPU usage which means less CPU available for serving traffic.
 
-# how many attempts per-loop should be made to detect workers?
-num_workers_threshold=100 # 100 attempts with a .10s interval is 10 seconds
-num_workers_attempt=0 # which attempt are we on now?
-
 # same again, but for the sqs watcher
 num_watches_threshold=100 # 100 attempts with a .10s interval is 10 seconds
 num_watches_attempt=0
@@ -17,20 +13,8 @@ echo "Waiting for empty queues"
 
 while true; do
     # `|| true` avoid failing the script when there are no results
-    workers=$(pgrep -cf gearman:worker || true)
     watches=$(pgrep -cf queue:watch || true)
 
-    if [ "$workers" -eq "0" ]; then
-        if [ "$num_workers_attempt" = "$num_workers_threshold" ]; then
-            echo "No alive gearman:worker processes"
-            break
-        fi
-        num_workers_attempt=$((num_workers_attempt+1))
-        sleep .10 # ten checks in a second
-        continue
-    fi
-    num_workers_attempt=0 # we've found a worker, reset the counter
-
     if [ "$watches" -eq "0" ]; then
         if [ "$num_watches_attempt" = "$num_watches_threshold" ]; then
             echo "No alive queue:watch processes"
@@ -44,9 +28,7 @@ while true; do
 
     sqs_queue=$(bin/console queue:count)
     echo "Job in SQS queue (approximate): $sqs_queue"
-    gearman_queue=$(gearadmin -h ${GEARMAN_HOST:-localhost} --status | cut -f 2 | paste -sd+ | bc)
-    echo "Job in Gearman queue: $gearman_queue"
-    if [ "$sqs_queue" -eq 0 ] && [ "$gearman_queue" -eq "0" ]; then
+    if [ "$sqs_queue" -eq 0 ]; then
         echo "No more jobs, finished waiting"
         exit 0
     fi

composer.json

@@ -14,7 +14,6 @@
     },
     "require": {
         "php": "^7.4",
-        "ext-gearman": "*",
         "aws/aws-sdk-php": "^3.19",
         "clue/block-react": "^1.1",
         "doctrine/cache": "^1.6",

config.php.dist

@@ -10,7 +10,6 @@ return [
     'elastic_logging' => true,
     'elastic_force_sync' => true,
     'elastic_ssl_verification' => false,
-    'gearman_servers' => ['gearman'],
     'api_url' => 'http://api-dummy:8080',
     'api_requests_batch' => 10,
     'rate_limit_minimum_page' => 2,

docker-compose.yaml

@@ -38,14 +38,6 @@ services:
             timeout: 2s
             retries: 10
 
-    gearman:
-        image: artefactual/gearmand:latest
-        healthcheck:
-            test: ["CMD-SHELL", "gearadmin --status"]
-            interval: 2s
-            timeout: 2s
-            retries: 3
-
     localstack:
         image: localstack/localstack:2.3.2
         environment:
@@ -76,8 +68,6 @@ services:
         depends_on:
             opensearch:
                 condition: service_healthy
-            gearman:
-                condition: service_healthy
             localstack:
                 condition: service_healthy
 
@@ -105,21 +95,6 @@ services:
         extra_hosts:
             host.docker.internal: host-gateway
 
-    gearman-worker:
-        build:
-            context: .
-            dockerfile: Dockerfile
-            target: dev
-        command: bin/console gearman:worker
-        environment:
-            - GEARMAN_HOST=gearman
-        volumes:
-            - ./:/app
-            - ./config.php:/app/config.php
-        depends_on:
-            setup:
-                condition: service_completed_successfully
-
     queue-watcher:
         build:
             context: .