[TASK] Simplify EXT:Solr* plugins ingetration : EXT:Tika

* Simplifies the integration/installation of EXT:Tika
* Improves tika test environment
   Set environment variables in order to run tika integration and
   unit tests within the ddev environment.

Closes: #18

.ddev/commands/web/tests-integration

@@ -2,21 +2,21 @@
 
 ## Description: run integration tests in web container (don't forget to run `ddev composer-update--lock` if ext-solr*'s composer.json changed)
 
-INTEGRATION_PHPUINT_BOOTSTRAP="vendor/nimut/testing-framework/res/Configuration/FunctionalTestsBootstrap.php"
+INTEGRATION_BOOTSTRAP="vendor/nimut/testing-framework/res/Configuration/FunctionalTestsBootstrap.php"
 
-INTEGRATION_PHPUINT_CONFIGURATION="public/typo3conf/ext/solr/Build/Test/IntegrationTests.xml"
+INTEGRATION_CONFIGURATION="public/typo3conf/ext/solr/Build/Test/IntegrationTests.xml"
 ./vendor/bin/phpunit \
-  --bootstrap=$INTEGRATION_PHPUINT_BOOTSTRAP \
-  --configuration=$INTEGRATION_PHPUINT_CONFIGURATION \
+  --bootstrap=$INTEGRATION_BOOTSTRAP \
+  --configuration=$INTEGRATION_CONFIGURATION \
   --colors "$@"
 
 
 echo "Frontend Integration Tests:"
 echo "-"
-INTEGRATION_PHPUINT_CONFIGURATION="public/typo3conf/ext/solr/Build/Test/IntegrationFrontendTests.xml"
+INTEGRATION_CONFIGURATION="public/typo3conf/ext/solr/Build/Test/IntegrationFrontendTests.xml"
 ./vendor/bin/phpunit \
-  --bootstrap=$INTEGRATION_PHPUINT_BOOTSTRAP \
-  --configuration=$INTEGRATION_PHPUINT_CONFIGURATION \
+  --bootstrap=$INTEGRATION_BOOTSTRAP \
+  --configuration=$INTEGRATION_CONFIGURATION \
   --colors "$@"
 
 # todo: rewrite and reuse the same cibuild.sh scripts from EXT:Solr*

.ddev/commands/web/tests-integration-solrfal

@@ -2,6 +2,9 @@
 
 ## Description: run integration tests in web container
 
-UNIT_BOOTSTRAP="vendor/nimut/testing-framework/res/Configuration/FunctionalTestsBootstrap.php"
-SOLR_UNIT_TESTS_PATH="public/typo3conf/ext/solrfal/Build/Test/IntegrationTests.xml"
-./vendor/bin/phpunit --bootstrap=$UNIT_BOOTSTRAP --configuration=$SOLR_UNIT_TESTS_PATH --colors "$@"
+INTEGRATION_BOOTSTRAP="vendor/nimut/testing-framework/res/Configuration/FunctionalTestsBootstrap.php"
+INTEGRATION_CONFIGURATION="public/typo3conf/ext/solrfal/Build/Test/IntegrationTests.xml"
+./vendor/bin/phpunit \
+  --bootstrap=$INTEGRATION_BOOTSTRAP \
+  --configuration=$INTEGRATION_CONFIGURATION \
+  --colors "$@"

.ddev/commands/web/tests-integration-tika

@@ -1,9 +1,22 @@
 #!/bin/bash
 
 ## Description: run integration tests in web container
+TIKA_VERSION=$(yq e '.env.TIKA_VERSION' packages/ext-tika/.github/workflows/ci.yml)
+TIKA_SERVER_SOURCE="https://archive.apache.org/dist/tika/"
+if [ ! -f "$TIKA_PATH/tika-app-$TIKA_VERSION.jar" ]; then
+  echo "The the Tika App jar binaries will be downloaded. They are required for integration tests of tika."
+	wget ${TIKA_SERVER_SOURCE}tika-app-$TIKA_VERSION.jar -O "$TIKA_PATH/tika-app-$TIKA_VERSION.jar"
+	if [ ! -f "$TIKA_PATH/tika-app-$TIKA_VERSION.jar" ]; then
+		echo "Could not download tika-app-$TIKA_VERSION.jar from ${TIKA_SERVER_SOURCE}tika-app-$TIKA_VERSION.jar URL"
+		exit 1
+	fi
+	echo "Download of tika-app-$TIKA_VERSION.jar successful. See \".ddev/tika-jars\", which is mounted in web containers \"/opt/tika\" path."
+fi
 
-UNIT_BOOTSTRAP="vendor/nimut/testing-framework/res/Configuration/FunctionalTestsBootstrap.php"
-SOLR_UNIT_TESTS_PATH="public/typo3conf/ext/tika/Build/Test/IntegrationTests.xml"
 
-
-./vendor/bin/phpunit --colors -c $SOLR_UNIT_TESTS_PATH --bootstrap=$UNIT_BOOTSTRAP "$@"
+INTEGRATION_BOOTSTRAP="vendor/nimut/testing-framework/res/Configuration/FunctionalTestsBootstrap.php"
+INTEGRATION_CONFIGURATION="public/typo3conf/ext/tika/Build/Test/IntegrationTests.xml"
+./vendor/bin/phpunit \
+  --bootstrap=$INTEGRATION_BOOTSTRAP \
+  --configuration $INTEGRATION_CONFIGURATION \
+  --colors "$@"

.ddev/commands/web/tests-unit

@@ -3,6 +3,9 @@
 ## Description: run unit tests in web container (don't forget to run `ddev composer-update--lock` if ext-solr*'s composer.json changed)
 
 UNIT_BOOTSTRAP="vendor/nimut/testing-framework/res/Configuration/UnitTestsBootstrap.php"
-SOLR_UNIT_TESTS_PATH="public/typo3conf/ext/solr/Build/Test/UnitTests.xml"
+UNIT_CONFIGURATION="public/typo3conf/ext/solr/Build/Test/UnitTests.xml"
 
-./vendor/bin/phpunit --colors -c $SOLR_UNIT_TESTS_PATH --bootstrap=$UNIT_BOOTSTRAP "$@"
+./vendor/bin/phpunit \
+  --bootstrap=$UNIT_BOOTSTRAP \
+  --configuration $UNIT_CONFIGURATION \
+  --colors "$@"

.ddev/commands/web/tests-unit-solrfal

@@ -3,6 +3,9 @@
 ## Description: run unit tests in web container
 
 UNIT_BOOTSTRAP="vendor/nimut/testing-framework/res/Configuration/UnitTestsBootstrap.php"
-SOLR_UNIT_TESTS_PATH="public/typo3conf/ext/solrfal/Build/Test/UnitTests.xml"
+UNIT_CONFIGURATION="public/typo3conf/ext/solrfal/Build/Test/UnitTests.xml"
 
-./vendor/bin/phpunit --colors -c $SOLR_UNIT_TESTS_PATH --bootstrap=$UNIT_BOOTSTRAP "$@"
+./vendor/bin/phpunit \
+  --bootstrap=$UNIT_BOOTSTRAP \
+  --configuration $UNIT_CONFIGURATION \
+  --colors "$@"

.ddev/commands/web/tests-unit-tika

@@ -3,6 +3,9 @@
 ## Description: run unit tests in web container (don't forget to run `ddev composer-update--lock` if ext-solr*'s composer.json changed)
 
 UNIT_BOOTSTRAP="vendor/nimut/testing-framework/res/Configuration/UnitTestsBootstrap.php"
-SOLR_UNIT_TESTS_PATH="public/typo3conf/ext/tika/Build/Test/UnitTests.xml"
+UNIT_CONFIGURATION="public/typo3conf/ext/tika/Build/Test/UnitTests.xml"
 
-./vendor/bin/phpunit --colors -c $SOLR_UNIT_TESTS_PATH --bootstrap=$UNIT_BOOTSTRAP "$@"
+./vendor/bin/phpunit \
+  --bootstrap=$UNIT_BOOTSTRAP \
+  --configuration $UNIT_CONFIGURATION \
+  --colors "$@"

.ddev/config.yaml

@@ -21,7 +21,7 @@ hooks:
   - exec: composer install --prefer-source -d /var/www/html
   # setup system cron and run TYPO3 Scheduler tasks via system cron.
   - exec: sudo service cron start && echo '* * * * * TYPO3_CONTEXT=Development /usr/bin/php /var/www/html/vendor/bin/typo3cms scheduler:run' | crontab -
-webimage_extra_packages: [cron, jq, openjdk-11-jre-headless, parallel]
+webimage_extra_packages: [cron, jq, yq, openjdk-11-jre-headless, parallel]
 use_dns_when_possible: true
 timezone: UTC
 

.ddev/docker-compose.env.yaml

@@ -5,8 +5,6 @@ services:
     volumes:
       - "~/.composer/cache/:/home/.composer/cache/"
     environment:
-      - TIKA_VERSION=1.24.1
-
       - PHP_IDE_CONFIG=serverName=$DDEV_SITENAME.ddev.site
       - XDEBUG_CONFIG=idekey=PHPSTORM
 
@@ -27,6 +25,12 @@ services:
       - TYPO3_INSTALL_SITE_NAME=$DDEV_SITENAME (on DDEV)
       - TYPO3_INSTALL_SITE_SETUP_TYPE=no
 
+      # Env vars for EXT:Solr* stack
+      # EXT:tika
+      - TESTING_TIKA_SERVER_HOST=tika
+      - TIKA_PATH=/opt/tika
+      # TIKA_VERSION is fetched from packages/ext-tika/.github/workflows/ci.yml ->env.TIKA_VERSION
+
       - TYPO3_HOST=${DDEV_SITENAME}.ddev.local
 
       # test environment for "standard EXT:Solr CI Build" for running inside of ddev web container

.ddev/php/30-xdebug_code_coverage.ini

@@ -0,0 +1 @@
+xdebug.mode=debug,develop,coverage

.gitignore

@@ -20,3 +20,5 @@ nbproject
 .ddev/solr/tests
 
 /releses/
+
+.ddev/tika-jars/

packages/introduction_tika/composer.json

@@ -0,0 +1,17 @@
+{
+    "name": "apache-solr-for-typo3/introduction-tika",
+    "type": "typo3-cms-extension",
+    "description": "Solr development site : EXT:tika for Introduction",
+    "homepage": "http://www.dkd.de/",
+    "license": ["GPL-2.0-or-later"],
+    "keywords": ["TYPO3 CMS"],
+    "version": "1.0.0",
+    "require": {
+        "apache-solr-for-typo3/tika": "@dev"
+    },
+    "extra": {
+        "typo3/cms": {
+            "extension-key": "introduction_tika"
+        }
+    }
+}

packages/introduction_tika/ext_emconf.php

@@ -0,0 +1,26 @@
+<?php
+
+/**
+ * Extension Manager/Repository config file for ext "introduction_tika".
+ */
+$EM_CONF[$_EXTKEY] = [
+    'title' => 'Apache Solr for TYPO3 Sitepackage : EXT:tika for Introduction',
+    'description' => 'Solr development site : EXT:tika Introduction',
+    'category' => 'distribution',
+    'constraints' => [
+        'depends' => [
+            'typo3' => '10.4.0-10.4.99',
+            'tika' => '*'
+        ],
+        'conflicts' => [
+        ],
+    ],
+    'state' => 'stable',
+    'uploadfolder' => 0,
+    'createDirs' => '',
+    'clearCacheOnLoad' => 1,
+    'author' => 'dkd Internet Service GmbH',
+    'author_email' => '[email protected]',
+    'author_company' => 'dkd',
+    'version' => '1.0.0',
+];

packages/introduction_tika/ext_localconf.php

@@ -0,0 +1,2 @@
+<?php
+defined('TYPO3_MODE') || die();

public/typo3conf/AdditionalConfiguration.php

@@ -6,4 +6,40 @@
 $GLOBALS['TYPO3_CONF_VARS']['SYS']['trustedHostsPattern']  = 'localhost|solr-ddev-site.ddev.site';
 $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['sites'][1]['domains'][]  = 'solr-ddev-site.ddev.site';
 
-$GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['tika'] = 'a:14:{s:9:"extractor";s:4:"solr";s:7:"logging";s:1:"0";s:18:"showTikaSolrModule";s:1:"0";s:16:"excludeMimeTypes";s:0:"";s:13:"fileSizeLimit";s:3:"500";s:8:"tikaPath";s:0:"";s:14:"tikaServerPath";s:0:"";s:16:"tikaServerScheme";s:4:"http";s:14:"tikaServerHost";s:9:"localhost";s:14:"tikaServerPort";s:4:"9998";s:10:"solrScheme";s:4:"http";s:8:"solrHost";s:4:"solr";s:8:"solrPort";s:4:"8983";s:8:"solrPath";s:14:"/solr/core_en/";}';
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -*/
+/* Configure Tika within solr-ddev-site                                                                               */
+/*   Notes:                                                                                                           */
+/*     Only "Extractor" option is writable via BE->Setting->Extension Configuration->tika                             */
+if (isset($GLOBALS['TYPO3_CONF_VARS']['EXTENSIONS']['tika'])) {
+    \TYPO3\CMS\Core\Utility\ArrayUtility::mergeRecursiveWithOverrule(
+        $GLOBALS['TYPO3_CONF_VARS']['EXTENSIONS']['tika'],
+        [
+            // # General
+
+            // Enable Logging
+            'logging' => '0',
+            // Show Tika Backend Module ::: Enables a Tika module within the Solr backend module (experimental, only works with Tika server)
+            'showTikaSolrModule' => '0',
+            // List of mime types to be excluded in metadata extraction
+            'excludeMimeTypes' => '',
+            // File size limit when a file should be processed (configured in MB)
+            'fileSizeLimit' => 500,
+
+            // # Tika App
+            // Tika App configuration
+            'tikaPath' => '',
+
+            // # Tika Server // Uses ddev's configs from .ddev/docker-compose.tika.yaml
+            'tikaServerScheme' => 'http',
+            'tikaServerHost' => 'tika',
+            'tikaServerPort' => '9998',
+            'tikaServerPath' => '',         // Tika Server Jar Path ::: [Optional] The absolute path to your Apache Tika server jar file (tika-server-x.x.jar). When set you can use the backend module to start and stop the Tika server from the TYPO3 backend. Otherwise the host and port settings will be used.
+
+            // # Solr Server // Uses ddev's configs from  .ddev/docker-compose.solr.yaml
+            'solrScheme' => 'http',
+            'solrHost' => 'solr-site',
+            'solrPort' => '8983',
+            'solrPath' => '/solr/core_en/'
+        ]
+    );
+}