diff --git a/.ddev/addon-metadata/elasticsearch/manifest.yaml b/.ddev/addon-metadata/elasticsearch/manifest.yaml new file mode 100644 index 0000000..18cda67 --- /dev/null +++ b/.ddev/addon-metadata/elasticsearch/manifest.yaml @@ -0,0 +1,8 @@ +name: elasticsearch +repository: ddev/ddev-elasticsearch +version: v0.2.0 +install_date: "2023-10-13T15:08:20+13:00" +project_files: + - docker-compose.elasticsearch.yaml +global_files: [] +removal_actions: [] diff --git a/.ddev/config.yaml b/.ddev/config.yaml new file mode 100644 index 0000000..705773c --- /dev/null +++ b/.ddev/config.yaml @@ -0,0 +1,263 @@ +name: elastictest +type: silverstripe +docroot: public +php_version: "8.1" +webserver_type: apache-fpm +xdebug_enabled: false +additional_hostnames: [] +additional_fqdns: [] +database: + type: mariadb + version: "10.4" +use_dns_when_possible: false +composer_version: "2" +web_environment: [] +nodejs_version: "18" + +# Key features of DDEV's config.yaml: + +# name: # Name of the project, automatically provides +# http://projectname.ddev.site and https://projectname.ddev.site + +# type: # drupal6/7/8, backdrop, typo3, wordpress, php + +# docroot: # Relative path to the directory containing index.php. + +# php_version: "8.1" # PHP version to use, "5.6", "7.0", "7.1", "7.2", "7.3", "7.4", "8.0", "8.1", "8.2", "8.3" + +# You can explicitly specify the webimage but this +# is not recommended, as the images are often closely tied to DDEV's' behavior, +# so this can break upgrades. + +# webimage: # nginx/php docker image. + +# database: +# type: # mysql, mariadb, postgres +# version: # database version, like "10.4" or "8.0" +# MariaDB versions can be 5.5-10.8 and 10.11, MySQL versions can be 5.5-8.0 +# PostgreSQL versions can be 9-15. + +# router_http_port: # Port to be used for http (defaults to global configuration, usually 80) +# router_https_port: # Port for https (defaults to global configuration, usually 443) + +# xdebug_enabled: false # Set to true to enable Xdebug and "ddev start" or "ddev restart" +# Note that for most people the commands +# "ddev xdebug" to enable Xdebug and "ddev xdebug off" to disable it work better, +# as leaving Xdebug enabled all the time is a big performance hit. + +# xhprof_enabled: false # Set to true to enable Xhprof and "ddev start" or "ddev restart" +# Note that for most people the commands +# "ddev xhprof" to enable Xhprof and "ddev xhprof off" to disable it work better, +# as leaving Xhprof enabled all the time is a big performance hit. + +# webserver_type: nginx-fpm, apache-fpm, or nginx-gunicorn + +# timezone: Europe/Berlin +# This is the timezone used in the containers and by PHP; +# it can be set to any valid timezone, +# see https://en.wikipedia.org/wiki/List_of_tz_database_time_zones +# For example Europe/Dublin or MST7MDT + +# composer_root: +# Relative path to the Composer root directory from the project root. This is +# the directory which contains the composer.json and where all Composer related +# commands are executed. + +# composer_version: "2" +# You can set it to "" or "2" (default) for Composer v2 or "1" for Composer v1 +# to use the latest major version available at the time your container is built. +# It is also possible to use each other Composer version channel. This includes: +# - 2.2 (latest Composer LTS version) +# - stable +# - preview +# - snapshot +# Alternatively, an explicit Composer version may be specified, for example "2.2.18". +# To reinstall Composer after the image was built, run "ddev debug refresh". + +# nodejs_version: "18" +# change from the default system Node.js version to another supported version, like 14, 16, 18, 20. +# Note that you can use 'ddev nvm' or nvm inside the web container to provide nearly any +# Node.js version, including v6, etc. + +# additional_hostnames: +# - somename +# - someothername +# would provide http and https URLs for "somename.ddev.site" +# and "someothername.ddev.site". + +# additional_fqdns: +# - example.com +# - sub1.example.com +# would provide http and https URLs for "example.com" and "sub1.example.com" +# Please take care with this because it can cause great confusion. + +# upload_dirs: "custom/upload/dir" +# +# upload_dirs: +# - custom/upload/dir +# - ../private +# +# would set the destination paths for ddev import-files to /custom/upload/dir +# When Mutagen is enabled this path is bind-mounted so that all the files +# in the upload_dirs don't have to be synced into Mutagen. + +# disable_upload_dirs_warning: false +# If true, turns off the normal warning that says +# "You have Mutagen enabled and your 'php' project type doesn't have upload_dirs set" + +# working_dir: +# web: /var/www/html +# db: /home +# would set the default working directory for the web and db services. +# These values specify the destination directory for ddev ssh and the +# directory in which commands passed into ddev exec are run. + +# omit_containers: [db, ddev-ssh-agent] +# Currently only these containers are supported. Some containers can also be +# omitted globally in the ~/.ddev/global_config.yaml. Note that if you omit +# the "db" container, several standard features of DDEV that access the +# database container will be unusable. In the global configuration it is also +# possible to omit ddev-router, but not here. + +# performance_mode: "global" +# DDEV offers performance optimization strategies to improve the filesystem +# performance depending on your host system. Should be configured globally. +# +# If set, will override the global config. Possible values are: +# - "global": uses the value from the global config. +# - "none": disables performance optimization for this project. +# - "mutagen": enables Mutagen for this project. +# - "nfs": enables NFS for this project. +# +# See https://ddev.readthedocs.io/en/latest/users/install/performance/#nfs +# See https://ddev.readthedocs.io/en/latest/users/install/performance/#mutagen + +# fail_on_hook_fail: False +# Decide whether 'ddev start' should be interrupted by a failing hook + +# host_https_port: "59002" +# The host port binding for https can be explicitly specified. It is +# dynamic unless otherwise specified. +# This is not used by most people, most people use the *router* instead +# of the localhost port. + +# host_webserver_port: "59001" +# The host port binding for the ddev-webserver can be explicitly specified. It is +# dynamic unless otherwise specified. +# This is not used by most people, most people use the *router* instead +# of the localhost port. + +# host_db_port: "59002" +# The host port binding for the ddev-dbserver can be explicitly specified. It is dynamic +# unless explicitly specified. + +# mailpit_http_port: "8025" +# mailpit_https_port: "8026" +# The Mailpit ports can be changed from the default 8025 and 8026 + +# host_mailpit_port: "8025" +# The mailpit port is not normally bound on the host at all, instead being routed +# through ddev-router, but it can be bound directly to localhost if specified here. + +# webimage_extra_packages: [php7.4-tidy, php-bcmath] +# Extra Debian packages that are needed in the webimage can be added here + +# dbimage_extra_packages: [telnet,netcat] +# Extra Debian packages that are needed in the dbimage can be added here + +# use_dns_when_possible: true +# If the host has internet access and the domain configured can +# successfully be looked up, DNS will be used for hostname resolution +# instead of editing /etc/hosts +# Defaults to true + +# project_tld: ddev.site +# The top-level domain used for project URLs +# The default "ddev.site" allows DNS lookup via a wildcard +# If you prefer you can change this to "ddev.local" to preserve +# pre-v1.9 behavior. + +# ngrok_args: --basic-auth username:pass1234 +# Provide extra flags to the "ngrok http" command, see +# https://ngrok.com/docs/ngrok-agent/config or run "ngrok http -h" + +# disable_settings_management: false +# If true, DDEV will not create CMS-specific settings files like +# Drupal's settings.php/settings.ddev.php or TYPO3's AdditionalConfiguration.php +# In this case the user must provide all such settings. + +# You can inject environment variables into the web container with: +# web_environment: +# - SOMEENV=somevalue +# - SOMEOTHERENV=someothervalue + +# no_project_mount: false +# (Experimental) If true, DDEV will not mount the project into the web container; +# the user is responsible for mounting it manually or via a script. +# This is to enable experimentation with alternate file mounting strategies. +# For advanced users only! + +# bind_all_interfaces: false +# If true, host ports will be bound on all network interfaces, +# not the localhost interface only. This means that ports +# will be available on the local network if the host firewall +# allows it. + +# default_container_timeout: 120 +# The default time that DDEV waits for all containers to become ready can be increased from +# the default 120. This helps in importing huge databases, for example. + +#web_extra_exposed_ports: +#- name: nodejs +# container_port: 3000 +# http_port: 2999 +# https_port: 3000 +#- name: something +# container_port: 4000 +# https_port: 4000 +# http_port: 3999 +# Allows a set of extra ports to be exposed via ddev-router +# Fill in all three fields even if you don’t intend to use the https_port! +# If you don’t add https_port, then it defaults to 0 and ddev-router will fail to start. +# +# The port behavior on the ddev-webserver must be arranged separately, for example +# using web_extra_daemons. +# For example, with a web app on port 3000 inside the container, this config would +# expose that web app on https://.ddev.site:9999 and http://.ddev.site:9998 +# web_extra_exposed_ports: +# - name: myapp +# container_port: 3000 +# http_port: 9998 +# https_port: 9999 + +#web_extra_daemons: +#- name: "http-1" +# command: "/var/www/html/node_modules/.bin/http-server -p 3000" +# directory: /var/www/html +#- name: "http-2" +# command: "/var/www/html/node_modules/.bin/http-server /var/www/html/sub -p 3000" +# directory: /var/www/html + +# override_config: false +# By default, config.*.yaml files are *merged* into the configuration +# But this means that some things can't be overridden +# For example, if you have 'nfs_mount_enabled: true'' you can't override it with a merge +# and you can't erase existing hooks or all environment variables. +# However, with "override_config: true" in a particular config.*.yaml file, +# 'nfs_mount_enabled: false' can override the existing values, and +# hooks: +# post-start: [] +# or +# web_environment: [] +# or +# additional_hostnames: [] +# can have their intended affect. 'override_config' affects only behavior of the +# config.*.yaml file it exists in. + +# Many DDEV commands can be extended to run tasks before or after the +# DDEV command is executed, for example "post-start", "post-import-db", +# "pre-composer", "post-composer" +# See https://ddev.readthedocs.io/en/stable/users/extend/custom-commands/ for more +# information on the commands that can be extended and the tasks you can define +# for them. Example: +#hooks: diff --git a/.ddev/docker-compose.elasticsearch.yaml b/.ddev/docker-compose.elasticsearch.yaml new file mode 100644 index 0000000..a5051f3 --- /dev/null +++ b/.ddev/docker-compose.elasticsearch.yaml @@ -0,0 +1,29 @@ +#ddev-generated +services: + elasticsearch: + container_name: ddev-${DDEV_SITENAME}-elasticsearch + hostname: ${DDEV_SITENAME}-elasticsearch + image: elasticsearch:8.10.2 + expose: + - "9200" + - "9300" + environment: + - cluster.name=docker-cluster + - discovery.type=single-node + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - VIRTUAL_HOST=$DDEV_HOSTNAME + - HTTP_EXPOSE=9200:9200 + - HTTPS_EXPOSE=9201:9200 + - ELASTIC_PASSWORD=elastic + labels: + com.ddev.site-name: ${DDEV_SITENAME} + com.ddev.approot: $DDEV_APPROOT + volumes: + - elasticsearch:/usr/share/elasticsearch/data + - ".:/mnt/ddev_config" + healthcheck: + test: [ "CMD-SHELL", "curl --fail -k -s -u elastic:elastic https://localhost:9200" ] + +volumes: + elasticsearch: diff --git a/.ddev/php/xdebug.ini b/.ddev/php/xdebug.ini new file mode 100644 index 0000000..b2db12b --- /dev/null +++ b/.ddev/php/xdebug.ini @@ -0,0 +1 @@ +xdebug.mode=coverage \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index a9588ab..b82f3e5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,3 +8,4 @@ /.circleci export-ignore /.codeclimate.yml /.editorconfig +/.fixtures diff --git a/.github/assets/SearchIndex.php.test b/.github/assets/SearchIndex.php.test new file mode 100644 index 0000000..79df15b --- /dev/null +++ b/.github/assets/SearchIndex.php.test @@ -0,0 +1,12 @@ +=8.0", - "ext-json": "*", - "firesphere/searchbackend": "1.x-dev", - "elasticsearch/elasticsearch": "^8.10", - "elastic/enterprise-search": "^8.10" + "name": "firesphere/elastic-search", + "description": "Search a SilverStripe site with Elastic Enterprise or Elastic search", + "type": "silverstripe-vendormodule", + "license": "BSD-3-Clause", + "keywords": [ + "silverstripe", + "search", + "elastic", + "configuration" + ], + "authors": [ + { + "name": "Simon `Firesphere` Erkelens", + "email": "github@casa-laguna.net", + "homepage": "https://firesphere.dev", + "role": "Lead developer" }, - "require-dev": { - "phpunit/phpunit": "^9.5", - "friendsofphp/php-cs-fixer": "^3.35" - }, - "extra": { - "branch-alias": { - "dev-main": "1.0.x-dev" - } - }, - "autoload": { - "psr-4": { - "Firesphere\\ElasticSearch\\": "src" - } - }, - "autoload-dev": { - "psr-4": { - "Firesphere\\ElasticSearch\\Tests\\": "tests" - } - }, - "support": { - "issues": "https://github.com/firesphere/silverstripe-elastic-search/issues", - "source": "https://github.com/firesphere/silverstripe-elastic-search", - "chat": "https://silverstripe-users.slack.com/archives/C9KB3U0P8" - }, - "suggest": { - "ext-pcntl": "Support for multi-core indexing" - }, - "minimum-stability": "dev", - "prefer-stable": true + { + "name": "Marco `Sheepy` Hermo", + "email": "marco@silverstripe.com", + "role": "Lead developer" + } + ], + "require": { + "php": ">=8.0", + "ext-json": "*", + "firesphere/searchbackend": "1.x-dev", + "elasticsearch/elasticsearch": "^8.10", + "elastic/enterprise-search": "^8.10" + }, + "require-dev": { + "silverstripe/recipe-cms": "^4|^5", + "phpunit/phpunit": "^9.5", + "friendsofphp/php-cs-fixer": "^3.35" + }, + "extra": { + "branch-alias": { + "dev-main": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Firesphere\\ElasticSearch\\": "src" + } + }, + "autoload-dev": { + "psr-4": { + "Firesphere\\ElasticSearch\\Tests\\": "tests" + } + }, + "support": { + "issues": "https://github.com/firesphere/silverstripe-elastic-search/issues", + "source": "https://github.com/firesphere/silverstripe-elastic-search", + "chat": "https://silverstripe-users.slack.com/archives/C9KB3U0P8" + }, + "suggest": { + "ext-pcntl": "Support for multi-core indexing" + }, + "config": { + "sort-packages": true, + "process-timeout": 600, + "allow-plugins": { + "composer/installers": true, + "silverstripe/vendor-plugin": true, + "dealerdirect/phpcodesniffer-composer-installer": true, + "silverstripe/recipe-plugin": true, + "php-http/discovery": true + } + }, + "minimum-stability": "dev", + "prefer-stable": true } diff --git a/example.env b/example.env new file mode 100644 index 0000000..b0bd373 --- /dev/null +++ b/example.env @@ -0,0 +1,20 @@ + +# DB credentials +SS_DATABASE_CLASS="MySQLDatabase" +SS_DATABASE_SERVER="db" +SS_DATABASE_USERNAME="root" +SS_DATABASE_PASSWORD="root" +SS_DATABASE_NAME="db" + +# WARNING: in a live environment, change this to "live" instead of dev +SS_ENVIRONMENT_TYPE="dev" +SS_DEFAULT_ADMIN_USERNAME=admin +MAILER_DSN=smtp://localhost:1025 +SS_DEFAULT_ADMIN_PASSWORD=password +SS_DATABASE_PORT=3306 +ELASTIC_ENDPOINT=ddev-elastictest-elasticsearch +ELASTIC_PORT=9200 +ELASTIC_USERNAME=elastic +ELASTIC_PASSWORD=elastic +ELASTIC_PROTOCOL=https +ELASTIC_DISABLE_SSLCHECK=true \ No newline at end of file diff --git a/phpunit.xml.dist b/phpunit.xml.dist new file mode 100644 index 0000000..0421f2d --- /dev/null +++ b/phpunit.xml.dist @@ -0,0 +1,21 @@ + + + + + + . + + + tests/ + thirdparty/ + + + + + tests + + + diff --git a/src/Extensions/DataObjectElasticExtension.php b/src/Extensions/DataObjectElasticExtension.php index e4d96ec..e883056 100644 --- a/src/Extensions/DataObjectElasticExtension.php +++ b/src/Extensions/DataObjectElasticExtension.php @@ -10,7 +10,9 @@ use Psr\Container\NotFoundExceptionInterface; use Psr\Log\LoggerInterface; use SilverStripe\Core\Injector\Injector; +use SilverStripe\ORM\ArrayList; use SilverStripe\ORM\DataExtension; +use SilverStripe\ORM\DataList; use SilverStripe\ORM\DataObject; use SilverStripe\ORM\ValidationException; use SilverStripe\Versioned\Versioned; @@ -36,28 +38,8 @@ public function onAfterDelete() $idx = Injector::inst()->get($index); $config = ElasticIndex::config()->get($idx->getIndexName()); if (in_array($this->owner->ClassName, $config['Classes'])) { - $deleteQuery = [ - 'index' => $index, - 'body' => [ - 'query' => [ - 'match' => [ - 'id' => sprintf('%s-%s', $this->owner->ClassName, $this->owner->ID) - ] - ] - ] - ]; - try { - $service->getClient()->deleteByQuery($deleteQuery); - } catch (Exception $e) { - $dirty = $this->owner->getDirtyClass('DELETE'); - $ids = json_decode($dirty->IDs ?? '[]'); - $ids[] = $this->owner->ID; - $dirty->IDs = json_encode($ids); - $dirty->write(); - /** @var LoggerInterface $logger */ - $logger = Injector::inst()->get(LoggerInterface::class); - $logger->error($e->getMessage(), $e->getTrace()); - } + $deleteQuery = $this->getDeleteQuery($index); + $this->executeQuery($service, $deleteQuery); } } } @@ -83,7 +65,8 @@ public function onAfterWrite() */ private function doIndex() { - $list = DataObject::get($this->owner->ClassName, "ID = " . $this->owner->ID); + $list = ArrayList::create(); + $list->push($this->owner); /** @var ElasticCoreService $service */ $service = Injector::inst()->get(ElasticCoreService::class); foreach ($service->getValidIndexes() as $indexStr) { @@ -95,4 +78,44 @@ private function doIndex() } } } + + /** + * @param mixed $index + * @return array + */ + public function getDeleteQuery(mixed $index): array + { + return [ + 'index' => $index, + 'body' => [ + 'query' => [ + 'match' => [ + 'id' => sprintf('%s-%s', $this->owner->ClassName, $this->owner->ID) + ] + ] + ] + ]; + } + + /** + * @param ElasticCoreService $service + * @param array $deleteQuery + * @return void + * @throws NotFoundExceptionInterface + */ + protected function executeQuery(ElasticCoreService $service, array $deleteQuery): void + { + try { + $service->getClient()->deleteByQuery($deleteQuery); + } catch (Exception $e) { + $dirty = $this->owner->getDirtyClass('DELETE'); + $ids = json_decode($dirty->IDs ?? '[]'); + $ids[] = $this->owner->ID; + $dirty->IDs = json_encode($ids); + $dirty->write(); + /** @var LoggerInterface $logger */ + $logger = Injector::inst()->get(LoggerInterface::class); + $logger->error($e->getMessage(), $e->getTrace()); + } + } } diff --git a/src/Indexes/ElasticIndex.php b/src/Indexes/ElasticIndex.php index 69e58cc..d6b3b3f 100644 --- a/src/Indexes/ElasticIndex.php +++ b/src/Indexes/ElasticIndex.php @@ -150,4 +150,14 @@ public function addClass($class, $options = []): self return $this; } + + public function getClientQuery(): array + { + return $this->clientQuery; + } + + public function setClientQuery(array $clientQuery): void + { + $this->clientQuery = $clientQuery; + } } diff --git a/src/Queries/ElasticQuery.php b/src/Queries/ElasticQuery.php index ce29765..3832981 100644 --- a/src/Queries/ElasticQuery.php +++ b/src/Queries/ElasticQuery.php @@ -152,7 +152,7 @@ public function setOrFilters(array $orFilters): void $this->orFilters = $orFilters; } - public function addOrFilters(string $key, array $orFilters): void + public function addOrFilters(string $key, string $orFilters): void { $this->orFilters[$key] = $orFilters; } diff --git a/src/Services/ElasticCoreService.php b/src/Services/ElasticCoreService.php index c1ae8f3..840714b 100644 --- a/src/Services/ElasticCoreService.php +++ b/src/Services/ElasticCoreService.php @@ -43,17 +43,7 @@ class ElasticCoreService extends BaseService */ public function __construct() { - $config = self::config()->get('config'); - if ($config['endpoint'] === 'ENVIRONMENT') { - $endpoint0 = []; - foreach (self::ENVIRONMENT_VARS as $envVar => $elasticVar) { - $endpoint0[$elasticVar] = Environment::getEnv($envVar); - } - } else { - $endpoint0 = $config['endpoint'][0]; - } - // default to https - $endpoint0['protocol'] = $endpoint0['protocol'] ?? 'https'; + $endpoint0 = $this->getEndpointConfig(); $uri = str_replace(['https://', 'http://'], '', $endpoint0['host']); $uri = sprintf( '%s://%s:%s', @@ -61,13 +51,7 @@ public function __construct() $uri, $endpoint0['port'] ?: 9200 ); - $builder = ClientBuilder::create() - ->setHosts([$uri]); - if ($endpoint0['apiKey']) { - $builder->setApiKey($endpoint0['apiKey']); - } elseif ($endpoint0['username'] && $endpoint0['password']) { - $builder->setBasicAuthentication($endpoint0['username'], $endpoint0['password']); - } + $builder = $this->getBuilder($uri, $endpoint0); $this->client = $builder->build(); parent::__construct(ElasticIndex::class); } @@ -85,38 +69,24 @@ public function setClient(Client $client): void /** * @param ElasticIndex $index * @param SS_List $items + * @return void|array * @throws NotFoundExceptionInterface * @throws ClientResponseException * @throws ServerResponseException */ - public function updateIndex($index, $items) + public function updateIndex($index, $items, $returnDocs = false) { $fields = $index->getFieldsForIndexing(); $factory = $this->getFactory($items); $docs = $factory->buildItems($fields, $index); + $body = ['body' => []]; if (count($docs)) { - $body = [ - 'body' => [ - 'index' => $index->getIndexName() - ] - ]; - if (self::config()->get('pipeline')) { - $body['pipeline'] = self::config()->get('pipeline'); - } - foreach ($docs as $doc) { - $body['body'][] = [ - 'index' => [ - '_index' => $index->getIndexName(), - '_id' => $doc[self::ID_KEY] - ] - ]; - $doc['_extract_binary_content'] = true; - $doc['_reduce_whitespace'] = true; - $doc['_run_ml_inference'] = false; - $body['body'][] = $doc; - } + $body = $this->buildBody($docs, $index); $this->client->bulk($body); } + if ($returnDocs ) { + return $body['body']; + } } /** @@ -135,4 +105,76 @@ protected function getFactory($items): DocumentFactory return $factory; } + + /** + * @return array + */ + private function getEndpointConfig(): array + { + $config = self::config()->get('config'); + if ($config['endpoint'] === 'ENVIRONMENT') { + $endpoint0 = []; + foreach (self::ENVIRONMENT_VARS as $envVar => $elasticVar) { + $endpoint0[$elasticVar] = Environment::getEnv($envVar); + } + } else { + $endpoint0 = $config['endpoint'][0]; + } + // default to https + $endpoint0['protocol'] = $endpoint0['protocol'] ?? 'https'; + + return $endpoint0; + } + + /** + * @param string $uri + * @param array $endpoint0 + * @return ClientBuilder + */ + private function getBuilder(string $uri, array $endpoint0): ClientBuilder + { + $builder = ClientBuilder::create() + ->setHosts([$uri]); + if ($endpoint0['apiKey']) { + $builder->setApiKey($endpoint0['apiKey']); + } elseif ($endpoint0['username'] && $endpoint0['password']) { + $builder->setBasicAuthentication($endpoint0['username'], $endpoint0['password']); + } + // Disable the SSL Certificate check + if (Environment::getEnv('ELASTIC_DISABLE_SSLCHECK')) { + $builder->setSSLVerification(false); + } + + return $builder; + } + + /** + * @param array $docs + * @param ElasticIndex $index + * @return array + */ + public function buildBody(array $docs, ElasticIndex $index): array + { + $body = ['body' => []]; + if (self::config()->get('pipeline')) { + $body['body'] = [ // @todo Check if this is indeed how it works + 'index' => $index->getIndexName(), + 'pipeline' => self::config()->get('pipeline') + ]; + } + foreach ($docs as $doc) { + $body['body'][] = [ + 'index' => [ + '_index' => $index->getIndexName(), + '_id' => $doc[self::ID_KEY] + ] + ]; + $doc['_extract_binary_content'] = true; + $doc['_reduce_whitespace'] = true; + $doc['_run_ml_inference'] = false; + $body['body'][] = $doc; + } + + return $body; + } } diff --git a/tests/E2E/EndToEndTest.php b/tests/E2E/EndToEndTest.php new file mode 100644 index 0000000..1eff41c --- /dev/null +++ b/tests/E2E/EndToEndTest.php @@ -0,0 +1,32 @@ +addTerm('Silverstripe'); + $results = $index->doSearch($query); + $this->assertArrayHasKey('index', $index->getClientQuery()); + $this->assertGreaterThan(0, $results->getTotalItems()); + $this->assertInstanceOf(SearchResult::class, $results); + $this->assertInstanceOf(PaginatedList::class, $results->getPaginatedMatches()); + } +} \ No newline at end of file diff --git a/tests/unit/ElasticCoreServiceTest.php b/tests/unit/ElasticCoreServiceTest.php deleted file mode 100644 index f670426..0000000 --- a/tests/unit/ElasticCoreServiceTest.php +++ /dev/null @@ -1,25 +0,0 @@ -get(ElasticCoreService::class); - $this->assertInstanceOf(Client::class, $service->getClient()); - $this->assertNotEmpty($service->getValidIndexes()); - } - - protected function setUp(): void - { - parent::setUp(); - ClassLoader::inst()->init(1); - } -} diff --git a/tests/unit/Extensions/DataObjectElasticExtensionTest.php b/tests/unit/Extensions/DataObjectElasticExtensionTest.php new file mode 100644 index 0000000..6cd8a34 --- /dev/null +++ b/tests/unit/Extensions/DataObjectElasticExtensionTest.php @@ -0,0 +1,50 @@ + 'Test page']); + $page->write(); + $extension = new DataObjectElasticExtension(); + $extension->setOwner($page); + $extension->onAfterWrite(); + sleep(5); // Wait for Elastic to do its job + $query = new ElasticQuery(); + $query->addTerm('Page'); + /** @var ElasticIndex $index */ + $index = new SearchIndex(); + $result = $index->doSearch($query); + $this->assertEquals(3, $result->getTotalItems()); + $page->publishSingle(); + $extension->onAfterWrite(); + } + + public function testOnAfterDelete() + { + $pages = \Page::get(); + $extension = new DataObjectElasticExtension(); + foreach ($pages as $page) { + $extension->setOwner($page); + $page->delete(); + $extension->onAfterDelete(); + } + $query = new ElasticQuery(); + $query->addTerm('Page'); + // Elastic isn't fast enough to have processed the request + // So... Unsure how to fix this with a proper assertion + // Even despite the wait in PHP, it doesn't help + $this->assertEquals(0, \Page::get()->count()); + } +} \ No newline at end of file diff --git a/tests/unit/SynonymSetTest.php b/tests/unit/Models/SynonymSetTest.php similarity index 58% rename from tests/unit/SynonymSetTest.php rename to tests/unit/Models/SynonymSetTest.php index 94a426b..4c9641f 100644 --- a/tests/unit/SynonymSetTest.php +++ b/tests/unit/Models/SynonymSetTest.php @@ -3,9 +3,7 @@ namespace Firesphere\ElasticSearch\Tests; use Firesphere\ElasticSearch\Models\SynonymSet; -use SilverStripe\Control\HTTPRequest; use SilverStripe\Dev\SapphireTest; -use SilverStripe\ORM\DatabaseAdmin; class SynonymSetTest extends SapphireTest { @@ -13,10 +11,12 @@ class SynonymSetTest extends SapphireTest public function testRequireDefaultRecords() { - $request = new HTTPRequest('GET', 'dev/build', ['quiet' => true, 'flush' => 1]); - DatabaseAdmin::singleton()->setRequest($request)->build(); - + (new SynonymSet())->requireDefaultRecords(); $this->assertEquals(1, SynonymSet::get()->count(), 'There can be only one'); + $key = SynonymSet::get()->first()->Key; $this->assertNotNull(SynonymSet::get()->first()->Key); + (new SynonymSet())->requireDefaultRecords(); + $this->assertEquals(1, SynonymSet::get()->count(), 'There can be only one'); + $this->assertEquals($key, SynonymSet::get()->first()->Key); } } diff --git a/tests/unit/Queries/QueryBuilderTest.php b/tests/unit/Queries/QueryBuilderTest.php new file mode 100644 index 0000000..00197a8 --- /dev/null +++ b/tests/unit/Queries/QueryBuilderTest.php @@ -0,0 +1,75 @@ + 'search-testindex', + 'from' => 0, + 'size' => 10, + 'body' => [ + 'query' => [ + 'bool' => [ + 'must' => [ + [ + 'match' => [ + '_text' => 'TestSearch' + ] + ] + ], + 'filter' => [ + 'bool' => [ + 'must' => [ + [ + 'terms' => [ + 'ViewStatus' => [ + "null", + 'LoggedIn' + ] + ] + ], + [ + 'terms' => [ + 'SiteTree.Title' => [ + 'Home' + ] + ] + ] + ], + 'should' => [ + [ + 'terms' => [ + 'SiteTree.Title' => [ + 'Away' + ] + ] + ] + ] + ] + ] + ] + ] + ] + ]; + + public function testBuildQuery() + { + $query = new ElasticQuery(); + $query->addTerm('TestSearch'); + $query->addFilter('SiteTree.Title', 'Home'); + $query->addOrFilters('SiteTree.Title', 'Away'); + + $this->assertEquals('Home', $query->getFilters()['SiteTree.Title']); + $this->assertEquals('Away', $query->getOrFilters()['SiteTree.Title']); + $this->assertEquals([['text' => 'TestSearch', 'fields' => []]], $query->getTerms()); + + $query = QueryBuilder::buildQuery($query, new SearchIndex()); + + $this->assertEquals(self::$expected_query, $query); + + } +} \ No newline at end of file diff --git a/tests/unit/Services/ElasticCoreServiceTest.php b/tests/unit/Services/ElasticCoreServiceTest.php new file mode 100644 index 0000000..8d311fb --- /dev/null +++ b/tests/unit/Services/ElasticCoreServiceTest.php @@ -0,0 +1,37 @@ +get(ElasticCoreService::class); + $this->assertInstanceOf(Client::class, $service->getClient()); + $this->assertNotEmpty($service->getValidIndexes()); + } + + + public function testUpdateIndex() + { + /** @var ElasticCoreService $service */ + $service = Injector::inst()->get(ElasticCoreService::class); + \Page::create(['Title' => 'Home'])->write(); + $docs = $service->updateIndex(new SearchIndex(), \Page::get(), true); + $count = $service->getClient()->count(['index' => 'search-testindex']); + $this->assertInstanceOf(Elasticsearch::class, $count); + $this->assertGreaterThan(0, count($docs)); + } +} diff --git a/tests/unit/Tasks/ConfigureSynonymsTaskTest.php b/tests/unit/Tasks/ConfigureSynonymsTaskTest.php new file mode 100644 index 0000000..14b6344 --- /dev/null +++ b/tests/unit/Tasks/ConfigureSynonymsTaskTest.php @@ -0,0 +1,36 @@ +requireDefaultRecords(); + $request = new HTTPRequest('GET', 'dev/tasks/ElasticSynonymTask'); + $task = new ElasticConfigureSynonymsTask(); + + $task->run($request); + /** @var ElasticCoreService $service */ + $service = Injector::inst()->get(ElasticCoreService::class); + $set = SynonymSet::get()->first(); + $result = $service->getClient()->synonyms()->getSynonym(['id' => $set->Key])->asArray(); + $this->assertGreaterThan(0, $result); + // Using the defaults, there are at least 21000 US<=>UK synonyms/spelling differences + $this->assertGreaterThan(21000, $result['count']); + $firstBaseSynonym = [ + 'id' => 'base-AWOL', + 'synonyms' => 'AWOL,awol' + ]; + // Expect the first synonym + $this->assertEquals($firstBaseSynonym, $result['synonyms_set'][0]); + } +} \ No newline at end of file