Skip to content

Commit

Permalink
Implementing AWS Opensearch serverless connection
Browse files Browse the repository at this point in the history
  • Loading branch information
@romainruaud
romainruaud committed Oct 25, 2024
1 parent 4ce3819 commit 6640f11
Show file tree
Hide file tree
Showing 9 changed files with 252 additions and 15 deletions.
42 changes: 42 additions & 0 deletions src/module-elasticsuite-core/Api/Client/ClientConfigurationInterface.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,48 @@ public function getMaxParallelHandles();
*/
public function getMaxRetries();

/**
* Check if SSL verification is enabled.
*
* @return bool
*/
public function isVerifyEnabled();

/**
* Check if AWS Sig4 verification is enabled.
*
* @return bool
*/
public function isAwsSig4Enabled();

/**
* Fetch the AWS Service to be used
*
* @return string
*/
public function getAwsService();

/**
* Get the AWS Region
*
* @return string
*/
public function getAwsRegion();

/**
* Get the AWS Sig4 Key
*
* @return string
*/
public function getAwsSig4Key();

/**
* Get the AWS Sig4 Secret
*
* @return string
*/
public function getAwsSig4Secret();

/**
* Client config options.
*
Expand Down
26 changes: 26 additions & 0 deletions src/module-elasticsuite-core/Client/Client.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ public function __construct(
*/
public function info()
{
if ($this->isAoss()) {
return ['version' => ['number' => '2 ?', 'distribution' => 'AWS OpenSearch Service Serverless']];
}

return $this->getEsClient()->info();
}

Expand All @@ -96,6 +100,10 @@ public function cluster()
*/
public function ping()
{
if ($this->isAoss()) {
return true;
}

return $this->getEsClient()->ping();
}

Expand Down Expand Up @@ -128,6 +136,12 @@ public function indexExists($indexName)
*/
public function putIndexSettings($indexName, $indexSettings)
{
if ($this->isAoss()) {
unset($indexSettings['number_of_replicas']);
unset($indexSettings['refresh_interval']);
unset($indexSettings['translog.durability']);
}

$this->getEsClient()->indices()->putSettings(['index' => $indexName, 'body' => $indexSettings]);
}

Expand Down Expand Up @@ -320,4 +334,16 @@ private function getEsClient(): \OpenSearch\Client

return $this->esClient;
}

/**
* Check if using the AWS OpenSerch Service Serverless, because some operations are not permitted or behaving differently :
* - setting number of replicas per index is not allowed
* - client info is empty
*
* @return bool
*/
private function isAoss(): bool
{
return ($this->clientConfiguration->isAwsSig4Enabled() && $this->clientConfiguration->getAwsService() === 'aoss');
}
}
74 changes: 70 additions & 4 deletions src/module-elasticsuite-core/Client/ClientBuilder.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

namespace Smile\ElasticsuiteCore\Client;

use Aws\Credentials\CredentialProvider;
use Aws\Credentials\Credentials;
use OpenSearch\ConnectionPool\Selectors\StickyRoundRobinSelector;

/**
Expand Down Expand Up @@ -48,6 +50,9 @@ class ClientBuilder
'max_parallel_handles' => 100, // As per default Elasticsearch Handler configuration.
'max_retries' => 2,
'verify' => true,
'enable_aws_sig4' => false,
'aws_region' => 'eu-central-1',
'aws_service' => 'es',
];

/**
Expand All @@ -60,6 +65,11 @@ class ClientBuilder
*/
private $namespaceBuilders = [];

/**
* @var callable|null
*/
private $sig4CredentialsProvider = null;

/**
* Constructor.
*
Expand Down Expand Up @@ -119,10 +129,24 @@ public function build(array $options = []): \OpenSearch\Client
$clientBuilder->setTracer($this->logger);
}

$handlerParams = [];
if ($options['max_parallel_handles']) {
$handlerParams = ['max_handles' => (int) $options['max_parallel_handles']];
$handler = \OpenSearch\ClientBuilder::defaultHandler($handlerParams);
$clientBuilder->setHandler($handler);
}
$handler = \OpenSearch\ClientBuilder::defaultHandler($handlerParams);
$clientBuilder->setHandler($handler);

if ($this->isAwsSig4Enabled($options)) {
// The ->setSigV4* methods are only available starting from opensearch/opensearch-php 2.0.1.
try {
$clientBuilder->setSigV4Region($options['aws_region'])
->setSigV4Service($options['aws_service'])
->setSigV4CredentialProvider($this->getSig4CredentialsProvider($options));
} catch (\Exception $exception) {
$this->logger->error($exception->getMessage());
$message = "Cannot configure SigV4 on OpenSearch client. This can be due to an outdated (<2.0.1) version of the opensearch-project/opensearch-php package.";
throw new \Exception($message . " Error was : " . $exception->getMessage());
}
}

$connectionParams = $this->getConnectionParams($options);
Expand Down Expand Up @@ -168,18 +192,25 @@ private function getHosts(array $options): array

foreach ($options['servers'] as $host) {
if (!empty($host)) {
[$hostname, $port] = array_pad(explode(':', trim($host), 2), 2, 9200);
[$hostname, $port] = array_pad(explode(':', trim(preg_replace('/^https?:\/\//', '', $host)), 2), 2, 9200);
preg_match('/^(https?):\/\//', $host, $matches);

$currentHostConfig = [
'host' => $hostname,
'port' => $port,
'scheme' => isset($options['enable_https_mode']) ? 'https' : $options['scheme'] ?? 'http',
'scheme' => isset($options['enable_https_mode']) ? 'https' : $matches[1] ?? 'http',
];

if ($options['enable_http_auth']) {
$currentHostConfig['user'] = $options['http_auth_user'];
$currentHostConfig['pass'] = $options['http_auth_pwd'];
}

if ($this->isAwsSig4Enabled($options) === true) {
$currentHostConfig['scheme'] = 'https'; // AOSS is alway behind HTTPS url.
$currentHostConfig['host'] = $host; // Use raw host string for AOSS.
}

$hosts[] = $currentHostConfig;
}
}
Expand Down Expand Up @@ -210,4 +241,39 @@ private function getConnectionParams(array $options): array
],
] : [];
}

/**
* Check if AWS Signature v4 should be used for signing the requests
*
* @param array $options The Options
*
* @return bool
*/
private function isAwsSig4Enabled($options)
{
return (((bool) $options['enable_aws_sig4']) === true);
}

/**
* Get CredentialProvider Instance to be used with Sig4 authentication.
*
* @param array $options The client options
*
* @return callable
*/
private function getSig4CredentialsProvider($options)
{
if (null === $this->sig4CredentialsProvider) {
$this->sig4CredentialsProvider = CredentialProvider::fromCredentials(
new Credentials(
$options['aws_key'],
$options['aws_secret'],
isset($options['aws_token']) ? $options['aws_token'] : null,
isset($options['aws_expires']) ? $options['aws_expires'] : null
)
);
}

return $this->sig4CredentialsProvider;
}
}
45 changes: 45 additions & 0 deletions src/module-elasticsuite-core/Client/ClientConfiguration.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,46 @@ public function isVerifyEnabled()
return (bool) $this->getElasticsearchClientConfigParam('enable_certificate_validation');
}

/**
* @return bool
*/
public function isAwsSig4Enabled()
{
return (bool) $this->getElasticsearchClientConfigParam('enable_aws_sig4');
}

/**
* @return string
*/
public function getAwsService()
{
return (string) $this->getElasticsearchClientConfigParam('aws_service');
}

/**
* @return string
*/
public function getAwsRegion()
{
return (string) $this->getElasticsearchClientConfigParam('aws_region');
}

/**
* @return string
*/
public function getAwsSig4Key()
{
return (string) $this->getElasticsearchClientConfigParam('aws_key');
}

/**
* @return string
*/
public function getAwsSig4Secret()
{
return (string) $this->getElasticsearchClientConfigParam('aws_secret');
}

/**
* {@inheritDoc}
*/
Expand All @@ -154,6 +194,11 @@ public function getOptions()
'max_parallel_handles' => $this->getMaxParallelHandles(),
'max_retries' => $this->getMaxRetries(),
'verify' => $this->isVerifyEnabled(),
'enable_aws_sig4' => $this->isAwsSig4Enabled(),
'aws_service' => $this->getAwsService(),
'aws_region' => $this->getAwsRegion(),
'aws_key' => $this->getAwsSig4Key(),
'aws_secret' => $this->getAwsSig4Secret(),
];

return $options;
Expand Down
7 changes: 6 additions & 1 deletion src/module-elasticsuite-core/Index/IndexOperation.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,12 @@ public function installIndex(\Smile\ElasticsuiteCore\Api\Index\IndexInterface $i
$indexName = $index->getName();
$indexAlias = $this->indexSettings->getIndexAliasFromIdentifier($indexIdentifier, $store);

$this->client->forceMerge($indexName);
try {
$this->client->forceMerge($indexName);
} catch (\Exception $e) {
$this->logger->error($e->getMessage());
}

$this->client->putIndexSettings($indexName, $this->indexSettings->getInstallIndexSettings($indexIdentifier));

$this->proceedIndexInstall($indexName, $indexAlias);
Expand Down
54 changes: 44 additions & 10 deletions src/module-elasticsuite-core/etc/adminhtml/system.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,53 +30,87 @@

<group id="es_client" translate="label" type="text" sortOrder="10" showInDefault="1" showInWebsite="1" showInStore="1">
<label>Elasticsearch Client</label>
<field id="servers" translate="label comment" type="text" sortOrder="51" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<field id="servers" translate="label comment" type="text" sortOrder="10" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<label>Elasticsearch Servers List</label>
<comment>List of servers in [host]:[port] format separated by a comma (e.g. : "es-node1.fqdn:9200, es-node2.fqdn:9200")</comment>
</field>
<field id="enable_https_mode" translate="label comment" type="select" sortOrder="52" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<field id="enable_https_mode" translate="label comment" type="select" sortOrder="20" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<label>Use HTTPS</label>
<comment>Select yes if you want to connect to your Elasticsearch server over HTTPS.</comment>
<source_model>Magento\Config\Model\Config\Source\Yesno</source_model>
</field>
<field id="enable_certificate_validation" translate="label comment" type="select" sortOrder="52" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<field id="enable_certificate_validation" translate="label comment" type="select" sortOrder="30" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<label>Use SSL certificate Validation</label>
<comment>Select no if you are using self-signed SSL certificate.</comment>
<source_model>Magento\Config\Model\Config\Source\Yesno</source_model>
</field>
<field id="enable_http_auth" translate="label comment" type="select" sortOrder="52" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<field id="enable_http_auth" translate="label comment" type="select" sortOrder="40" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<label>Enable basic HTTP authentication</label>
<comment>Enable this option when your Elasticsearch server use basic HTTP authentication.</comment>
<source_model>Magento\Config\Model\Config\Source\Yesno</source_model>
</field>
<field id="enable_http_auth_encoding" translate="label comment" type="select" sortOrder="52" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<field id="enable_http_auth_encoding" translate="label comment" type="select" sortOrder="41" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<label>Encode HTTP authorization headers</label>
<comment>Enable this option when you want to base64 encode the Authorization headers. (Open Distro requires this)</comment>
<source_model>Magento\Config\Model\Config\Source\Yesno</source_model>
<depends>
<field id="enable_http_auth">1</field>
</depends>
</field>
<field id="http_auth_user" translate="label comment" type="text" sortOrder="53" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<field id="http_auth_user" translate="label comment" type="text" sortOrder="42" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<label>Basic HTTP authentication user</label>
<depends>
<field id="enable_http_auth">1</field>
</depends>
</field>
<field id="http_auth_pwd" translate="label comment" type="text" sortOrder="54" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<field id="http_auth_pwd" translate="label comment" type="text" sortOrder="43" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<label>Basic HTTP authentication password</label>
<depends>
<field id="enable_http_auth">1</field>
</depends>
</field>
<field id="enable_debug_mode" translate="label comment" type="select" sortOrder="55" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<field id="enable_aws_sig4" translate="label comment" type="select" sortOrder="50" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<label>Enable AWS Signature V4</label>
<comment><![CDATA[Enable this option when you are using AWS Opensearch Service (serverless or not) and you want to use <a href="https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html">AWS Signature V4.</a>]]></comment>
<source_model>Magento\Config\Model\Config\Source\Yesno</source_model>
</field>
<field id="aws_service" translate="label comment" type="text" sortOrder="51" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<label>AWS Service</label>
<depends>
<field id="enable_aws_sig4">1</field>
</depends>
<comment><![CDATA[<strong>es</strong> for AWS OpenSearch Service, and <strong>aoss</strong> for AWS OpenSearch Service Serverless]]></comment>
</field>
<field id="aws_region" translate="label comment" type="text" sortOrder="52" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<label>AWS Region</label>
<depends>
<field id="enable_aws_sig4">1</field>
</depends>
<comment>Eg : eu-west-1</comment>
</field>
<field id="aws_key" translate="label comment" type="text" sortOrder="53" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<label>AWS Signature V4 Key</label>
<depends>
<field id="enable_aws_sig4">1</field>
</depends>
</field>
<field id="aws_secret" translate="label comment" type="text" sortOrder="54" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<label>AWS Signature V4 Secret</label>
<depends>
<field id="enable_aws_sig4">1</field>
</depends>
</field>
<field id="enable_debug_mode" translate="label comment" type="select" sortOrder="60" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<label>Enable Debug Mode</label>
<comment>When enabled the module will produce logs through Magento logging system.</comment>
<source_model>Magento\Config\Model\Config\Source\Yesno</source_model>
</field>
<field id="connection_timeout" translate="label comment" type="text" sortOrder="56" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<field id="connection_timeout" translate="label comment" type="text" sortOrder="70" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<label>Server Connection Timeout</label>
<comment>In seconds.</comment>
<frontend_class>validate-number</frontend_class>
</field>
<field id="max_retries" translate="label comment" type="text" sortOrder="56" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<field id="max_retries" translate="label comment" type="text" sortOrder="75" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
<label>Elasticsearch Client Maximum Number of Retries</label>
<comment>Maximum number of times to retry connection when there is a connection failure</comment>
<frontend_class>validate-number</frontend_class>
Expand Down
Loading

0 comments on commit 6640f11

Please sign in to comment.