Refactor config validation

Smile-SA · Jan 28, 2025 · 7743d01 · 7743d01
1 parent 8c50e72
commit 7743d01
Show file tree

Hide file tree

Showing 21 changed files with 514 additions and 152 deletions.
diff --git a/app/config/schema.json b/app/config/schema.json
@@ -28,17 +28,21 @@
         "faker": {
             "$ref": "#/definitions/faker"
         },
-        "variables": {
-            "type": "object",
-            "additionalProperties": {
-                "type": "string"
-            }
-        },
         "tables": {
-            "type": "object",
-            "additionalProperties": {
-                "$ref": "#/definitions/table"
-            }
+            "oneOf": [
+                {
+                    "type": "object",
+                    "additionalProperties": {
+                        "$ref": "#/definitions/table"
+                    }
+                },
+                {
+                    "type": "array",
+                    "items": {
+                        "maxLength": 0
+                    }
+                }
+            ]
         },
         "tables_whitelist": {
             "type": "array",
@@ -53,6 +57,22 @@
                 "type": "string",
                 "minLength": 1
             }
+        },
+        "variables": {
+            "oneOf": [
+                {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
+                {
+                    "type": "array",
+                    "items": {
+                        "maxLength": 0
+                    }
+                }
+            ]
         }
     },
     "additionalProperties": false,

diff --git a/app/config/services.yaml b/app/config/services.yaml
@@ -69,12 +69,18 @@ services:
         arguments:
             - !tagged_iterator config.compiler_processor
 
-    config.compiler.dump_output:
+    config.compiler.processor.default_settings:
+        class: 'Smile\GdprDump\Config\Compiler\Processor\DefaultSettingsProcessor'
+
+    config.compiler.processor.dump_output:
         class: 'Smile\GdprDump\Config\Compiler\Processor\DumpOutputProcessor'
 
     config.compiler.processor.env_var:
         class: 'Smile\GdprDump\Config\Compiler\Processor\EnvVarProcessor'
 
+    config.compiler.processor.query_validation:
+        class: 'Smile\GdprDump\Config\Compiler\Processor\QueryValidationProcessor'
+
     config.compiler.processor.version:
         class: 'Smile\GdprDump\Config\Compiler\Processor\VersionProcessor'
 

diff --git a/app/config/templates/sylius.yaml b/app/config/templates/sylius.yaml
@@ -0,0 +1,77 @@
+---
+tables:
+    sylius_address_log_entries:
+        truncate: true
+
+    sylius_address:
+        converters:
+            first_name:
+                converter: 'randomizeText'
+            last_name:
+                converter: 'randomizeText'
+            phone_number:
+                converter: 'randomizeNumber'
+            street:
+                converter: 'randomizeText'
+            company:
+                converter: 'randomizeText'
+            city:
+                converter: 'randomizeText'
+            postcode:
+                converter: 'randomizeNumber'
+
+    sylius_admin_user:
+        converters:
+            username:
+                converter: 'randomizeText'
+                unique: true
+            username_canonical:
+                converter: 'fromContext'
+                parameters:
+                    key: 'processed_data.username'
+            email:
+                converter: 'randomizeEmail'
+                unique: true
+            email_canonical:
+                converter: 'fromContext'
+                parameters:
+                    key: 'processed_data.email'
+            first_name:
+                converter: 'randomizeText'
+            last_name:
+                converter: 'randomizeText'
+
+    sylius_customer:
+        converters:
+            email:
+                converter: 'randomizeEmail'
+                unique: true
+            email_canonical:
+                converter: 'fromContext'
+                parameters:
+                    key: 'processed_data.email'
+            first_name:
+                converter: 'randomizeText'
+            last_name:
+                converter: 'randomizeText'
+            birthday:
+                converter: 'anonymizeDateTime'
+            phone_number:
+                converter: 'randomizeNumber'
+
+    sylius_shop_user:
+        converters:
+            username:
+                converter: 'randomizeText'
+                unique: true
+            username_canonical:
+                converter: 'fromContext'
+                parameters:
+                    key: 'processed_data.username'
+            email:
+                converter: 'randomizeEmail'
+                unique: true
+            email_canonical:
+                converter: 'fromContext'
+                parameters:
+                    key: 'processed_data.email'
diff --git a/composer.json b/composer.json
@@ -17,7 +17,7 @@
         "doctrine/dbal": "^3.1",
         "druidfi/mysqldump-php": "^1.1",
         "fakerphp/faker": "^1.17",
-        "justinrainbow/json-schema": "^5.2",
+        "justinrainbow/json-schema": "^5.3",
         "symfony/config": "^6.1",
         "symfony/console": "^6.1",
         "symfony/dependency-injection": "^6.1",

diff --git a/src/Config/Compiler/Processor/DefaultSettingsProcessor.php b/src/Config/Compiler/Processor/DefaultSettingsProcessor.php
@@ -0,0 +1,74 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Smile\GdprDump\Config\Compiler\Processor;
+
+use Smile\GdprDump\Config\ConfigInterface;
+
+final class DefaultSettingsProcessor implements ProcessorInterface
+{
+    public function process(ConfigInterface $config): void
+    {
+        $data = $config->toArray();
+        $data += [
+            'database' => [],
+            'dump' => [],
+            'faker' => [],
+            'filter_propagation' => [],
+            'tables_whitelist' => [],
+            'tables_blacklist' => [],
+            'tables' => [],
+            'variables' => [],
+        ];
+
+        foreach ($this->getDefaultSettings() as $key => $value) {
+            $data[$key] += $value;
+        }
+
+        $config->reset($data);
+    }
+
+    /**
+     * Get default settings.
+     */
+    private function getDefaultSettings(): array
+    {
+        return [
+            'dump' => [
+                'output' => 'php://stdout',
+                'add_drop_database' => false,
+                'add_drop_table' => true, // false in MySQLDump-PHP
+                'add_drop_trigger' => true,
+                'add_locks' => true,
+                'complete_insert' => false,
+                'compress' => 'none',
+                'default_character_set' => 'utf8',
+                'disable_keys' => true,
+                'events' => false,
+                'extended_insert' => true,
+                'hex_blob' => false, // true in MySQLDump-PHP
+                'init_commands' => [],
+                'insert_ignore' => false,
+                'lock_tables' => false, // true in MySQLDump-PHP
+                'net_buffer_length' => 1000000,
+                'no_autocommit' => true,
+                'no_create_info' => false,
+                'routines' => false,
+                'single_transaction' => true,
+                'skip_comments' => false,
+                'skip_definer' => false,
+                'skip_dump_date' => false,
+                'skip_triggers' => false,
+                'skip_tz_utc' => false,
+            ],
+            'filter_propagation' => [
+                'enabled' => true,
+                'ignored_foreign_keys' => [],
+            ],
+            'faker' => [
+                'locale' => '',
+            ],
+        ];
+    }
+}
diff --git a/src/Config/Compiler/Processor/DumpOutputProcessor.php b/src/Config/Compiler/Processor/DumpOutputProcessor.php
@@ -4,26 +4,38 @@
 
 namespace Smile\GdprDump\Config\Compiler\Processor;
 
+use Smile\GdprDump\Config\ConfigException;
 use Smile\GdprDump\Config\ConfigInterface;
 
 final class DumpOutputProcessor implements ProcessorInterface
 {
     /**
-     * Process date placeholders in dump output (e.g. "dump-{Y-m-d-H.i.s}.sql").
+     * Process placeholders in the dump output setting.
      */
     public function process(ConfigInterface $config): void
     {
-        $dumpSettings = (array) $config->get('dump', []);
-        if (!array_key_exists('output', $dumpSettings)) {
-            return;
-        }
+        $dumpSettings = (array) $config->get('dump');
+        $dumpSettings['output'] = $this->processDatePlaceholder((string) $dumpSettings['output']);
+        $config->set('dump', $dumpSettings);
+    }
 
-        $dumpSettings['output'] = preg_replace_callback(
+    /**
+     * Replace date placeholders.
+     *
+     * @throws ConfigException
+     */
+    private function processDatePlaceholder(string $input): string
+    {
+        $input = preg_replace_callback(
             '/{([^}]+)}/',
             fn (array $matches) => date($matches[1]),
-            $dumpSettings['output']
+            $input
         );
 
-        $config->set('dump', $dumpSettings);
+        if ($input === null) {
+            throw new ConfigException('Failed to replace placeholders in value "%s".', $input);
+        }
+
+        return $input;
     }
 }
diff --git a/src/Config/Compiler/Processor/QueryValidationProcessor.php b/src/Config/Compiler/Processor/QueryValidationProcessor.php
@@ -0,0 +1,52 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Smile\GdprDump\Config\Compiler\Processor;
+
+use Exception;
+use Smile\GdprDump\Config\ConfigException;
+use Smile\GdprDump\Config\ConfigInterface;
+use Smile\GdprDump\Util\QueryValidator;
+
+final class QueryValidationProcessor implements ProcessorInterface
+{
+    /**
+     * Assert that SQL queries don't include forbidden keywords.
+     */
+    public function process(ConfigInterface $config): void
+    {
+        try {
+            $this->validateVarQueries($config);
+            $this->validateInitCommands($config);
+        } catch (Exception $e) {
+            throw new ConfigException($e->getMessage(), $e);
+        }
+    }
+
+    /**
+     * Validate SQL queries found in the "variables" section of the config.
+     */
+    private function validateVarQueries(ConfigInterface $config): void
+    {
+        $selectQueryValidator = new QueryValidator(['select']);
+        $varQueries = (array) $config->get('variables');
+
+        foreach ($varQueries as $query) {
+            $selectQueryValidator->validate($query);
+        }
+    }
+
+    /**
+     * Validate SQL queries found in the "dump.init_commands" section of the config.
+     */
+    private function validateInitCommands(ConfigInterface $config): void
+    {
+        $selectQueryValidator = new QueryValidator(['set']);
+        $dumpSettings = (array) $config->get('dump');
+
+        foreach ($dumpSettings['init_commands'] as $query) {
+            $selectQueryValidator->validate($query);
+        }
+    }
+}
diff --git a/src/Dumper/Config/ConfigProcessor.php b/src/Dumper/Config/ConfigProcessor.php
@@ -4,9 +4,9 @@
 
 namespace Smile\GdprDump\Dumper\Config;
 
+use RuntimeException;
 use Smile\GdprDump\Config\ConfigInterface;
 use Smile\GdprDump\Database\Metadata\MetadataInterface;
-use Smile\GdprDump\Dumper\Config\Validation\ValidationException;
 
 final class ConfigProcessor
 {
@@ -31,7 +31,7 @@ public function process(ConfigInterface $config): DumperConfig
     }
 
     /**
-     * Remote tables that don't exist and resolve patterns (e.g. "log_*") for included/excluded tables.
+     * Remove tables that don't exist and resolve patterns (e.g. "log_*") for included/excluded tables.
      */
     private function processTableLists(ConfigInterface $config): void
     {
@@ -127,6 +127,8 @@ private function findTablesByName(string $pattern): array
 
     /**
      * Raise an exception if the table data contains a converter that references an undefined column.
+     *
+     * @throws RuntimeException
      */
     private function validateTableColumns(string $tableName, array $tableData): void
     {
@@ -141,7 +143,7 @@ private function validateTableColumns(string $tableName, array $tableData): void
 
             if (!$disabled && !in_array($columnName, $columns)) {
                 $message = 'The table "%s" uses a converter on an undefined column "%s".';
-                throw new ValidationException(sprintf($message, $tableName, $columnName));
+                throw new RuntimeException(sprintf($message, $tableName, $columnName));
             }
         }
     }

diff --git a/src/Dumper/Config/Definition/TableConfig.php b/src/Dumper/Config/Definition/TableConfig.php
@@ -5,7 +5,6 @@
 namespace Smile\GdprDump\Dumper\Config\Definition;
 
 use Smile\GdprDump\Dumper\Config\Definition\Table\SortOrder;
-use Smile\GdprDump\Dumper\Config\Validation\ValidationException;
 use Smile\GdprDump\Dumper\Config\Validation\WhereExprValidator;
 use UnexpectedValueException;
 
@@ -117,12 +116,12 @@ private function prepareConfig(array $tableData): void
     /**
      * Prepare the table filters.
      *
-     * @throws ValidationException
+     * @throws UnexpectedValueException
      */
     private function prepareFilters(array $tableData): void
     {
         if (array_key_exists('filters', $tableData)) {
-            throw new ValidationException(
+            throw new UnexpectedValueException(
                 'The property "filters" is no longer supported. Use "where" instead.'
             );
         }