Skip to content

Commit

Permalink
Feature/combine queries for ptosc (#13)
Browse files Browse the repository at this point in the history 
* feat(PtOnlineSchemaChange): Initial support for automatically combining adjacent queries before converting to PTOSC command(s).

* fix(PtOnlineSchemaChange): Limit combining to those known to be supported.

* fix(PtOnlineSchemaChange): Do not attempt to rewrite table-rename queries since they are unsupported by PTOSC.

* fix(PtOnlineSchemaChange): Correct regression when combinable queries differ by table name.
feat(PtOnlineSchemaChange): Allow combining all those ALTER TABLE queries documented as such in Mysql docs.
  • Loading branch information
@paulrrogers
paulrrogers authored Apr 9, 2019
1 parent 066f55e commit 09967d2
Show file tree
Hide file tree
Showing 8 changed files with 273 additions and 72 deletions.
7 changes: 7 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,13 @@ class MyMigration extends Migration
use \OrisIntel\OnlineMigrator\InnodbOnlineDdl
```

Do not combine queries for a migration when using PTOSC:
``` php
class MyMigration extends Migration
{
use \OrisIntel\OnlineMigrator\CombineIncompatible
```

Adding foreign key with index to existing table:
``` php
class MyColumnWithFkMigration extends Migration
Expand Down
9 changes: 9 additions & 0 deletions src/CombineIncompatible.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<?php

namespace OrisIntel\OnlineMigrator;

trait CombineIncompatible
{
/** @var array containing migrate methods incompatible w/combining SQL */
public $combineIncompatible = true;
}
5 changes: 1 addition & 4 deletions src/OnlineMigrator.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,8 @@ protected function getQueries($migration, $method)
// END: Copied from parent.

$strategy = self::getStrategy($migration);
foreach ($queries as &$query) {
$query['query'] = $strategy::getQueryOrCommand($query, $db);
}

return $queries;
return $strategy::getQueriesAndCommands($queries, $db, $migration->combineIncompatible ?? false);
}

/**
Expand Down
26 changes: 22 additions & 4 deletions src/Strategy/InnodbOnlineDdl.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

use Illuminate\Database\Connection;

class InnodbOnlineDdl implements StrategyInterface
final class InnodbOnlineDdl implements StrategyInterface
{
private const INPLACE_INCOMPATIBLE = [
'ALTER\s+TABLE\s+`?[^`\s]+`?\s+(CHANGE|MODIFY)', // CONSIDER: Only when type changes.
Expand All @@ -13,15 +13,33 @@ class InnodbOnlineDdl implements StrategyInterface
// Foreign keys depend upon state of foreign_key_checks.
];

/**
* Get queries and commands, converting "ALTER TABLE " statements to on-line commands/queries.
*
* @param array $queries
* @param array $connection
* @param bool $combineIncompatible
*
* @return array of queries and--where supported--commands
*/
public static function getQueriesAndCommands(array &$queries, Connection $connection, bool $combineIncompatible = false) : array
{
foreach ($queries as &$query) {
$query['query'] = self::getQueryOrCommand($query, $connection);
}

return $queries;
}

/**
* Get query or command, converting "ALTER TABLE " statements to on-line commands/queries.
*
* @param array $query
* @param array $db_config
* @param array $connection
*
* @return string
*/
public static function getQueryOrCommand(array &$query, Connection $connection)
public static function getQueryOrCommand(array &$query, Connection $connection) : string
{
$query_or_command_str = rtrim($query['query'], '; ');

Expand Down Expand Up @@ -106,7 +124,7 @@ private static function isInplaceCompatible(string $query_str, Connection $conne
*
* @return void
*/
public static function runQueryOrCommand(array &$query, Connection $connection)
public static function runQueryOrCommand(array &$query, Connection $connection) : void
{
// Always run unchanged query since this strategy does not need to
// execute commands of other tools.
Expand Down
245 changes: 187 additions & 58 deletions src/Strategy/PtOnlineSchemaChange.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,90 +10,219 @@

use Illuminate\Database\Connection;

class PtOnlineSchemaChange implements StrategyInterface
final class PtOnlineSchemaChange implements StrategyInterface
{
/**
* Get query or command, converting "ALTER TABLE " statements to on-line commands/queries.
* Get queries and commands, converting "ALTER TABLE " statements to on-line commands/queries.
*
* @param array $query
* @param array $db_config
* @param array $queries
* @param array $connection
* @param bool $combineIncompatible
*
* @return string
* @return array of queries and--where supported--commands
*/
public static function getQueryOrCommand(array &$query, Connection $connection)
public static function getQueriesAndCommands(array &$queries, Connection $connection, bool $combineIncompatible = false) : array
{
$query_or_command_str = $query['query'];
// CONSIDER: Executing --dry-run (only during pretend?) first to validate all will work.
/*** @var array like ['table_name' => string, 'changes' => array]. */
$combining = [];

$queries_commands = [];
foreach ($queries as &$query) {
if (
! $combineIncompatible
&& $combinable = self::getCombinableParts($query)
) {
// First adjacent combinable.
if (empty($combining)) {
$combining = $combinable;
continue;
}

// Same table, so combine changes into comma-separated string.
if ($combining['table_name'] === $combinable['table_name']) {
$combining['changes'] =
(!empty($combining['changes']) ? $combining['changes'] . ', ' : '')
. $combinable['changes'];
continue;
}

// Different table, so store previous combinables and reset.
$queries_commands[] = self::getCombinedWithBindings($combining, $connection);
$combining = $combinable;
continue;
}

// Not combinable, so store any previous combinables.
if (! empty($combining)) {
$queries_commands[] = self::getCombinedWithBindings($combining, $connection);
$combining = [];
}

$query['query'] = self::getQueryOrCommand($query, $connection);
$queries_commands[] = $query;
}

// Store residual combinables so they aren't lost.
if (! empty($combining)) {
$queries_commands[] = self::getCombinedWithBindings($combining, $connection);
}

return $queries_commands;
}

/**
* @param array $combining like ['table_name' => string, 'changes' => string]
* @param Connection $connection
*
* @return array like ['query' => string, 'binding' => array, 'time' => float].
*/
private static function getCombinedWithBindings(array $combining, Connection $connection) : array
{
$query_bindings_time = [
'query' => "ALTER TABLE $combining[escape]$combining[table_name]$combining[escape] $combining[changes]",
'bindings' => [],
'time' => 0.0,
];
$query_bindings_time['query'] = self::getQueryOrCommand($query_bindings_time, $connection);

return $query_bindings_time;
}

/**
* @return array like 'table_name' => string, 'changes' => string].
*/
public static function getCombinableParts(array $query) : array
{
// CONSIDER: Combining if all named or all unnamed.
if (! empty($query['bindings'])) {
return [];
}

$parts = self::getOnlineableParts($query['query']);

// CONSIDER: Supporting combinable partition options.
if (preg_match('/\A\s*(ADD|ALTER|DROP|CHANGE)\b/imu', $parts['changes'] ?? '')) {
return $parts;
}

return [];
}

/**
* @param string $query_string
*
* @return array like ['table_name' => ?string, 'changes' => ?string].
*/
private static function getOnlineableParts(string $query_string) : array
{
$table_name = null;
$changes = null;
$escape = null;

$alter_re = '/\A\s*ALTER\s+TABLE\s+[`"]?([^\s`"]+)[`"]?\s*/imu';
$alter_re = '/\A\s*ALTER\s+TABLE\s+([`"]?[^\s`"]+[`"]?)\s*/imu';
$create_re = '/\A\s*CREATE\s+'
. '((UNIQUE|FULLTEXT|SPATIAL)\s+)?'
. 'INDEX\s+[`"]?([^`"\s]+)[`"]?\s+ON\s+[`"]?([^`"\s]+)[`"]?\s+?/imu';
. 'INDEX\s+[`"]?([^`"\s]+)[`"]?\s+ON\s+([`"]?[^`"\s]+[`"]?)\s+?/imu';
$drop_re = '/\A\s*DROP\s+'
. 'INDEX\s+[`"]?([^`"\s]+)[`"]?\s+ON\s+[`"]?([^`"\s]+)[`"]?\s*?/imu';
if (preg_match($alter_re, $query_or_command_str, $alter_parts)) {
. 'INDEX\s+[`"]?([^`"\s]+)[`"]?\s+ON\s+([`"]?[^`"\s]+[`"]?)\s*?/imu';
if (preg_match($alter_re, $query_string, $alter_parts)) {
$table_name = $alter_parts[1];
// Changing query so pretendToRun output will match command.
// CONSIDER: Separate index and overriding pretendToRun instead.
$changes = preg_replace($alter_re, '', $query_or_command_str);
} elseif (preg_match($create_re, $query_or_command_str, $create_parts)) {
$changes = preg_replace($alter_re, '', $query_string);

// Alter-table-rename-to-as is not supported by PTOSC.
if (preg_match('/\A\s*RENAME(\s+(TO|AS))?\s+[^\s]+\s*(;|\z)/imu', $changes)) {
return [];
}
} elseif (preg_match($create_re, $query_string, $create_parts)) {
$index_name = $create_parts[3];
$table_name = $create_parts[4];
$changes = "ADD $create_parts[2] INDEX $index_name "
. preg_replace($create_re, '', $query_or_command_str);
} elseif (preg_match($drop_re, $query_or_command_str, $drop_parts)) {
. preg_replace($create_re, '', $query_string);
} elseif (preg_match($drop_re, $query_string, $drop_parts)) {
$index_name = $drop_parts[1];
$table_name = $drop_parts[2];
$changes = "DROP INDEX $index_name "
. preg_replace($drop_re, '', $query_or_command_str);
. preg_replace($drop_re, '', $query_string);
} else {
// Query not supported by PTOSC.
return [];
}

if ($table_name && $changes) {
// HACK: Workaround PTOSC quirk with escaping and defaults.
$changes = str_replace(
["default '0'", "default '1'"],
['default 0', 'default 1'], $changes);

// Dropping FKs with PTOSC requires prefixing constraint name with
// '_'; adding another if it already starts with '_'.
$changes = preg_replace('/(\bDROP\s+FOREIGN\s+KEY\s+[`"]?)([^`"\s]+)/imu', '\01_\02', $changes);

// Keeping defaults here so overriding one does not discard all, as
// would happen if left to `config/online-migrator.php`.
$ptosc_defaults = [
'--alter-foreign-keys-method=auto',
'--no-check-alter', // ASSUMES: Users accept risks w/RENAME.
// ASSUMES: All are known to be unique.
// CONSIDER: Extracting/re-creating automatic uniqueness checks
// and running them here in PHP beforehand.
'--no-check-unique-key-change',
];
$ptosc_options_str = self::getOptionsForShell(
config('online-migrator.ptosc-options'), $ptosc_defaults);

if (false !== strpos($ptosc_options_str, '--dry-run')) {
throw new \InvalidArgumentException(
'Cannot run PTOSC with --dry-run because it would incompletely change the database. Remove from PTOSC_OPTIONS.');
}
$escape = preg_match('/^([`"])/u', $table_name, $m) ? $m[1] : null;
$table_name = trim($table_name, '`"');

// HACK: Workaround PTOSC quirk with escaping and defaults.
$changes = str_replace(
["default '0'", "default '1'"],
['default 0', 'default 1'], $changes);

// Dropping FKs with PTOSC requires prefixing constraint name with
// '_'; adding another if it already starts with '_'.
$changes = preg_replace('/(\bDROP\s+FOREIGN\s+KEY\s+[`"]?)([^`"\s]+)/imu', '\01_\02', $changes);

return [
'table_name' => $table_name,
'changes' => $changes,
'escape' => $escape,
];
}

$db_config = $connection->getConfig();
$query_or_command_str = 'pt-online-schema-change --alter '
. escapeshellarg($changes)
. ' D=' . escapeshellarg($db_config['database'] . ',t=' . $table_name)
. ' --host ' . escapeshellarg($db_config['host'])
. ' --port ' . escapeshellarg($db_config['port'])
. ' --user ' . escapeshellarg($db_config['username'])
// CONSIDER: Redacting password during pretend
. ' --password ' . escapeshellarg($db_config['password'])
. $ptosc_options_str
. ' --execute'
. ' 2>&1';
/**
* Get query or command, converting "ALTER TABLE " statements to on-line commands/queries.
*
* @param array $query
* @param array $connection
*
* @return string
*/
public static function getQueryOrCommand(array &$query, Connection $connection) : string
{
// CONSIDER: Executing --dry-run (only during pretend?) first to validate all will work.

$onlineable = self::getOnlineableParts($query['query']);

// Leave unchanged when not supported by PTOSC.
if (
empty($onlineable['table_name'])
|| empty($onlineable['changes'])
) {
return $query['query'];
}

return $query_or_command_str;
// Keeping defaults here so overriding one does not discard all, as
// would happen if left to `config/online-migrator.php`.
$ptosc_defaults = [
'--alter-foreign-keys-method=auto',
'--no-check-alter', // ASSUMES: Users accept risks w/RENAME.
// ASSUMES: All are known to be unique.
// CONSIDER: Extracting/re-creating automatic uniqueness checks
// and running them here in PHP beforehand.
'--no-check-unique-key-change',
];
$ptosc_options_str = self::getOptionsForShell(
config('online-migrator.ptosc-options'), $ptosc_defaults);

if (false !== strpos($ptosc_options_str, '--dry-run')) {
throw new \InvalidArgumentException(
'Cannot run PTOSC with --dry-run because it would incompletely change the database. Remove from PTOSC_OPTIONS.');
}

$db_config = $connection->getConfig();
$command = 'pt-online-schema-change --alter '
. escapeshellarg($onlineable['changes'])
. ' D=' . escapeshellarg($db_config['database'] . ',t=' . $onlineable['table_name'])
. ' --host ' . escapeshellarg($db_config['host'])
. ' --port ' . escapeshellarg($db_config['port'])
. ' --user ' . escapeshellarg($db_config['username'])
// CONSIDER: Redacting password during pretend
. ' --password ' . escapeshellarg($db_config['password'])
. $ptosc_options_str
. ' --execute'
. ' 2>&1';

return $command;
}

/**
Expand Down Expand Up @@ -153,7 +282,7 @@ public static function getOptionsForShell(?string $option_csv, array $defaults =
*
* @return void
*/
public static function runQueryOrCommand(array &$query, Connection $connection)
public static function runQueryOrCommand(array &$query, Connection $connection) : void
{
// CONSIDER: Using unmodified migration code when small and not
// currently locked table.
Expand Down
Loading

0 comments on commit 09967d2

Please sign in to comment.