Display skipped table

.github/workflows/clt.yml

@@ -10,7 +10,7 @@ jobs:
   clt:
     runs-on: ubuntu-22.04
     steps:
-      - uses: manticoresoftware/[email protected].11
+      - uses: manticoresoftware/[email protected].15
         with:
           image: 'ghcr.io/manticoresoftware/manticoresearch:test-kit-latest'
           init_code: |

APP_VERSION

@@ -1 +1 @@
-1.1.11
+1.3.1

src/Lib/ManticoreBackup.php

@@ -77,6 +77,9 @@ public static function validateVersions(FileStorage $storage): bool {
 		println(LogLevel::Info, 'Current versions: ' . json_encode($currentVersions));
 		$versionsEqual = true;
 		foreach ($currentVersions as $k => $v) {
+			if ($k === 'buddy') {
+				continue;
+			}
 			if ($k === 'backup') {
 				$currentMajor = (int)strtok($currentVersions[$k], '.');
 				$storedMajor = (int)strtok($v, '.');
@@ -179,8 +182,12 @@ protected static function store(ManticoreClient $client, FileStorage $storage, a
 		foreach ($tables as $index => $type) {
 		  // We will have no directory for distributed indexes and so should not back it up
 			if ($type === 'distributed') {
-				  println(LogLevel::Info, '  ' . colored('SKIP', TextColor::LightYellow));
-				  continue;
+				println(
+					LogLevel::Info,
+					'  ' . $index . ' ('  . $type . ')...'
+				);
+				println(LogLevel::Info, '  ' . colored('SKIP', TextColor::LightYellow));
+				continue;
 			}
 
 			$files = $client->freeze($index);

src/Lib/ManticoreClient.php

@@ -28,6 +28,7 @@ public function __construct(ManticoreConfig $config) {
 
 		$versions = $this->getVersions();
 		$verNum = strtok($versions['manticore'], ' ');
+		metric(labels: $versions);
 
 		if ($verNum === false || version_compare($verNum, Searchd::MIN_VERSION) <= 0) {
 			$verSfx = strtok(' ');
@@ -47,10 +48,13 @@ public function __construct(ManticoreConfig $config) {
 				);
 			}
 		}
+
 		echo PHP_EOL . 'Manticore versions:' . PHP_EOL
 			. '  manticore: ' . $versions['manticore'] . PHP_EOL
 			. '  columnar: ' . $versions['columnar'] . PHP_EOL
 			. '  secondary: ' . $versions['secondary'] . PHP_EOL
+			. '  knn: ' . $versions['knn'] . PHP_EOL
+			. '  buddy: ' . $versions['buddy'] . PHP_EOL
 		;
 
 	  // Validate config path or fail
@@ -81,37 +85,7 @@ public function getConfig(): ManticoreConfig {
    */
 	public static function init(string $configPath): self {
 		$config = new ManticoreConfig($configPath);
-		$client = new ManticoreClient($config);
-
-		$versions = $client->getVersions();
-		metric(labels: $versions);
-		$verNum = strtok($versions['manticore'], ' ');
-
-		if ($verNum === false || version_compare($verNum, Searchd::MIN_VERSION) <= 0) {
-			$verSfx = strtok(' ');
-			if (false === $verSfx) {
-				throw new \RuntimeException('Failed to find the version of the manticore searchd');
-			}
-
-			$isOld = $verNum < Searchd::MIN_VERSION;
-			if (!$isOld) {
-				[, $verDate] = explode('@', $verSfx);
-				$isOld = $verDate < Searchd::MIN_DATE;
-			}
-
-			if ($isOld) {
-				throw new \RuntimeException(
-					'You are running old version of manticore searchd, minimum required: ' . Searchd::MIN_VERSION
-				);
-			}
-		}
-		echo PHP_EOL . 'Manticore versions:' . PHP_EOL
-			. '  manticore: ' . $versions['manticore'] . PHP_EOL
-			. '  columnar: ' . $versions['columnar'] . PHP_EOL
-			. '  secondary: ' . $versions['secondary'] . PHP_EOL
-		;
-
-		return $client;
+		return new ManticoreClient($config);
 	}
 
   /**
@@ -190,8 +164,8 @@ public function getTables(): array {
   /**
    * Get manticore, protocol and columnar versions
    *
-   * @return array{manticore:string,columnar:string,secondary:string}
-   *  Parsed list of versions available with keys of [manticore, columnar, secondary]
+   * @return array{manticore:string,columnar:string,secondary:string,knn:string,buddy:string}
+   *  Parsed list of versions available with keys of [manticore, columnar, secondary, knn, buddy]
    */
 	public function getVersions(): array {
 		$result = $this->execute('SHOW STATUS LIKE \'version\'');
@@ -201,7 +175,7 @@ public function getVersions(): array {
 
 	/**
 	 * Get versions for manticore but using CLI instead of sql
-	 * @return array{backup:string,manticore:string,columnar:string,secondary:string}
+	 * @return array{backup:string,manticore:string,columnar:string,secondary:string,knn:string,buddy:string}
 	 *  Parsed list of versions available with keys of [backup, manticore, columnar, secondary]
 	 */
 	public static function getVersionsFromCli(): array {
@@ -213,20 +187,23 @@ public static function getVersionsFromCli(): array {
 
 	/**
 	 * @param string $version
-	 * @return array{backup:string,manticore:string,columnar:string,secondary:string}
+	 * @return array{backup:string,manticore:string,columnar:string,secondary:string,knn:string,buddy:string}
 	 */
 	protected static function parseVersions(string $version): array {
 		$verPattern = '(\d+\.\d+\.\d+[^\(\)]+)';
-		$matchExpr = "/^(?:Manticore )?{$verPattern}(\(columnar\s{$verPattern}\))?"
-			. "([^\(]*\(secondary\s{$verPattern}\))?$/ius"
+		$matchExpr = "/^(?:Manticore\s)?{$verPattern}(?:[^\(]*\(columnar\s{$verPattern}\))?"
+			. "(?:[^\(]*\(secondary\s{$verPattern}\))?"
+			. "(?:[^\(]*\(knn\s{$verPattern}\))?"
+			. '(?:[^\(]*\(buddy\sv?(\d+\.\d+\.\d+)\))?$/ius'
 		;
 		preg_match($matchExpr, $version, $m);
-
 		return [
 			'backup' => ManticoreBackup::getVersion(),
 			'manticore' => trim($m[1] ?? '0.0.0'),
-			'columnar' => trim($m[3] ?? '0.0.0'),
-			'secondary' => trim($m[5] ?? '0.0.0'),
+			'columnar' => trim($m[2] ?? '0.0.0'),
+			'secondary' => trim($m[3] ?? '0.0.0'),
+			'knn' => trim($m[4] ?? '0.0.0'),
+			'buddy' => trim($m[5] ?? '0.0.0'),
 		];
 	}
 

tests/backup-all-ok.rec

@@ -67,16 +67,15 @@ Manticore versions:
 manticore: %{SEMVER} %{COMMITDATE} dev
 columnar: %{SEMVER} %{COMMITDATE}
 secondary: %{SEMVER} %{COMMITDATE}
-Manticore versions:
-manticore: %{SEMVER} %{COMMITDATE} dev
-columnar: %{SEMVER} %{COMMITDATE}
-secondary: %{SEMVER} %{COMMITDATE}
+knn: %{SEMVER} %{COMMITDATE}
+buddy: %{SEMVER}
 %{DATETIME} [Info] Starting the backup...
 %{DATETIME} [Info] Backing up config files...
 %{DATETIME} [Info]   config files - OK
 %{DATETIME} [Info] Backing up global state files...
 %{DATETIME} [Info]   global state files – OK
 %{DATETIME} [Info] Backing up tables...
+%{DATETIME} [Info]   distributed_index (distributed)...
 %{DATETIME} [Info]   SKIP
 %{DATETIME} [Info]   rt_with_columnar (rt) [2.897K]...
 %{DATETIME} [Info]    OK

tests/backup-fails.rec

@@ -72,10 +72,8 @@ Manticore versions:
 manticore: %{SEMVER} %{COMMITDATE} dev
 columnar: %{SEMVER} %{COMMITDATE}
 secondary: %{SEMVER} %{COMMITDATE}
-Manticore versions:
-manticore: %{SEMVER} %{COMMITDATE} dev
-columnar: %{SEMVER} %{COMMITDATE}
-secondary: %{SEMVER} %{COMMITDATE}
+knn: %{SEMVER} %{COMMITDATE}
+buddy: %{SEMVER}
 %{DATETIME} [Info] Starting the backup...
 %{DATETIME} [Error] Can't find some of the tables: none
 ––– input –––

tests/backup-restore-ok.rec

@@ -23,16 +23,15 @@ Manticore versions:
 manticore: %{SEMVER} %{COMMITDATE} dev
 columnar: %{SEMVER} %{COMMITDATE}
 secondary: %{SEMVER} %{COMMITDATE}
-Manticore versions:
-manticore: %{SEMVER} %{COMMITDATE} dev
-columnar: %{SEMVER} %{COMMITDATE}
-secondary: %{SEMVER} %{COMMITDATE}
+knn: %{SEMVER} %{COMMITDATE}
+buddy: %{SEMVER}
 %{DATETIME} [Info] Starting the backup...
 %{DATETIME} [Info] Backing up config files...
 %{DATETIME} [Info]   config files - OK
 %{DATETIME} [Info] Backing up global state files...
 %{DATETIME} [Info]   global state files – OK
 %{DATETIME} [Info] Backing up tables...
+%{DATETIME} [Info]   distributed_index (distributed)...
 %{DATETIME} [Info]   SKIP
 %{DATETIME} [Info]   rt_with_columnar (rt) [2.897K]...
 %{DATETIME} [Info]    OK
@@ -68,8 +67,8 @@ manticore-backup --backup-dir=/tmp --restore=backup-111
 Copyright (c) %{YEAR}, Manticore Software LTD (https://manticoresearch.com)
 Manticore config file:
 Backup dir: /tmp
-%{DATETIME} [Info] Stored versions: {"backup":"%{SEMVER}","manticore":"%{SEMVER} %{COMMITDATE} dev","columnar":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","secondary":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#"}
-%{DATETIME} [Info] Current versions: {"backup":"%{SEMVER}","manticore":"%{SEMVER} %{COMMITDATE} dev","columnar":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","secondary":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#"}
+%{DATETIME} [Info] Stored versions: {"backup":"%{SEMVER}","manticore":"%{SEMVER} %{COMMITDATE} dev","columnar":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","secondary":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","knn":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","buddy":"%{SEMVER}"}
+%{DATETIME} [Info] Current versions: {"backup":"%{SEMVER}","manticore":"%{SEMVER} %{COMMITDATE} dev","columnar":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","secondary":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","knn":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","buddy":"0.0.0"}
 %{DATETIME} [Info] Starting to restore...
 %{DATETIME} [Error] Cannot initiate the restore process due to searchd daemon is running.
 ––– input –––
@@ -81,8 +80,8 @@ manticore-backup --backup-dir=/tmp --restore=backup-111
 Copyright (c) %{YEAR}, Manticore Software LTD (https://manticoresearch.com)
 Manticore config file:
 Backup dir: /tmp
-%{DATETIME} [Info] Stored versions: {"backup":"%{SEMVER}","manticore":"%{SEMVER} %{COMMITDATE} dev","columnar":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","secondary":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#"}
-%{DATETIME} [Info] Current versions: {"backup":"%{SEMVER}","manticore":"%{SEMVER} %{COMMITDATE} dev","columnar":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","secondary":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#"}
+%{DATETIME} [Info] Stored versions: {"backup":"%{SEMVER}","manticore":"%{SEMVER} %{COMMITDATE} dev","columnar":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","secondary":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","knn":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","buddy":"%{SEMVER}"}
+%{DATETIME} [Info] Current versions: {"backup":"%{SEMVER}","manticore":"%{SEMVER} %{COMMITDATE} dev","columnar":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","secondary":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","knn":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","buddy":"0.0.0"}
 %{DATETIME} [Info] Starting to restore...
 Manticore config
 endpoint =  http://%{IPADDR}:9312
@@ -96,8 +95,8 @@ manticore-backup --backup-dir=/tmp --restore=backup-111
 Copyright (c) %{YEAR}, Manticore Software LTD (https://manticoresearch.com)
 Manticore config file:
 Backup dir: /tmp
-%{DATETIME} [Info] Stored versions: {"backup":"%{SEMVER}","manticore":"%{SEMVER} %{COMMITDATE} dev","columnar":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","secondary":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#"}
-%{DATETIME} [Info] Current versions: {"backup":"%{SEMVER}","manticore":"%{SEMVER} %{COMMITDATE} dev","columnar":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","secondary":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#"}
+%{DATETIME} [Info] Stored versions: {"backup":"%{SEMVER}","manticore":"%{SEMVER} %{COMMITDATE} dev","columnar":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","secondary":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","knn":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","buddy":"%{SEMVER}"}
+%{DATETIME} [Info] Current versions: {"backup":"%{SEMVER}","manticore":"%{SEMVER} %{COMMITDATE} dev","columnar":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","secondary":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","knn":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","buddy":"0.0.0"}
 %{DATETIME} [Info] Starting to restore...
 Manticore config
 endpoint =  http://%{IPADDR}:9312
@@ -111,8 +110,8 @@ manticore-backup --backup-dir=/tmp --restore=backup-111
 Copyright (c) %{YEAR}, Manticore Software LTD (https://manticoresearch.com)
 Manticore config file:
 Backup dir: /tmp
-%{DATETIME} [Info] Stored versions: {"backup":"%{SEMVER}","manticore":"%{SEMVER} %{COMMITDATE} dev","columnar":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","secondary":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#"}
-%{DATETIME} [Info] Current versions: {"backup":"%{SEMVER}","manticore":"%{SEMVER} %{COMMITDATE} dev","columnar":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","secondary":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#"}
+%{DATETIME} [Info] Stored versions: {"backup":"%{SEMVER}","manticore":"%{SEMVER} %{COMMITDATE} dev","columnar":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","secondary":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","knn":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","buddy":"%{SEMVER}"}
+%{DATETIME} [Info] Current versions: {"backup":"%{SEMVER}","manticore":"%{SEMVER} %{COMMITDATE} dev","columnar":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","secondary":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","knn":"%{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#","buddy":"0.0.0"}
 %{DATETIME} [Info] Starting to restore...
 Manticore config
 endpoint =  http://%{IPADDR}:9312
@@ -123,23 +122,23 @@ endpoint =  http://%{IPADDR}:9312
 %{DATETIME} [Info] Restoring data files...
 %{DATETIME} [Info]   tables' files - OK
 %{DATETIME} [Info] The backup '/tmp/backup-111' was successfully restored.
-%{DATETIME} [Info] Elapsed time: #!/[0-9]+/!#.#!/[0-9]+/!#s
+%{DATETIME} [Info] Elapsed time: #!/([0-9]+\.[0-9]+|0)/!#s
 %{DATETIME} [Info] Done
 ––– input –––
 searchd
 ––– output –––
-Manticore %{SEMVER} %{COMMITDATE} dev (columnar %{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#) (secondary %{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#)
+Manticore %{SEMVER} %{COMMITDATE} dev (columnar %{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#) (secondary %{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#) (knn %{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#)
 Copyright (c) 2001-2016, Andrew Aksyonoff
 Copyright (c) 2008-2016, Sphinx Technologies Inc (http://sphinxsearch.com)
 Copyright (c) 2017-%{YEAR}, Manticore Software LTD (https://manticoresearch.com)
 [#!/[0-9]{2}:[0-9]{2}\.[0-9]{3}/!#] [#!/[0-9]+/!#] using config file '/etc/manticoresearch/manticore.conf' (#!/[0-9]+/!# chars)...
-starting daemon version '%{SEMVER} %{COMMITDATE} dev (columnar %{SEMVER} %{COMMITDATE}) (secondary %{SEMVER} %{COMMITDATE})' ...
-listening on all interfaces for mysql, port=9306
-listening on UNIX socket /var/run/mysqld/mysqld.sock
-listening on #!/all interfaces|[0-9\.]+:9312/!# for sphinx and http(s)#!/, port=9312|/!#
-listening on all interfaces for sphinx and http(s), port=9308
+starting daemon version '%{SEMVER} %{COMMITDATE} dev (columnar %{SEMVER} %{COMMITDATE}) (secondary %{SEMVER} %{COMMITDATE}) (knn %{SEMVER} %{COMMITDATE})' ...
+listening on %{IPADDR}:9312 for sphinx and http(s)
+listening on %{IPADDR}:9306 for mysql
+listening on %{IPADDR}:9308 for sphinx and http(s)
 precaching table 'rt_with_columnar'
 precaching table 'rt_without_columnar'
+WARNING: table 'distributed_index': ha_strategy defined, but no ha agents in the table
 precached 2 tables in 0.#!/[0-9]{3}/!# sec
 ––– input –––
 mysql -h0 -P9306 -e 'show tables'

tests/backup-shebang-config-ok.rec

@@ -20,16 +20,15 @@ Manticore versions:
 manticore: %{SEMVER} %{COMMITDATE} dev
 columnar: %{SEMVER} %{COMMITDATE}
 secondary: %{SEMVER} %{COMMITDATE}
-Manticore versions:
-manticore: %{SEMVER} %{COMMITDATE} dev
-columnar: %{SEMVER} %{COMMITDATE}
-secondary: %{SEMVER} %{COMMITDATE}
+knn: %{SEMVER} %{COMMITDATE}
+buddy: %{SEMVER}
 %{DATETIME} [Info] Starting the backup...
 %{DATETIME} [Info] Backing up config files...
 %{DATETIME} [Info]   config files - OK
 %{DATETIME} [Info] Backing up global state files...
 %{DATETIME} [Info]   global state files – OK
 %{DATETIME} [Info] Backing up tables...
+%{DATETIME} [Info]   distributed_index (distributed)...
 %{DATETIME} [Info]   SKIP
 %{DATETIME} [Info]   rt_with_columnar (rt) [2.897K]...
 %{DATETIME} [Info]    OK

tests/backup-single-table-ok.rec

@@ -67,10 +67,8 @@ Manticore versions:
 manticore: %{SEMVER} %{COMMITDATE} dev
 columnar: %{SEMVER} %{COMMITDATE}
 secondary: %{SEMVER} %{COMMITDATE}
-Manticore versions:
-manticore: %{SEMVER} %{COMMITDATE} dev
-columnar: %{SEMVER} %{COMMITDATE}
-secondary: %{SEMVER} %{COMMITDATE}
+knn: %{SEMVER} %{COMMITDATE}
+buddy: %{SEMVER}
 %{DATETIME} [Info] Starting the backup...
 %{DATETIME} [Info] Backing up config files...
 %{DATETIME} [Info]   config files - OK

tests/start-searchd.recb

@@ -1,16 +1,15 @@
 ––– input –––
 searchd
 ––– output –––
-Manticore %{SEMVER} %{COMMITDATE} dev (columnar %{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#) (secondary %{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#)
+Manticore %{SEMVER} %{COMMITDATE} dev (columnar %{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#) (secondary %{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#) (knn %{SEMVER} #!/[a-f0-9]{7}@[0-9]{6}/!#)
 Copyright (c) 2001-2016, Andrew Aksyonoff
 Copyright (c) 2008-2016, Sphinx Technologies Inc (http://sphinxsearch.com)
 Copyright (c) 2017-%{YEAR}, Manticore Software LTD (https://manticoresearch.com)
 [#!/[0-9]{2}:[0-9]{2}\.[0-9]{3}/!#] [#!/[0-9]+/!#] using config file '/etc/manticoresearch/manticore.conf' (#!/[0-9]+/!# chars)...
-starting daemon version '%{SEMVER} %{COMMITDATE} dev (columnar %{SEMVER} %{COMMITDATE}) (secondary %{SEMVER} %{COMMITDATE})' ...
-listening on all interfaces for mysql, port=9306
-listening on UNIX socket /var/run/mysqld/mysqld.sock
-listening on #!/all interfaces|[0-9\.]+:9312/!# for sphinx and http(s)#!/, port=9312|/!#
-listening on all interfaces for sphinx and http(s), port=9308
+starting daemon version '%{SEMVER} %{COMMITDATE} dev (columnar %{SEMVER} %{COMMITDATE}) (secondary %{SEMVER} %{COMMITDATE}) (knn %{SEMVER} %{COMMITDATE})' ...
+listening on %{IPADDR}:9312 for sphinx and http(s)
+listening on %{IPADDR}:9306 for mysql
+listening on %{IPADDR}:9308 for sphinx and http(s)
 ––– input –––
 manticore-backup --version
 ––– output –––