diff --git a/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerRunServiceCommand.php b/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerRunServiceCommand.php index d757bc47be..ae26d8c9d9 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerRunServiceCommand.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Command/WorkerRunServiceCommand.php @@ -4,6 +4,7 @@ use Alchemy\Phrasea\Command\Command; use Alchemy\Phrasea\WorkerManager\Worker\Resolver\WorkerResolverInterface; +use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; @@ -15,8 +16,8 @@ public function __construct() parent::__construct('worker:run-service'); $this->setDescription('Execute a service') - ->addArgument('type') - ->addArgument('body') + ->addArgument('type', InputArgument::REQUIRED) + ->addArgument('body', InputArgument::OPTIONAL) ->addOption('preserve-payload', 'p', InputOption::VALUE_NONE, 'Preserve temporary payload file'); return $this; @@ -28,20 +29,25 @@ protected function doExecute(InputInterface $input, OutputInterface $output) $workerResolver = $this->container['alchemy_worker.type_based_worker_resolver']; $type = $input->getArgument('type'); - $body = file_get_contents($input->getArgument('body')); + $body = $input->getArgument('body'); - if ($body === false) { - $output->writeln('Unable to read payload file'); - return; - } + $body = []; + if($input->getArgument('body')) { + $body = @file_get_contents($input->getArgument('body')); + + if ($body === false) { + $output->writeln(sprintf('Unable to read payload file %s', $input->getArgument('body'))); - $body = json_decode($body, true); + return; + } - if (json_last_error() !== JSON_ERROR_NONE) { - $output->writeln('Invalid message body'); + $body = json_decode($body, true); + if (json_last_error() !== JSON_ERROR_NONE) { + $output->writeln('Invalid message body'); - return; + return; + } } $worker = $workerResolver->getWorker($type, $body); @@ -49,7 +55,7 @@ protected function doExecute(InputInterface $input, OutputInterface $output) $worker->process($body); if (! $input->getOption('preserve-payload')) { - unlink($input->getArgument('body')); + @unlink($input->getArgument('body')); } } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Controller/AdminConfigurationController.php b/lib/Alchemy/Phrasea/WorkerManager/Controller/AdminConfigurationController.php index 5cd591e9ce..6783ed508d 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Controller/AdminConfigurationController.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Controller/AdminConfigurationController.php @@ -6,6 +6,7 @@ use Alchemy\Phrasea\Controller\Controller; use Alchemy\Phrasea\Model\Entities\WorkerRunningJob; use Alchemy\Phrasea\Model\Repositories\WorkerRunningJobRepository; +use Alchemy\Phrasea\Plugin\Exception\JsonValidationException; use Alchemy\Phrasea\SearchEngine\Elastic\ElasticsearchOptions; use Alchemy\Phrasea\Twig\PhraseanetExtension; use Alchemy\Phrasea\WorkerManager\Event\PopulateIndexEvent; @@ -17,8 +18,9 @@ use Alchemy\Phrasea\WorkerManager\Form\WorkerValidationReminderType; use Alchemy\Phrasea\WorkerManager\Queue\AMQPConnection; use Alchemy\Phrasea\WorkerManager\Queue\MessagePublisher; -use Alchemy\Phrasea\WorkerManager\Worker\RecordsActionsWorker; +use Alchemy\Phrasea\WorkerManager\Worker\RecordsActionsWorker\RecordsActionsWorker; use Doctrine\ORM\OptimisticLockException; +use Exception; use Symfony\Component\EventDispatcher\EventDispatcherInterface; use Symfony\Component\Form\Form; use Symfony\Component\Form\FormInterface; @@ -117,7 +119,7 @@ public function infoAction(PhraseaApplication $app, Request $request) if ($timeFilter != null) { try { $dateTimeFilter = (new \DateTime())->sub(new \DateInterval($timeFilter)); - } catch (\Exception $e) { + } catch (Exception $e) { } } @@ -509,28 +511,32 @@ public function recordsActionsAction(PhraseaApplication $app, Request $request) public function recordsActionsFacilityAction(PhraseaApplication $app, Request $request) { - $ret = ['tasks' => []]; - $job = new RecordsActionsWorker($app); - switch ($request->get('ACT')) { - case 'PLAYTEST': - $sxml = simplexml_load_string($request->get('xml')); - if (isset($sxml->tasks->task)) { - foreach ($sxml->tasks->task as $sxtask) { - $ret['tasks'][] = $job->calcSQL($app, $sxtask, true); + $ret = [ + 'error' => null, + 'tasks' => [] + ]; + try { + $job = new RecordsActionsWorker($app); + switch ($request->get('ACT')) { + case 'PLAYTEST': + case 'CALCTEST': + case 'CALCSQL': + $sxml = simplexml_load_string($request->get('xml')); + if ((string)$sxml['version'] !== '2') { + throw new JsonValidationException(sprintf("bad settings version (%s), should be \"2\"", (string)$sxml['version'])); } - } - break; - case 'CALCTEST': - case 'CALCSQL': - $sxml = simplexml_load_string($request->get('xml')); - if (isset($sxml->tasks->task)) { - foreach ($sxml->tasks->task as $sxtask) { - $ret['tasks'][] = $job->calcSQL($app, $sxtask, false); + if (isset($sxml->tasks->task)) { + foreach ($sxml->tasks->task as $sxtask) { + $ret['tasks'][] = $job->calcSQL($sxtask, $request->get('ACT') === 'PLAYTEST'); + } } - } - break; - default: - throw new NotFoundHttpException('Route not found.'); + break; + default: + throw new NotFoundHttpException('Route not found.'); + } + } + catch (Exception $e) { + $ret['error'] = $e->getMessage(); } return $app->json($ret); @@ -557,73 +563,134 @@ private function getDefaultRecordsActionsSettings() --> - keep offline (sb4 = 1) all docs before their "go online" date and after credate (record column) + + + + + + + + + + - Put online (sb4 = 0) all docs from 'public' collection and between the online date and the date of archiving - - - 5, 6, 7 are "public" collections - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - Warn 10 days before archiving (raise sb5) + + + + + + + + + - - - - - - - - + + + + + + + + + - Move to 'archive' collection - - - - - - - reset status of archived documents - - 666 is the "archive" collection - - + + + + + + + + - Delete the documents that are in the trash collection unmodified from 3 months - - - - - + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + EOF; } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Provider/AlchemyWorkerServiceProvider.php b/lib/Alchemy/Phrasea/WorkerManager/Provider/AlchemyWorkerServiceProvider.php index 6556491ce4..cc1e50dcdd 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Provider/AlchemyWorkerServiceProvider.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Provider/AlchemyWorkerServiceProvider.php @@ -9,6 +9,7 @@ use Alchemy\Phrasea\WorkerManager\Worker\AssetsIngestWorker; use Alchemy\Phrasea\WorkerManager\Worker\CreateRecordWorker; use Alchemy\Phrasea\WorkerManager\Worker\DeleteRecordWorker; +use Alchemy\Phrasea\WorkerManager\Worker\EditRecordWorker; use Alchemy\Phrasea\WorkerManager\Worker\ExportMailWorker; use Alchemy\Phrasea\WorkerManager\Worker\ExposeUploadWorker; use Alchemy\Phrasea\WorkerManager\Worker\Factory\CallableWorkerFactory; @@ -17,8 +18,7 @@ use Alchemy\Phrasea\WorkerManager\Worker\PopulateIndexWorker; use Alchemy\Phrasea\WorkerManager\Worker\ProcessPool; use Alchemy\Phrasea\WorkerManager\Worker\PullAssetsWorker; -use Alchemy\Phrasea\WorkerManager\Worker\EditRecordWorker; -use Alchemy\Phrasea\WorkerManager\Worker\RecordsActionsWorker; +use Alchemy\Phrasea\WorkerManager\Worker\RecordsActionsWorker\RecordsActionsWorker; use Alchemy\Phrasea\WorkerManager\Worker\Resolver\TypeBasedWorkerResolver; use Alchemy\Phrasea\WorkerManager\Worker\ShareBasketWorker; use Alchemy\Phrasea\WorkerManager\Worker\SubdefCreationWorker; @@ -27,7 +27,6 @@ use Alchemy\Phrasea\WorkerManager\Worker\WebhookWorker; use Alchemy\Phrasea\WorkerManager\Worker\WorkerInvoker; use Alchemy\Phrasea\WorkerManager\Worker\WriteMetadatasWorker; -use Monolog\Handler\RotatingFileHandler; use Monolog\Handler\StreamHandler; use Monolog\Logger; use Psr\Log\LoggerAwareInterface; diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/RecordsActionsWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/RecordsActionsWorker.php deleted file mode 100644 index 1bad097566..0000000000 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/RecordsActionsWorker.php +++ /dev/null @@ -1,569 +0,0 @@ -app = $app; - $this->conf = $this->app['conf']; - $this->logger = $this->app['alchemy_worker.logger']; - $this->repoWorker = $app['repo.worker-running-job']; - } - - public function process(array $payload) - { - $xmlSettings = $this->conf->get(['workers', 'records_actions', 'xmlSetting'], null); - - if (empty($xmlSettings)) { - $this->logger->error("Can't find the xml setting!"); - - return 0; - } else { - $em = $this->repoWorker->getEntityManager(); - $em->beginTransaction(); - - try { - $workerRunningJob = new WorkerRunningJob(); - $workerRunningJob - ->setWork(MessagePublisher::RECORDS_ACTIONS_TYPE) - ->setPublished(new \DateTime('now')) - ->setStatus(WorkerRunningJob::RUNNING) - ; - - $em->persist($workerRunningJob); - - $em->flush(); - - $em->commit(); - } catch (\Exception $e) { - $em->rollback(); - } - - $settings = simplexml_load_string($xmlSettings); - $tasks = array(); - foreach($settings->tasks->task as $task) { - $tasks[] = $task; - } - - try { - $data = $this->getData($this->app, $tasks); - foreach ($data as $record) { - $this->processData($this->app, $record); - } - } catch(\Exception $e) { - $this->logger->error('Exception when processing data: ' . $e->getMessage()); - - $workerRunningJob - ->setStatus(WorkerRunningJob::ERROR) - ->setInfo($e->getMessage()) - ->setFinished(new \DateTime('now')) - ; - - $this->repoWorker->reconnect(); - - $em->persist($workerRunningJob); - - $em->flush(); - - return 0; - } - - if ($workerRunningJob != null) { - $workerRunningJob - ->setStatus(WorkerRunningJob::FINISHED) - ->setFinished(new \DateTime('now')) - ; - - $this->repoWorker->reconnect(); - - $em->persist($workerRunningJob); - - $em->flush(); - } - } - - } - - private function getData(Application $app, array $tasks) - { - $ret = []; - foreach ($tasks as $sxtask) { - $task = $this->calcSQL($app, $sxtask); - - if (!$task['active'] || !$task['sql']) { - continue; - } - - $this->logger->info(sprintf("playing task '%s' on base '%s'", $task['name'], $task['basename'] ? $task['basename'] : '')); - - try { - /** @var databox $databox */ - $databox = $app->findDataboxById($task['databoxId']); - } catch (\Exception $e) { - $this->logger->error(sprintf("can't connect databoxId %s", $task['databoxId'])); - continue; - } - - $stmt = $databox->get_connection()->prepare($task['sql']['real']['sql']); - $stmt->execute(); - while (false !== $row = $stmt->fetch(\PDO::FETCH_ASSOC)) { - $tmp = [ - 'databoxId' => $task['databoxId'], - 'record_id' => $row['record_id'], - 'action' => $task['action'] - ]; - - $rec = $databox->get_record($row['record_id']); - switch ($task['action']) { - case 'UPDATE': - // change collection ? - if (($x = (int) ($sxtask->to->coll['id'])) > 0) { - $tmp['coll'] = $x; - } - // change sb ? - if (($x = $sxtask->to->status['mask'])) { - $tmp['sb'] = $x; - } - $ret[] = $tmp; - break; - case 'DELETE': - $tmp['deletechildren'] = false; - if ($sxtask['deletechildren'] && $rec->isStory()) { - $tmp['deletechildren'] = true; - } - $ret[] = $tmp; - break; - case 'TRASH': - $ret[] = $tmp; - break; - } - } - $stmt->closeCursor(); - } - - return $ret; - } - - private function processData(Application $app, $row) - { - $databox = $app->findDataboxById($row['databoxId']); - $rec = $databox->get_record($row['record_id']); - - switch ($row['action']) { - case 'UPDATE': - // change collection ? - if (array_key_exists('coll', $row)) { - $coll = \collection::getByCollectionId($app, $databox, $row['coll']); - $rec->move_to_collection($coll); - $this->logger->info(sprintf("on databoxId %s move recordId %s to coll %s \n", $row['databoxId'], $row['record_id'], $coll->get_coll_id())); - } - - // change sb ? - if (array_key_exists('sb', $row)) { - $status = str_split($rec->getStatus()); - foreach (str_split(strrev($row['sb'])) as $bit => $val) { - if ($val == '0' || $val == '1') { - $status[31 - $bit] = $val; - } - } - $status = implode('', $status); - $rec->setStatus($status); - $this->logger->info(sprintf("on databoxId %s set recordId %s status to %s \n", $row['databoxId'], $row['record_id'], $status)); - } - break; - - case 'DELETE': - if ($row['deletechildren'] && $rec->isStory()) { - /** @var record_adapter $child */ - foreach ($rec->getChildren() as $child) { - $child->delete(); - $this->logger->info(sprintf("on databoxId %s delete (grp child) recordId %s \n", $row['databoxId'], $child->getRecordId())); - } - } - $rec->delete(); - $this->logger->info(sprintf("on databoxId %s delete recordId %s \n", $row['databoxId'], $rec->getRecordId())); - break; - case 'TRASH': - // move to trash collection if exist - $trashCollection = $databox->getTrashCollection(); - if ($trashCollection != null) { - $rec->move_to_collection($trashCollection); - $this->logger->info(sprintf("on databoxId %s move recordId %s to trash.", $row['databoxId'], $row['record_id'])); - // disable permalinks - foreach ($rec->get_subdefs() as $subdef) { - if ( ($pl = $subdef->get_permalink()) ) { - $pl->set_is_activated(false); - } - } - } - - break; - } - - return $this; - } - - public function calcSQL(Application $app, $sxtask, $playTest = false) - { - $databoxId = (int) $sxtask['databoxId']; - - $ret = [ - 'name' => $sxtask['name'] ? (string) $sxtask['name'] : 'sans nom', - 'name_htmlencoded' => \p4string::MakeString(($sxtask['name'] ? $sxtask['name'] : 'sans nom'), 'html'), - 'active' => trim($sxtask['active']) === '1', - 'databoxId' => $databoxId, - 'basename' => '', - 'basename_htmlencoded' => '', - 'action' => strtoupper($sxtask['action']), - 'sql' => null, - 'err' => '', - 'err_htmlencoded' => '', - ]; - - try { - /** @var databox $dbox */ - $dbox = $app->findDataboxById($databoxId); - - $ret['basename'] = $dbox->get_label($app['locale']); - $ret['basename_htmlencoded'] = htmlentities($ret['basename']); - try { - switch ($ret['action']) { - case 'UPDATE': - $ret['sql'] = $this->calcUPDATE($app, $databoxId, $sxtask, $playTest); - break; - case 'DELETE': - $ret['sql'] = $this->calcDELETE($app, $databoxId, $sxtask, $playTest); - $ret['deletechildren'] = (int)($sxtask['deletechildren']); - break; - case 'TRASH': - if ($dbox->getTrashCollection() === null) { - $ret['err'] = "trash collection not found on databoxId = ". $databoxId; - $ret['err_htmlencoded'] = htmlentities($ret['err']); - } else { - // there is no to tag, just from tag - // so it's the same as calcDELETE - $ret['sql'] = $this->calcDELETE($app, $databoxId, $sxtask, $playTest); - } - - break; - default: - $ret['err'] = "bad action '" . $ret['action'] . "'"; - $ret['err_htmlencoded'] = htmlentities($ret['err']); - break; - } - } catch (\Exception $e) { - $ret['err'] = $e->getMessage(); - $ret['err_htmlencoded'] = htmlentities($e->getMessage()); - } - } catch (\Exception $e) { - $ret['err'] = "bad databoxId '" . $databoxId . "'"; - $ret['err_htmlencoded'] = htmlentities($ret['err']); - } - - return $ret; - } - - private function calcUPDATE(Application $app, $databoxId, &$sxtask, $playTest) - { - $tws = array(); // NEGATION of updates, used to build the 'test' sql - - // set coll_id ? - if (($x = (int) ($sxtask->to->coll['id'])) > 0) { - $tws[] = 'coll_id!=' . $x; - } - - // set status ? - $x = trim($sxtask->to->status['mask']); - $x = preg_replace('/[^0-1]/', 'x', $x); - - $mx = str_replace(' ', '0', ltrim(str_replace(array('0', 'x'), array(' ', ' '), $x))); - $ma = str_replace(' ', '0', ltrim(str_replace(array('x', '0'), array(' ', '1'), $x))); - if ($mx && $ma) { - $tws[] = '((status ^ 0b' . $mx . ') & 0b' . $ma . ')!=0'; - } - elseif ($mx) { - $tws[] = '(status ^ 0b' . $mx . ')!=0'; - } - elseif ($ma) { - $tws[] = '(status & 0b' . $ma . ')!=0'; - } - - // compute the 'where' clause - list($tw, $join, $err) = $this->calcWhere($app, $databoxId, $sxtask); - - if (!empty($err)) { - throw(new \Exception($err)); - } - - // ... complete the where to build the TEST - if (count($tws) == 1) { - $tw[] = $tws[0]; - } elseif (count($tws) > 1) { - $tw[] = '(' . implode(') OR (', $tws) . ')'; - } - - // build the TEST sql (select) - $sql_test = 'SELECT record_id FROM record' . $join; - if (count($tw) > 0) { - $sql_test .= ' WHERE ' . ((count($tw) == 1) ? $tw[0] : '(' . implode(') AND (', $tw) . ')'); - } - - // build the real sql (select) - $sql_real = 'SELECT record_id FROM record' . $join; - if (count($tw) > 0) { - $sql_real .= ' WHERE ' . ((count($tw) == 1) ? $tw[0] : '(' . implode(') AND (', $tw) . ')'); - } - - $ret = array( - 'real' => array( - 'sql' => $sql_real, - 'sql_htmlencoded' => htmlentities($sql_real), - ), - 'test' => array( - 'sql' => $sql_test, - 'sql_htmlencoded' => htmlentities($sql_test), - 'result' => null, - 'err' => null - ) - ); - - if ($playTest) { - $ret['test']['result'] = $this->playTest($app, $databoxId, $sql_test); - } - - return $ret; - } - - private function calcDELETE(Application $app, $databoxId, &$sxtask, $playTest) - { - // compute the 'where' clause - list($tw, $join, $err) = $this->calcWhere($app, $databoxId, $sxtask); - - if (!empty($err)) { - throw(new \Exception($err)); - } - - // build the TEST sql (select) - $sql_test = 'SELECT record_id FROM record' . $join; - if (count($tw) > 0) { - $sql_test .= ' WHERE ' . ((count($tw) == 1) ? $tw[0] : '(' . implode(') AND (', $tw) . ')'); - } - - // build the real sql (select) - $sql_real = 'SELECT record_id FROM record' . $join; - if (count($tw) > 0) { - $sql_real .= ' WHERE ' . ((count($tw) == 1) ? $tw[0] : '(' . implode(') AND (', $tw) . ')'); - } - - $ret = [ - 'real' => [ - 'sql' => $sql_real, - 'sql_htmlencoded' => htmlentities($sql_real), - ], - 'test' => [ - 'sql' => $sql_test, - 'sql_htmlencoded' => htmlentities($sql_test), - 'result' => null, - 'err' => null - ] - ]; - - if ($playTest) { - $ret['test']['result'] = $this->playTest($app, $databoxId, $sql_test); - } - - return $ret; - } - - private function playTest(Application $app, $databoxId, $sql) - { - /** @var databox $databox */ - $databox = $app->findDataboxById($databoxId); - $connbas = $databox->get_connection(); - $result = ['rids' => [], 'err' => '', 'n' => null]; - - $result['n'] = $connbas->query('SELECT COUNT(*) AS n FROM (' . $sql . ') AS x')->fetchColumn(); - - $stmt = $connbas->prepare('SELECT record_id FROM (' . $sql . ') AS x LIMIT 10'); - if ($stmt->execute([])) { - while (($row = $stmt->fetch(\PDO::FETCH_ASSOC))) { - $result['rids'][] = $row['record_id']; - } - $stmt->closeCursor(); - } else { - $result['err'] = $connbas->errorInfo(); - } - - return $result; - } - - private function calcWhere(Application $app, $databoxId, &$sxtask) - { - $err = ""; - /** @var databox $databox */ - $databox = $app->findDataboxById($databoxId); - /** @var Connection $connbas */ - $connbas = $databox->get_connection(); - - $struct = $databox->get_meta_structure(); - - $tw = array(); - $join = ''; - - $ijoin = 0; - - // criteria - if (($x = $sxtask->from->type['type']) !== null) { - switch (strtoupper($x)) { - case 'RECORD': - $tw[] = 'parent_record_id!=record_id'; - break; - case 'STORY': - $tw[] = 'parent_record_id=record_id'; - break; - } - } - - // criteria - foreach ($sxtask->from->text as $x) { - $field = $struct->get_element_by_name($x['field']); - if ($field != null) { - $ijoin++; - $comp = trim($x['compare']); - if (in_array($comp, array('<', '>', '<=', '>=', '=', '!='))) { - $s = 'p' . $ijoin . '.meta_struct_id=' . $connbas->quote($field->get_id()) . ' AND p' . $ijoin . '.value' . $comp - . '' . $connbas->quote($x['value']) . ''; - - $tw[] = $s; - $join .= ' INNER JOIN metadatas AS p' . $ijoin . ' USING(record_id)'; - } else { - // bad comparison operator - $err .= sprintf("bad comparison operator (%s)\n", $comp); - } - } else { - // unknown field ? - $err .= sprintf("unknown field (%s)\n", $x['field']); - } - } - - // criteria - foreach ($sxtask->from->date as $x) { - $dir = strtoupper($x['direction']); - $delta = (int)($x['delta']); - switch ($x['field']) { - case '#moddate': - case '#credate': - $s = 'NOW()'; - $dbField = substr($x['field'], 1); - if (in_array($dir, array('BEFORE', 'AFTER'))) { - // prevent malformed dates to act - $tw[] = '!ISNULL(CAST('. $dbField . ' AS DATETIME))'; - $s .= ($dir == 'BEFORE') ? '<' : '>='; - - if ($delta > 0) { - $s .= '(' . $dbField . '+INTERVAL ' . $delta . ' DAY)'; - } elseif ($delta < 0) { - $s .= '(' . $dbField . '-INTERVAL ' . -$delta . ' DAY)'; - } else { - $s .= 'CAST(' . $dbField . ' AS DATETIME)'; - } - } else { - // bad direction - $err .= sprintf("bad direction (%s)\n", $x['direction']); - } - $tw[] = $s; - - break; - default: - $field = $struct->get_element_by_name($x['field']); - if ($field != null) { - $ijoin++; - $s = 'p' . $ijoin . '.meta_struct_id=' . $connbas->quote($field->get_id()) . ' AND NOW()'; - if (in_array($dir, array('BEFORE', 'AFTER'))) { - // prevent malformed dates to act - $tw[] = '!ISNULL(CAST(p' . $ijoin . '.value AS DATETIME))'; - $s .= ($dir == 'BEFORE') ? '<' : '>='; - if ($delta > 0) { - $s .= '(p' . $ijoin . '.value+INTERVAL ' . $delta . ' DAY)'; - } elseif ($delta < 0) { - $s .= '(p' . $ijoin . '.value-INTERVAL ' . -$delta . ' DAY)'; - } else { - $s .= 'CAST(p' . $ijoin . '.value AS DATETIME)'; - } - - $tw[] = $s; - $join .= ' INNER JOIN metadatas AS p' . $ijoin . ' USING(record_id)'; - } else { - // bad direction - $err .= sprintf("bad direction (%s)\n", $x['direction']); - } - } - else { - // unknown field ? - $err .= sprintf("unknown field (%s)\n", $x['field']); - } - - break; - } - } - - // criteria - if (($x = $sxtask->from->coll) ) { - $tcoll = explode(',', $x['id']); - foreach ($tcoll as $i => $c) { - $tcoll[$i] = (int)$c; - } - if ($x['compare'] == '=') { - if (count($tcoll) == 1) { - $tw[] = 'coll_id = ' . $tcoll[0]; - } else { - $tw[] = 'coll_id IN(' . implode(',', $tcoll) . ')'; - } - } elseif ($x['compare'] == '!=') { - if (count($tcoll) == 1) { - $tw[] = 'coll_id != ' . $tcoll[0]; - } else { - $tw[] = 'coll_id NOT IN(' . implode(',', $tcoll) . ')'; - } - } else { - // bad operator - $err .= sprintf("bad comparison operator (%s)\n", $x['compare']); - } - } - - // criteria - $x = trim($sxtask->from->status['mask']); - $x = preg_replace('/[^0-1]/', 'x', $x); - - $mx = str_replace(' ', '0', ltrim(str_replace(array('0', 'x'), array(' ', ' '), $x))); - $ma = str_replace(' ', '0', ltrim(str_replace(array('x', '0'), array(' ', '1'), $x))); - if ($mx && $ma) { - $tw[] = '((status ^ 0b'. $mx . ') & 0b'. $ma . ')=0'; - } elseif ($mx) { - $tw[] = '(status ^ 0b' . $mx . ')=0'; - } elseif ($ma) { - $tw[] = '(status & 0b' . $ma . ")=0"; - } - - return array($tw, $join, $err); - } -} diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/RecordsActionsWorker/GetByIdOrNameHelper.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/RecordsActionsWorker/GetByIdOrNameHelper.php new file mode 100644 index 0000000000..6e383b3072 --- /dev/null +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/RecordsActionsWorker/GetByIdOrNameHelper.php @@ -0,0 +1,99 @@ +app = $app; + + // allow to access databox/collections by id or name + foreach ($app->getDataboxes() as $databox) { + $bid = (string)($databox->get_sbas_id()); + $this->databoxes[$bid] = [ + 'db' => $databox, + 'collections' => [], + 'fields' => [] + ]; + $this->databoxes[$databox->get_dbname()] = &$this->databoxes[$bid]; + + foreach ($databox->get_collections() as $coll) { + $cid = (string)($coll->get_coll_id()); + $this->databoxes[$bid]['collections'][$cid] = $coll; + $this->databoxes[$bid]['collections'][$coll->get_name()] = &$this->databoxes[$bid]['collections'][$cid]; + } + + foreach($databox->get_meta_structure() as $field) { + $fid = $field->get_id(); + $this->databoxes[$bid]['fields'][$fid] = $field; + $this->databoxes[$bid]['fields'][$field->get_name()] = &$this->databoxes[$bid]['fields'][$fid]; + } + } + } + + /** + * @param int|string $dbIdOrName + * @return databox|null + */ + public function getDatabox($dbIdOrName) + { + $dbIdOrName = (string)$dbIdOrName; + if(array_key_exists($dbIdOrName, $this->databoxes)) { + return $this->databoxes[$dbIdOrName]['db']; + } + return null; + } + + /** + * @param databox|int|string $dbIdOrName + * @param int|string $collIdOrName + * @return collection|null + */ + public function getCollection($dbIdOrName, $collIdOrName) + { + if($dbIdOrName instanceof databox) { + $dbIdOrName = $dbIdOrName->get_sbas_id(); + } + $dbIdOrName = (string)$dbIdOrName; + if (array_key_exists($dbIdOrName, $this->databoxes)) { + $collIdOrName = (string)$collIdOrName; + if (array_key_exists($collIdOrName, $this->databoxes[$dbIdOrName]['collections'])) { + return $this->databoxes[$dbIdOrName]['collections'][$collIdOrName]; + } + } + return null; + } + + /** + * @param databox|int|string $dbIdOrName + * @param int|string $fieldIdOrName + * @return databox_field|null + */ + public function getField($dbIdOrName, $fieldIdOrName) + { + if($dbIdOrName instanceof databox) { + $dbIdOrName = $dbIdOrName->get_sbas_id(); + } + $dbIdOrName = (string)$dbIdOrName; + if (array_key_exists($dbIdOrName, $this->databoxes)) { + $fieldIdOrName = (string)$fieldIdOrName; + if (array_key_exists($fieldIdOrName, $this->databoxes[$dbIdOrName]['fields'])) { + return $this->databoxes[$dbIdOrName]['fields'][$fieldIdOrName]; + } + } + return null; + } +} diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/RecordsActionsWorker/RecordsActionsWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/RecordsActionsWorker/RecordsActionsWorker.php new file mode 100644 index 0000000000..5bdbfbba10 --- /dev/null +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/RecordsActionsWorker/RecordsActionsWorker.php @@ -0,0 +1,939 @@ +app = $app; + $this->conf = $this->app['conf']; + $this->logger = $this->app['alchemy_worker.logger']; + $this->repoWorker = $app['repo.worker-running-job']; + $this->getByIdOrNameHelper = new GetByIdOrNameHelper($app); + } + + + public function process(array $payload) + { + $xmlSettings = $this->conf->get(['workers', 'records_actions', 'xmlSetting'], null); + + if (empty($xmlSettings)) { + $this->logger->error("Can't find the xml setting!"); + + return 0; + } + else { + $sxSettings = simplexml_load_string($xmlSettings); + if((string)$sxSettings['version'] !== "2") { + throw new JsonValidationException(sprintf("bad settings version (%s), should be \"2\"", (string)$sxml['version'])); + } + $em = $this->repoWorker->getEntityManager(); + $em->beginTransaction(); + + try { + $workerRunningJob = new WorkerRunningJob(); + $workerRunningJob + ->setWork(MessagePublisher::RECORDS_ACTIONS_TYPE) + ->setPublished(new \DateTime('now')) + ->setStatus(WorkerRunningJob::RUNNING) + ; + + $em->persist($workerRunningJob); + + $em->flush(); + + $em->commit(); + } catch (Exception $e) { + $em->rollback(); + } + + $sxTasks = array(); + foreach($sxSettings->tasks->task as $sxTask) { + $sxTasks[] = $sxTask; + } + + try { + // process will act on db, so we first fetch all... + $data = $this->getData($sxTasks); + // ... then process + foreach ($data as $record) { + $this->processData($record); + } + } catch(Exception $e) { + $this->logger->error('Exception when processing data: ' . $e->getMessage()); + + $workerRunningJob + ->setStatus(WorkerRunningJob::ERROR) + ->setInfo($e->getMessage()) + ->setFinished(new \DateTime('now')) + ; + + $this->repoWorker->reconnect(); + + $em->persist($workerRunningJob); + + $em->flush(); + + return 0; + } + + if ($workerRunningJob != null) { + $workerRunningJob + ->setStatus(WorkerRunningJob::FINISHED) + ->setFinished(new \DateTime('now')) + ; + + $this->repoWorker->reconnect(); + + $em->persist($workerRunningJob); + + $em->flush(); + } + } + } + + private function getData(array $sxTasks) + { + $ret = []; + + /** @var SimpleXMLElement $sxtask */ + foreach ($sxTasks as $sxtask) { + $task = $this->calcSQL($sxtask); + + if (!$task['active'] || !$task['sql']) { + continue; + } + + $this->logger->info(sprintf("playing task '%s' on base '%s'", $task['name'], $task['basename'] ? $task['basename'] : '')); + + $databox = $this->getByIdOrNameHelper->getDatabox($task['databoxId']); + if(!$databox) { + $this->logger->error(sprintf("unknown databox %s", $task['databoxId'])); + continue; + } + + $to_coll = null; + if (($x = trim((string) ($sxtask->then->coll['id']))) !== "") { + $coll = $this->getByIdOrNameHelper->getCollection($databox->get_sbas_id(), $x); + if(!$coll) { + $this->logger->error(sprintf("unknown collection %s", $x)); + continue; + } + $to_coll = $coll; + } + + $stmt = $databox->get_connection()->prepare($task['sql']['real']['sql']); + $stmt->execute(); + while (false !== $row = $stmt->fetch(PDO::FETCH_ASSOC)) { + $tmp = [ + 'databoxId' => $databox->get_sbas_id(), + 'record_id' => $row['record_id'], + 'action' => $task['action'], + 'dry' => $task['dry'], + 'set' => [], // fields to set, with k=name + ]; + + $rec = $databox->get_record($row['record_id']); + switch ($task['action']) { + case 'UPDATE': + // change collection ? + if($to_coll) { + $tmp['coll'] = $to_coll->get_coll_id(); + } + // change sb ? + if (($x = $sxtask->then->status['mask'])) { + $tmp['sb'] = $x; + } + // set field(s) ? + foreach($sxtask->then->set_field as $x) { + $field = trim($x['field']); + $value = (string)$x['value']; + if(substr($value, 0, 1) === '$') { + // ref to selected field + $k = substr($value, 1); + if(array_key_exists($k, $row)) { + $tmp['set'][$field] = $row[$k]; + } + } + else { + // constant + } + } + $ret[] = $tmp; + break; + case 'DELETE': + $tmp['deletechildren'] = false; + if ($sxtask['deletechildren'] && $rec->isStory()) { + $tmp['deletechildren'] = true; + } + $ret[] = $tmp; + break; + } + } + $stmt->closeCursor(); + } + + return $ret; + } + + private function processData($row) + { + $databox = $this->getByIdOrNameHelper->getDatabox($row['databoxId']); + if(!$databox) { + $this->logger->error(sprintf("unknown databox %s", $row['databoxId'])); + return $this; + } + $rec = $databox->get_record($row['record_id']); + + switch ($row['action']) { + case 'UPDATE': + $actions = []; // actions as defined in https://app.swaggerhub.com/apis-docs/alchemy-fr/phraseanet.api.v3/1.0.0-oas3#/record/patchRecord + + // change collection ? + if (array_key_exists('coll', $row)) { + $coll = collection::getByCollectionId($this->app, $databox, $row['coll']); + $actions['base_id'] = $coll->get_coll_id(); +// $this->logger->info(sprintf("on databox(%s), record(%s) : move record to coll %s \n", $row['databoxId'], $row['record_id'], $coll->get_coll_id())); + } + + // change sb ? + if (array_key_exists('sb', $row)) { + $actions['status'] = []; + foreach (str_split(strrev($row['sb'])) as $bit => $val) { + if ($val == '0' || $val == '1') { + $actions['status'][] = ['bit' => $bit, 'state' => $val=='1']; + } + } +// $this->logger->info(sprintf("on databox(%s), record(%s) : set status to \"%s\" \n", $row['databoxId'], $row['record_id'], $status)); + } + + // set some field values ? + foreach ($row['set'] as $field => $value) { + if(!array_key_exists('metadatas', $actions)) { + $actions['metadatas'] = []; + } + $actions['metadatas'][] = ['field_name' => $field, 'value' => $value]; +// $this->logger->info(sprintf("on databox(%s), record(%s) : set field %s to \"%s\" \n", $row['databoxId'], $row['record_id'], $field, $value)); + } + + if(!empty($actions)) { + $js = json_encode($actions); + $this->logger->info(sprintf("on databox(%s), record(%s) :%s js=%s \n", + $row['databoxId'], + $row['record_id'], + $row['dry'] ? " [DRY]" : '', + $js + )); + + if(!$row|'dry') { + $rec->setMetadatasByActions(json_decode($js, false)); // false: setMetadatasByActions expects object, not array ! + } + } + break; + + case 'DELETE': + if ($row['deletechildren'] && $rec->isStory()) { + /** @var record_adapter $child */ + foreach ($rec->getChildren() as $child) { + $child->delete(); + $this->logger->info(sprintf( + "on databox (%s) record (%s) :%s delete child record (%s) \n", + $row['databoxId'], + $rec->getRecordId(), + $row['dry'] ? " [DRY]" : '', + $child->getRecordId() + )); + } + } + $this->logger->info(sprintf( + "on databox (%s) record (%s) :%s delete record \n", + $row['databoxId'], + $rec->getRecordId(), + $row['dry'] ? " [DRY]" : '' + )); + if(!$row['dry']) { + $rec->delete(); + } + break; + } + + return $this; + } + + public function calcSQL(SimpleXMLElement $sxtask, bool $playTest = false): array + { + $ret = [ + 'name' => $sxtask['name'] ? (string) $sxtask['name'] : 'sans nom', + 'name_htmlencoded' => \p4string::MakeString(($sxtask['name'] ? $sxtask['name'] : 'sans nom'), 'html'), + 'active' => trim($sxtask['active']) === '1', + 'dry' => trim($sxtask['dry']) === '1', + 'databoxId' => null, + 'basename' => '', + 'basename_htmlencoded' => '', + 'action' => strtoupper($sxtask['action']), + 'sql' => null, + 'err' => '', + 'err_htmlencoded' => '', + ]; + + try { + $databox = $this->getByIdOrNameHelper->getDatabox($sxtask['databoxId']); + if(!$databox) { + throw new Exception(sprintf("unknown databox \"%s\"", $sxtask['databoxId'])); + } + + $sqlBuilder = new SqlBuilder($databox); + + $ret['databoxId'] = $databox->get_sbas_id(); + + $ret['basename'] = $databox->get_label($this->app['locale']); + $ret['basename_htmlencoded'] = htmlentities($ret['basename']); + switch ($ret['action']) { + case 'UPDATE': + $ret['sql'] = $this->calcUPDATE($sqlBuilder, $databox, $sxtask, $playTest); + break; + case 'DELETE': + if($sxtask->then) { + throw new Exception("\"delete\" action cannot heve \"then\" clause"); + } + $ret['sql'] = $this->calcUPDATE($sqlBuilder, $databox, $sxtask, $playTest); + $ret['deletechildren'] = (int)($sxtask['deletechildren']); + break; + default: + throw new Exception(sprintf("bad action \"%s\"", $ret['action'])); + } + } + catch (Exception $e) { + $ret['err'] = $e->getMessage(); + $ret['err_htmlencoded'] = htmlentities($e->getMessage()); + $this->logger->error($e->getMessage()); + } + + return $ret; + } + + private function calcUPDATE(SqlBuilder $sqlBuilder, databox $databox, &$sxtask, $playTest) + { + $sqlBuilder->addFrom('record'); + $sqlBuilder->addSelect("record.record_id"); + + // build the 'if' clause + // + $this->add_IF_Clauses($sqlBuilder, $databox, $sxtask->if); + + + // build the "then" clauses to be negatively added to "where" + // + $this->add_THEN_Clauses($sqlBuilder, $databox, $sxtask->then); + + $sql_real = $sql_test = $sqlBuilder->getSql(); + $ret = array( + 'real' => array( + 'sql' => $sql_real, + 'sql_htmlencoded' => htmlentities($sql_real), + ), + 'test' => array( + 'sql' => $sql_test, + 'sql_htmlencoded' => htmlentities($sql_test), + 'result' => null, + 'err' => '' + ) + ); + + if ($playTest) { + $ret['test']['result'] = $this->playTest($databox, $sql_test); + } + return $ret; + } + + private function playTest(databox $databox, $sql) + { + $connbas = $databox->get_connection(); + $result = ['rids' => [], 'err' => '', 'n' => null]; + + $result['n'] = $connbas->query('SELECT COUNT(*) AS n FROM (' . $sql . ') AS x')->fetchColumn(); + + $stmt = $connbas->prepare('SELECT record_id FROM (' . $sql . ') AS x LIMIT 10'); + if ($stmt->execute([])) { + while (($row = $stmt->fetch(PDO::FETCH_ASSOC))) { + $result['rids'][] = $row['record_id']; + } + $stmt->closeCursor(); + } else { + $result['err'] = $connbas->errorInfo(); + } + + return $result; + } + + /** + * compute the sql parts implementing the "from" clauses + * + * @param SqlBuilder $sqlBuilder + * @param databox $databox + * @param SimpleXMLElement $sxIf + * @return void + * @throws Exception + */ + private function add_IF_Clauses(SqlBuilder $sqlBuilder, databox $databox, $sxIf) + { + if($sxIf->count() == 0) { + return; + } + + // criteria + foreach ($sxIf->record_type as $x) { + $this->add_IF_RecordTypeClause($sqlBuilder, $databox, trim($x['type'])); + } + + // criteria + foreach ($sxIf->text as $x) { + $this->add_IF_TextClause($sqlBuilder, $databox, trim($x['field']), trim($x['compare']), (string)$x['value']); + } + + // criteria + foreach ($sxIf->number as $x) { + $this->add_IF_NumberClause($sqlBuilder, $databox, trim($x['field']), trim($x['compare']), (double)($x['value'])); + } + + // criteria + foreach ($sxIf->is_set as $x) { + $this->add_IF_FieldSetClause($sqlBuilder, $databox, trim($x['field'])); + } + + // criteria + foreach ($sxIf->is_unset as $x) { + $this->add_IF_FieldUnsetClause($sqlBuilder, $databox, trim($x['field'])); + } + + // criteria + foreach ($sxIf->date as $x) { + $this->add_IF_DateClause($sqlBuilder, $databox, strtoupper(trim($x['direction'])), trim($x['field']), strtoupper(trim($x['delta']))); + } + + // criteria + foreach ($sxIf->coll as $x) { + $this->add_IF_CollClause($sqlBuilder, $databox, trim($x['id']), trim($x['compare'])); + } + + // criteria + foreach($sxIf->status as $x) { + $this->add_IF_StatusClause($sqlBuilder, trim($x['mask'])); + } + } + + /** + * compute the sql parts implementing the "then" clauses + * + * @param SqlBuilder $sqlBuilder + * @param databox $databox + * @param SimpleXMLElement $sxThen + * @return void + * @throws Exception + */ + private function add_THEN_Clauses(SqlBuilder $sqlBuilder, databox $databox, $sxThen) + { + if($sxThen->count() == 0) { + return; + } + + // action + foreach ($sxThen->compute_date as $x) { + $this->add_THEN_ComputeDateClause($sqlBuilder, $databox, strtoupper(trim($x['direction'])), trim($x['field']), strtoupper(trim($x['delta'])), trim($x['computed'])); + } + + // action + foreach ($sxThen->coll as $x) { + $this->add_THEN_CollClause($sqlBuilder, $databox, trim($x['id'])); + } + + // action + foreach($sxThen->status as $x) { + $this->add_THEN_StatusClause($sqlBuilder, trim($x['mask'])); + } + + // action + foreach ($sxThen->set_field as $x) { + $this->add_THEN_SetFieldClause($sqlBuilder, $databox, trim($x['field']), (string)$x['value']); + } + } + + private function add_IF_RecordTypeClause(SqlBuilder $sqlBuilder, databox $databox, string $type) + { + switch (strtoupper($type)) { + case 'RECORD': + $sqlBuilder->addWhere('parent_record_id!=record_id'); + break; + case 'STORY': + $sqlBuilder->addWhere('parent_record_id=record_id'); + break; + default: + throw new Exception(sprintf("bad record_type (%s)\n", $type)); + } + } + + private function add_IF_NumberClause(SqlBuilder $sqlBuilder, databox $databox, string $fieldName, string $operator, float $value) + { + if (!in_array($operator, array('<', '>', '<=', '>=', '=', '!='))) { + throw new Exception(sprintf("bad comparison operator (%s)\n", $operator)); + } + switch ($fieldName) { + case "#filesize": + $ijoin = $sqlBuilder->incIjoin(); + $sqlBuilder->addFrom(sprintf('INNER JOIN subdef AS p%d ON(p%d.record_id=record.record_id)', $ijoin, $ijoin)); + $sqlBuilder->addWhere(sprintf( + 'p%d.name=%s', + $ijoin, + $databox->get_connection()->quote('document') + )); + $sqlBuilder->addWhere(sprintf( + 'p%d.size%s%s', + $ijoin, + $operator, + $value + )); + break; + default: + $field = $this->getByIdOrNameHelper->getField($databox, $fieldName); + if (!$field) { + throw new Exception(sprintf("unknown field (%s)\n", $fieldName)); + } + $ijoin = $sqlBuilder->incIjoin(); + $sqlBuilder->addFrom(sprintf('INNER JOIN metadatas AS p%d ON(p%d.record_id=record.record_id)', $ijoin, $ijoin)); + $sqlBuilder->addWhere(sprintf( + 'p%d.meta_struct_id=%s', + $ijoin, + $databox->get_connection()->quote($field->get_id()) + )); + $sqlBuilder->addWhere(sprintf( + 'CAST(p%d.value AS DECIMAL)%s%s', + $ijoin, + $operator, + $value + )); + break; + } + } + + private function add_IF_FieldSetClause(SqlBuilder $sqlBuilder, databox $databox, string $fieldName) + { + $field = $this->getByIdOrNameHelper->getField($databox, $fieldName); + if (!$field) { + throw new Exception(sprintf("unknown field (%s)\n", $fieldName)); + } + $ijoin = $sqlBuilder->incIjoin(); + $sqlBuilder->addFrom(sprintf( + "INNER JOIN metadatas AS p%d ON(p%d.record_id=record.record_id)", + $ijoin, + $ijoin + )); + $sqlBuilder->addWhere(sprintf( + "p%d.meta_struct_id=%s", + $ijoin, + $databox->get_connection()->quote($field->get_id()) + )); + } + + private function add_IF_FieldUnsetClause(SqlBuilder $sqlBuilder, databox $databox, string $fieldName) + { + $field = $this->getByIdOrNameHelper->getField($databox, $fieldName); + if (!$field) { + throw new Exception(sprintf("unknown field (%s)\n", $fieldName)); + } + $ijoin = $sqlBuilder->incIjoin(); + $sqlBuilder->addFrom(sprintf( + 'LEFT JOIN metadatas AS p%d ON(record.record_id=p%d.record_id AND p%d.meta_struct_id=%s)', + $ijoin, + $ijoin, + $ijoin, + $databox->get_connection()->quote($field->get_id()) + )); + $sqlBuilder->addWhere(sprintf( + "ISNULL(p%d.id)", + $ijoin + )); + } + + private function add_IF_TextClause(SqlBuilder $sqlBuilder, databox $databox, string $fieldName, string $operator, string $value) + { + $field = $this->getByIdOrNameHelper->getField($databox, $fieldName); + if (!$field) { + throw new Exception(sprintf("unknown field (%s)\n", $fieldName)); + } + if (!in_array($operator, array('<', '>', '<=', '>=', '=', '!='))) { + throw new Exception(sprintf("bad comparison operator (%s)\n", $operator)); + } + $ijoin = $sqlBuilder->incIjoin(); + $sqlBuilder->addFrom(sprintf("INNER JOIN metadatas AS p%d ON(p%d.record_id=record.record_id)", $ijoin, $ijoin)); + $sqlBuilder->addWhere(sprintf( + "p%d.meta_struct_id=%s ", + $ijoin, + $databox->get_connection()->quote($field->get_id()) + )); + $sqlBuilder->addWhere(sprintf( + "p%d.value%s%s", + $ijoin, + $operator, + $databox->get_connection()->quote($value) + )); + } + + private function add_IF_DateClause(SqlBuilder $sqlBuilder, databox $databox, string $dir, string $fieldName, string $delta) + { + $unit = "DAY"; + $matches = []; + $computedSql = null; + if($delta === "") { + $delta = 0; + } + else { + if (preg_match('/^([-+]?\d+)(\s+(HOUR|DAY|WEEK|MONTH|YEAR)S?)?$/', $delta, $matches) === 1) { + if (count($matches) === 4) { + $delta = (int)($matches[1]); + $unit = $matches[3]; + } + else if (count($matches) === 2) { + $delta = (int)($matches[1]); + } + else { + throw new Exception(sprintf("bad delta (%s)\n", $delta)); + } + } + else { + throw new Exception(sprintf("bad delta (%s)\n", $delta)); + } + } + + $dirop = ""; + if (in_array($dir, array('BEFORE', 'AFTER'))) { + $dirop .= ($dir == 'BEFORE') ? '<' : '>='; + } + else { + // bad direction + throw new Exception(sprintf("bad direction (%s)\n", $dir)); + } + + switch ($fieldName) { + case '#moddate': + case '#credate': + $dbField = substr($fieldName, 1); + if($delta == 0) { + $computedSql = sprintf("record.%s AS DATETIME", $dbField); + } + else { + $computedSql = sprintf("(record.%s%sINTERVAL %d %s)", $dbField, $delta > 0 ? '+' : '-', abs($delta), $unit); + } + $sqlBuilder->addWhere(sprintf( + "NOW()%s%s", + $dirop, + $computedSql + )); + break; + + default: + $field = $this->getByIdOrNameHelper->getField($databox, $fieldName); + if (!$field) { + throw new Exception(sprintf("unknown field (%s)\n", $fieldName)); + } + + $ijoin = $sqlBuilder->incIjoin(); + + // prevent malformed dates to act + $sqlBuilder->addWhere(sprintf( + "!ISNULL(CAST(p%d.value AS DATETIME))", + $ijoin + )); + + if($delta == 0) { + $computedSql = sprintf("CAST(p%d.value AS DATETIME)", $ijoin); + } + else { + $computedSql = sprintf("(p%d.value%sINTERVAL %d %s)", $ijoin, $delta > 0 ? '+' : '-', abs($delta), $unit); + } + + $sqlBuilder->addFrom(sprintf( + 'INNER JOIN metadatas AS p%d ON(p%d.record_id=record.record_id)', + $ijoin, + $ijoin + )); + $sqlBuilder->addWhere(sprintf( + "p%d.meta_struct_id=%s", + $ijoin, + $databox->get_connection()->quote($field->get_id()) + )); + $sqlBuilder->addWhere(sprintf( + "NOW()%s%s", + $dirop, + $computedSql + )); + + break; + } + } + + private function add_IF_StatusClause(SqlBuilder $sqlBuilder, string $mask) + { + $mask = preg_replace('/[^0-1]/', 'x', $mask); + $mx = str_replace(' ', '0', ltrim(str_replace(['0', 'x'], [' ', ' '], $mask))); + $ma = str_replace(' ', '0', ltrim(str_replace(['x', '0'], [' ', '1'], $mask))); + + if ($mx && $ma) { + $sqlBuilder->addWhere(sprintf("((status ^ 0b%s) & 0b%s)=0", $mx, $ma)); + } + elseif ($mx) { + $sqlBuilder->addWhere(sprintf("(status ^ 0b%s)=0", $mx)); + } + elseif ($ma) { + $sqlBuilder->addWhere(sprintf("(status & 0b%s)=0", $ma)); + } + } + + /** + * add coll clause to the query builder + * + * @param SqlBuilder $sqlBuilder + * @param databox $databox + * @param string $collList + * @param string $operator + * @return void + * @throws Exception + */ + private function add_IF_CollClause(SqlBuilder $sqlBuilder, databox $databox, string $collList, string $operator) + { + if(!in_array($operator, ['=', '!='])) { + // bad operator + throw new Exception(sprintf("bad comparison operator (%s)\n", $operator)); + } + $tcoll = explode(',', $collList); + foreach ($tcoll as $i => $c) { + $coll = $this->getByIdOrNameHelper->getCollection($databox->get_sbas_id(), $c); + if(!$coll) { + throw new Exception(sprintf("unknown collection %s", $c)); + } + $tcoll[$i] = $coll->get_coll_id(); + } + if(count($tcoll) > 0) { + if ($operator == '=') { + if (count($tcoll) == 1) { + $sqlBuilder->addWhere('coll_id=' . $tcoll[0]); + } + else { + $sqlBuilder->addWhere('coll_id IN(' . implode(',', $tcoll) . ')'); + } + } + else { + if (count($tcoll) == 1) { + $sqlBuilder->addWhere('coll_id!=' . $tcoll[0]); + } + else { + $sqlBuilder->addWhere('coll_id NOT IN(' . implode(',', $tcoll) . ')'); + } + } + } + } + + private function checkComputedRefKey(string $s) + { + if($s === '') { + throw new Exception(sprintf("mssing compute reference\n")); + } + $_s = strtolower($s); + foreach(str_split($_s) as $i => $c) { + if(!($c=='_' || ($c >= 'a' && $c <= 'z') || ($i>0 && $c >= '0' && $c <= '9'))) { + throw new Exception(sprintf("bad compute reference (%s)\n", $s)); + } + } + } + + private function add_THEN_ComputeDateClause(SqlBuilder $sqlBuilder, databox $databox, string $dir, string $fieldName, string $delta, string $computedRefKey) + { + $this->checkComputedRefKey($computedRefKey); + + $unit = "DAY"; + $matches = []; + $computedSql = null; + if($delta === "") { + $delta = 0; + } + else { + if (preg_match('/^([-+]?\d+)(\s+(HOUR|DAY|WEEK|MONTH|YEAR)S?)?$/', $delta, $matches) === 1) { + if (count($matches) === 4) { + $delta = (int)($matches[1]); + $unit = $matches[3]; + } + else if (count($matches) === 2) { + $delta = (int)($matches[1]); + } + else { + throw new Exception(sprintf("bad delta (%s)\n", $delta)); + } + } + else { + throw new Exception(sprintf("bad delta (%s)\n", $delta)); + } + } + + $dirop = ""; + if (in_array($dir, array('BEFORE', 'AFTER'))) { + $dirop .= ($dir == 'BEFORE') ? '<' : '>='; + } + else { + // bad direction + throw new Exception(sprintf("bad direction (%s)\n", $dir)); + } + + switch ($fieldName) { + case '#moddate': + case '#credate': + $dbField = substr($fieldName, 1); + if($delta == 0) { + $computedSql = sprintf("record.%s AS DATETIME", $dbField); + } + else { + $computedSql = sprintf("(record.%s%sINTERVAL %d %s)", $dbField, $delta > 0 ? '+' : '-', abs($delta), $unit); + } + break; + + default: + $field = $this->getByIdOrNameHelper->getField($databox, $fieldName); + if (!$field) { + throw new Exception(sprintf("unknown field (%s)\n", $fieldName)); + } + + $ijoin = $sqlBuilder->incIjoin(); + + // prevent malformed dates to act + $sqlBuilder->addWhere(sprintf( + "!ISNULL(CAST(p%d.value AS DATETIME))", + $ijoin + )); + + if($delta == 0) { + $computedSql = sprintf("CAST(p%d.value AS DATETIME)", $ijoin); + } + else { + $computedSql = sprintf("(p%d.value%sINTERVAL %d %s)", $ijoin, $delta > 0 ? '+' : '-', abs($delta), $unit); + } + + $sqlBuilder->addFrom(sprintf( + 'INNER JOIN metadatas AS p%d ON(p%d.record_id=record.record_id)', + $ijoin, + $ijoin + )); + + break; + } + + if($computedRefKey && $computedSql !== null) { + $sqlBuilder->addSelect(sprintf("%s AS %s", + $computedSql, + $databox->get_connection()->quoteIdentifier($computedRefKey) + )); + $sqlBuilder->addReference($computedRefKey, $computedSql); + } + } + + /** + * add THEN.coll clause to the query builder (negated) + * + * @param SqlBuilder $sqlBuilder + * @param databox $databox + * @param string $collId + * @return void + * @throws Exception + */ + private function add_THEN_CollClause(SqlBuilder $sqlBuilder, databox $databox, string $collId) + { + $coll = $this->getByIdOrNameHelper->getCollection($databox->get_sbas_id(), $collId); + if(!$coll) { + throw new Exception(sprintf("unknown collection %s", $collId)); + } + $sqlBuilder->addNegWhere('coll_id=' . $coll->get_coll_id()); + } + + private function add_THEN_StatusClause(SqlBuilder $sqlBuilder, string $mask) + { + $mask = preg_replace('/[^0-1]/', 'x', $mask); + $mx = str_replace(' ', '0', ltrim(str_replace(['0', 'x'], [' ', ' '], $mask))); + $ma = str_replace(' ', '0', ltrim(str_replace(['x', '0'], [' ', '1'], $mask))); + + if ($mx && $ma) { + $sqlBuilder->addNegWhere(sprintf("((status ^ 0b%s) & 0b%s)=0", $mx, $ma)); + } + elseif ($mx) { + $sqlBuilder->addNegWhere(sprintf("(status ^ 0b%s)=0", $mx)); + } + elseif ($ma) { + $sqlBuilder->addNegWhere(sprintf("(status & 0b%s)=0", $ma)); + } + } + + private function add_THEN_SetFieldClause(SqlBuilder $sqlBuilder, databox $databox, string $fieldName, string $value) + { + $field = $this->getByIdOrNameHelper->getField($databox, $fieldName); + if (!$field) { + throw new Exception(sprintf("unknown field (%s)\n", $fieldName)); + } + + if(substr($value, 0, 1) === '$') { + // reference to a previously computed expression (only THEN.compute_date does that) + $k = substr($value, 1); + $this->checkComputedRefKey($k); + + if(!($value = $sqlBuilder->getReference($k))) { + throw new Exception(sprintf("unknown reference (\$%s)\n", $k)); + } + } + else { + // constant + $value = $databox->get_connection()->quote($value); + } + + $ijoin = $sqlBuilder->incIjoin(); + $sqlBuilder->addFrom(sprintf( + "LEFT JOIN metadatas AS p%d ON(p%d.record_id=record.record_id AND p%d.meta_struct_id=%s AND p%d.value=%s)", + $ijoin, + $ijoin, + $ijoin, + $databox->get_connection()->quote($field->get_id()), + $ijoin, + $value + )); + $sqlBuilder->addWhere(sprintf( + "ISNULL(p%d.id)", + $ijoin + )); + } + +} diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/RecordsActionsWorker/SqlBuilder.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/RecordsActionsWorker/SqlBuilder.php new file mode 100644 index 0000000000..75aa5ded20 --- /dev/null +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/RecordsActionsWorker/SqlBuilder.php @@ -0,0 +1,122 @@ +databox = $databox; + } + + public function addReference($key, $value) + { + $this->references[$key] = $value; + } + + /** + * @param $key + * @return string|null + */ + public function getReference($key) + { + return $this->references[$key] ?: null; + } + + public function incIjoin(): int + { + $this->ijoin++; + return $this->ijoin; + } + + public function addSelect(string $s): self + { + $this->selectClauses[] = $s; + + return $this; + } + + public function addWhere(string $clause): self + { + $this->whereClauses[] = $clause; + return $this; + } + + public function addNegWhere(string $clause): self + { + $this->negWhereClauses[] = $clause; + return $this; + } + + public function addFrom(string $table): self + { + $this->fromClauses[] = $table; + + return $this; + } + + public function getWhereSql() + { + $w = $this->whereClauses; + + if(!empty($this->negWhereClauses)) { + if(count($this->negWhereClauses) == 1) { + $neg = $this->negWhereClauses[0]; + } + else { + $neg = "(" . join(") AND (", $this->negWhereClauses) . ")"; + } + $w[] = "NOT(" . $neg . ")"; + } + + if(empty($w)) { + return ""; + } + if(count($w) === 1) { + return $w[0]; + } + return "(" . join(") AND (", $w) . ")"; + } + + public function getSql(): string + { + $sql = ""; + + if(!empty($this->selectClauses)) { + $sql .= $sql ? ' ' : ''; + $sql .= sprintf("SELECT %s", + join(', ', $this->selectClauses) + ); + } + + if(!empty($this->fromClauses)) { + $sql .= $sql ? ' ' : ''; + $sql .= sprintf("FROM %s", + join(' ', $this->fromClauses) + ); + } + + if(!empty($this->whereClauses)) { + $sql .= $sql ? ' ' : ''; + $sql .= sprintf("WHERE %s", $this->getWhereSql()); + } + + return $sql; + } +} diff --git a/lib/conf.d/data_templates/DublinCore.xml b/lib/conf.d/data_templates/DublinCore.xml index 8dafea842c..ed4b57fbae 100644 --- a/lib/conf.d/data_templates/DublinCore.xml +++ b/lib/conf.d/data_templates/DublinCore.xml @@ -209,6 +209,7 @@ + @@ -232,14 +233,24 @@ - - - - + + diff --git a/templates/web/admin/worker-manager/worker_records_actions.html.twig b/templates/web/admin/worker-manager/worker_records_actions.html.twig index 81700e3aca..708bb923df 100644 --- a/templates/web/admin/worker-manager/worker_records_actions.html.twig +++ b/templates/web/admin/worker-manager/worker_records_actions.html.twig @@ -62,6 +62,12 @@ {{ form_row(form.xmlSetting, {'attr': {'style': 'width:99%;height:250px;'}}) }} +{# + +#} +
@@ -166,9 +172,20 @@ , dataType:'json' , type:"POST" , async:true + , error: function(data) { + $("#sqla").html(data.statusText); + } , success:function(data) { + if(data.error) { + $("#sqla").text(data.error); + return; + } t = ""; for (i in data.tasks) { + // o = $("
") + // .append($(" X ")) + // ; + //$("#sqla").append() t += "
 "; if (data.tasks[i].active) { t += " X  "; diff --git a/templates/web/admin/worker-manager/worker_records_actions.md b/templates/web/admin/worker-manager/worker_records_actions.md new file mode 100644 index 0000000000..824611e761 --- /dev/null +++ b/templates/web/admin/worker-manager/worker_records_actions.md @@ -0,0 +1,183 @@ +Act on records matching a list of criteria. + +# Changelog / bc break: + +`from` group is renamed `if` + +`to` group is renamed `then` + +`trash` action is removed ; use `update` with `then` `` + +`type` clause (used for record|story) is renamed `record_type` + +# Doc: + +The worker will play __tasks__, each task must specify the databox to act on +and the action to do on selected records, e.g.: + +```xml + + + + + + + ... + + + ... + + + + + ... + + + +``` + +A task marked as `active="0"` is ignored. + +A task marked as `dry="1"` is executed, but the actions on record are not executed. +This allows to check sql and actions (log) whithout altering data. + +The databox to act on is `databoxId` can be specified by __Id__ or __name__. + +The "Test the rules" button will display the select sql and the number or record selected for action. + +`action` can be one of: + +#### update + +update action can move record to another collection and / or change status-bits. + +#### delete + +delete the records + +## `if` clauses to select records to act on: + +__All__ clauses must match for the record to be selected. + +Some clauses (eg. coll ids) can be a list, allowing to define sort of "or" clauses. + +To set "or" clauses that can't be expressed with the rules syntax, one must define +many tasks. + +### select on type of record. + +```xml + +``` + +```xml + +``` + +### select on collection +`id` is a list of collection id ("base_id" API side) or collection name. + +```xml + + +``` + +```xml + + +``` +_nb:_ Since a record belongs to only one collection, specifiying many `coll` clauses has no sense. + +### select on set / unset field. + +```xml + +``` + +```xml + +``` + +### select on text values. + +```xml + +``` + +```xml + +``` + +_warning:_ comparison is made using __alphabetic__ value. +Using `< > <= >=` compare operators +__is possible__ but may have unexpected result, depending on case, accents, signs etc. + +### select on numeric values. + +```xml + +``` + +possible compare oerators are `= != < > <= >=` + +pseudo-field `#filesize` can be used to test document file size. +```xml + + +``` + +### select on date values + +```xml + + +``` + +`direction`: "after" or "before" + +`delta`: +/- N ("hour" or "day" or "week" or "month" or "year") + +pseudo-fields `#credate` and `#moddate` can be used to test creation date +and last modification date of records. + + +### select on status-bits + +```xml + + +``` + +## `then` actions (for task with "update" action) + +### change collection ; change status-bits ; set a field value +```xml + + + + + + + + +``` + +### set a field to a computed value + +The `compute_date` parameters are the same as `date` clause. The result is then referenced +by the `computed` reference. + +The computed value can then be used as a value for a `set_field` action. + +```xml + + + + +``` + +_nb_: For now this only allow to compute from / to a __datetime__ value. Computing from a "non-date" value +has unpredictable result. + + +