diff --git a/src/controllers/FeedsController.php b/src/controllers/FeedsController.php index 461fd22e..83733e09 100644 --- a/src/controllers/FeedsController.php +++ b/src/controllers/FeedsController.php @@ -206,7 +206,10 @@ public function actionDebug() ob_start(); - FeedMe::$plugin->process->debugFeed($feed, $limit, $offset); + // Keep track of processed elements here - particularly for paginated feeds + $processedElementIds = []; + + FeedMe::$plugin->process->debugFeed($feed, $limit, $offset, $processedElementIds); return ob_get_clean(); } @@ -226,6 +229,9 @@ private function _runImportTask($feed) $limit = $request->getParam('limit'); $offset = $request->getParam('offset'); + // Keep track of processed elements here - particularly for paginated feeds + $processedElementIds = []; + // Are we running from the CP? if ($request->getIsCpRequest()) { // if not using the direct param for this request, do UI stuff @@ -234,6 +240,7 @@ private function _runImportTask($feed) // Create the import task Craft::$app->getQueue()->delay(0)->push(new FeedImport([ 'feed' => $feed, + 'processedElementIds' => $processedElementIds, ])); } @@ -252,6 +259,7 @@ private function _runImportTask($feed) 'feed' => $feed, 'limit' => $limit, 'offset' => $offset, + 'processedElementIds' => $processedElementIds, ])); } diff --git a/src/queue/jobs/FeedImport.php b/src/queue/jobs/FeedImport.php index d520d479..bfac8367 100644 --- a/src/queue/jobs/FeedImport.php +++ b/src/queue/jobs/FeedImport.php @@ -14,6 +14,7 @@ class FeedImport extends BaseJob public $feed; public $limit; public $offset; + public $processedElementIds; // Public Methods @@ -46,7 +47,7 @@ public function execute($queue) foreach ($feedData as $key => $data) { try { - $element = FeedMe::$plugin->process->processFeed($key, $feedSettings); + $element = FeedMe::$plugin->process->processFeed($key, $feedSettings, $this->processedElementIds); } catch (\Throwable $e) { // We want to catch any issues in each iteration of the loop (and log them), but this allows the // rest of the feed to continue processing. @@ -62,10 +63,12 @@ public function execute($queue) 'feed' => $this->feed, 'limit' => $this->limit, 'offset' => $this->offset, + 'processedElementIds' => $this->processedElementIds, ])); + } else { + // Only perform the afterProcessFeed function after any/all pagination is done + FeedMe::$plugin->process->afterProcessFeed($feedSettings, $this->feed, $this->processedElementIds); } - - FeedMe::$plugin->process->afterProcessFeed($feedSettings, $this->feed); } catch (\Throwable $e) { // Even though we catch errors on each step of the loop, make sure to catch errors that can be anywhere // else in this function, just to be super-safe and not cause the queue job to die. diff --git a/src/services/Process.php b/src/services/Process.php index 672cde5b..7df103a1 100644 --- a/src/services/Process.php +++ b/src/services/Process.php @@ -33,8 +33,6 @@ class Process extends Component // Properties // ========================================================================= - private $_processedElements = []; - private $_processedElementIds = []; private $_time_start = null; private $_service = null; @@ -79,10 +77,6 @@ public function beforeProcessFeed($feed, $feedData) App::maxPowerCaptain(); - // Reset properties to allow an instance of this service to be reused - $this->_processedElements = []; - $this->_processedElementIds = []; - // Add some additional information to our FeedModel - for ease of use in processing // $return['fields'] = []; $return['existingElements'] = []; @@ -149,7 +143,7 @@ public function beforeProcessFeed($feed, $feedData) return $return; } - public function processFeed($step, $feed) + public function processFeed($step, $feed, &$processedElementIds) { $existingElement = false; $uniqueMatches = []; @@ -288,7 +282,7 @@ public function processFeed($step, $feed) if (DuplicateHelper::isDisable($feed, true) || DuplicateHelper::isDelete($feed, true)) { // If there's an existing element, we want to keep it, otherwise remove it if ($existingElement) { - $this->_processedElementIds[] = $existingElement->id; + $processedElementIds[] = $existingElement->id; } return; @@ -406,7 +400,7 @@ public function processFeed($step, $feed) FeedMe::debug($info); FeedMe::debug($contentData); - $this->_processedElementIds[] = $element->id; + $processedElementIds[] = $element->id; return; } @@ -437,7 +431,7 @@ public function processFeed($step, $feed) } // Store our successfully processed element for feedback in logs, but also in case we're deleting - $this->_processedElementIds[] = $element->id; + $processedElementIds[] = $element->id; FeedMe::info('Finished processing of node `#{i}`.', ['i' => ($step + 1)]); @@ -458,14 +452,14 @@ public function processFeed($step, $feed) } } - public function afterProcessFeed($settings, $feed) + public function afterProcessFeed($settings, $feed, $processedElementIds) { if (DuplicateHelper::isDelete($feed) && DuplicateHelper::isDisable($feed)) { FeedMe::info("You can't have Delete and Disabled enabled at the same time as an Import Strategy."); return; } - $elementsToDeleteDisable = array_diff($settings['existingElements'], $this->_processedElementIds); + $elementsToDeleteDisable = array_diff($settings['existingElements'], $processedElementIds); if ($elementsToDeleteDisable) { if (DuplicateHelper::isDisable($feed)) { @@ -488,7 +482,7 @@ public function afterProcessFeed($settings, $feed) FeedMe::$stepKey = null; - $message = 'Processing ' . count($this->_processedElementIds) . ' elements finished in ' . $execution_time . 's'; + $message = 'Processing ' . count($processedElementIds) . ' elements finished in ' . $execution_time . 's'; FeedMe::info($message); FeedMe::debug($message); @@ -500,7 +494,7 @@ public function afterProcessFeed($settings, $feed) $this->trigger(self::EVENT_AFTER_PROCESS_FEED, $event); } - public function debugFeed($feed, $limit, $offset) + public function debugFeed($feed, $limit, $offset, $processedElementIds) { $feed->debug = true; @@ -523,15 +517,15 @@ public function debugFeed($feed, $limit, $offset) $feedSettings = $this->beforeProcessFeed($feed, $feedData); foreach ($feedData as $key => $data) { - $element = $this->processFeed($key, $feedSettings); + $element = $this->processFeed($key, $feedSettings, $processedElementIds); } // Check if we need to paginate the feed to run again if ($feed->getNextPagination()) { - $this->debugFeed($feed, null, null); + $this->debugFeed($feed, null, null, $processedElementIds); + } else { + $this->afterProcessFeed($feedSettings, $feed, $processedElementIds); } - - $this->afterProcessFeed($feedSettings, $feed); }