Skip to content

Commit

Permalink
Refactor afterProcessFeed to work with pagination
Browse files Browse the repository at this point in the history
  • Loading branch information
@engram-design
engram-design committed Feb 11, 2019
1 parent 5b09d7b commit 816fcb5
Show file tree
Hide file tree
Showing 3 changed files with 27 additions and 22 deletions.
10 changes: 9 additions & 1 deletion src/controllers/FeedsController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,10 @@ public function actionDebug()

ob_start();

FeedMe::$plugin->process->debugFeed($feed, $limit, $offset);
// Keep track of processed elements here - particularly for paginated feeds
$processedElementIds = [];

FeedMe::$plugin->process->debugFeed($feed, $limit, $offset, $processedElementIds);

return ob_get_clean();
}
Expand All @@ -226,6 +229,9 @@ private function _runImportTask($feed)
$limit = $request->getParam('limit');
$offset = $request->getParam('offset');

// Keep track of processed elements here - particularly for paginated feeds
$processedElementIds = [];

// Are we running from the CP?
if ($request->getIsCpRequest()) {
// if not using the direct param for this request, do UI stuff
Expand All @@ -234,6 +240,7 @@ private function _runImportTask($feed)
// Create the import task
Craft::$app->getQueue()->delay(0)->push(new FeedImport([
'feed' => $feed,
'processedElementIds' => $processedElementIds,
]));
}

Expand All @@ -252,6 +259,7 @@ private function _runImportTask($feed)
'feed' => $feed,
'limit' => $limit,
'offset' => $offset,
'processedElementIds' => $processedElementIds,
]));
}

Expand Down
9 changes: 6 additions & 3 deletions src/queue/jobs/FeedImport.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ class FeedImport extends BaseJob
public $feed;
public $limit;
public $offset;
public $processedElementIds;


// Public Methods
Expand Down Expand Up @@ -46,7 +47,7 @@ public function execute($queue)

foreach ($feedData as $key => $data) {
try {
$element = FeedMe::$plugin->process->processFeed($key, $feedSettings);
$element = FeedMe::$plugin->process->processFeed($key, $feedSettings, $this->processedElementIds);
} catch (\Throwable $e) {
// We want to catch any issues in each iteration of the loop (and log them), but this allows the
// rest of the feed to continue processing.
Expand All @@ -62,10 +63,12 @@ public function execute($queue)
'feed' => $this->feed,
'limit' => $this->limit,
'offset' => $this->offset,
'processedElementIds' => $this->processedElementIds,
]));
} else {
// Only perform the afterProcessFeed function after any/all pagination is done
FeedMe::$plugin->process->afterProcessFeed($feedSettings, $this->feed, $this->processedElementIds);
}

FeedMe::$plugin->process->afterProcessFeed($feedSettings, $this->feed);
} catch (\Throwable $e) {
// Even though we catch errors on each step of the loop, make sure to catch errors that can be anywhere
// else in this function, just to be super-safe and not cause the queue job to die.
Expand Down
30 changes: 12 additions & 18 deletions src/services/Process.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,6 @@ class Process extends Component
// Properties
// =========================================================================

private $_processedElements = [];
private $_processedElementIds = [];
private $_time_start = null;

private $_service = null;
Expand Down Expand Up @@ -79,10 +77,6 @@ public function beforeProcessFeed($feed, $feedData)

App::maxPowerCaptain();

// Reset properties to allow an instance of this service to be reused
$this->_processedElements = [];
$this->_processedElementIds = [];

// Add some additional information to our FeedModel - for ease of use in processing
// $return['fields'] = [];
$return['existingElements'] = [];
Expand Down Expand Up @@ -149,7 +143,7 @@ public function beforeProcessFeed($feed, $feedData)
return $return;
}

public function processFeed($step, $feed)
public function processFeed($step, $feed, &$processedElementIds)
{
$existingElement = false;
$uniqueMatches = [];
Expand Down Expand Up @@ -288,7 +282,7 @@ public function processFeed($step, $feed)
if (DuplicateHelper::isDisable($feed, true) || DuplicateHelper::isDelete($feed, true)) {
// If there's an existing element, we want to keep it, otherwise remove it
if ($existingElement) {
$this->_processedElementIds[] = $existingElement->id;
$processedElementIds[] = $existingElement->id;
}

return;
Expand Down Expand Up @@ -406,7 +400,7 @@ public function processFeed($step, $feed)
FeedMe::debug($info);
FeedMe::debug($contentData);

$this->_processedElementIds[] = $element->id;
$processedElementIds[] = $element->id;

return;
}
Expand Down Expand Up @@ -437,7 +431,7 @@ public function processFeed($step, $feed)
}

// Store our successfully processed element for feedback in logs, but also in case we're deleting
$this->_processedElementIds[] = $element->id;
$processedElementIds[] = $element->id;

FeedMe::info('Finished processing of node `#{i}`.', ['i' => ($step + 1)]);

Expand All @@ -458,14 +452,14 @@ public function processFeed($step, $feed)
}
}

public function afterProcessFeed($settings, $feed)
public function afterProcessFeed($settings, $feed, $processedElementIds)
{
if (DuplicateHelper::isDelete($feed) && DuplicateHelper::isDisable($feed)) {
FeedMe::info("You can't have Delete and Disabled enabled at the same time as an Import Strategy.");
return;
}

$elementsToDeleteDisable = array_diff($settings['existingElements'], $this->_processedElementIds);
$elementsToDeleteDisable = array_diff($settings['existingElements'], $processedElementIds);

if ($elementsToDeleteDisable) {
if (DuplicateHelper::isDisable($feed)) {
Expand All @@ -488,7 +482,7 @@ public function afterProcessFeed($settings, $feed)

FeedMe::$stepKey = null;

$message = 'Processing ' . count($this->_processedElementIds) . ' elements finished in ' . $execution_time . 's';
$message = 'Processing ' . count($processedElementIds) . ' elements finished in ' . $execution_time . 's';
FeedMe::info($message);
FeedMe::debug($message);

Expand All @@ -500,7 +494,7 @@ public function afterProcessFeed($settings, $feed)
$this->trigger(self::EVENT_AFTER_PROCESS_FEED, $event);
}

public function debugFeed($feed, $limit, $offset)
public function debugFeed($feed, $limit, $offset, $processedElementIds)
{
$feed->debug = true;

Expand All @@ -523,15 +517,15 @@ public function debugFeed($feed, $limit, $offset)
$feedSettings = $this->beforeProcessFeed($feed, $feedData);

foreach ($feedData as $key => $data) {
$element = $this->processFeed($key, $feedSettings);
$element = $this->processFeed($key, $feedSettings, $processedElementIds);
}

// Check if we need to paginate the feed to run again
if ($feed->getNextPagination()) {
$this->debugFeed($feed, null, null);
$this->debugFeed($feed, null, null, $processedElementIds);
} else {
$this->afterProcessFeed($feedSettings, $feed, $processedElementIds);
}

$this->afterProcessFeed($feedSettings, $feed);
}


Expand Down

0 comments on commit 816fcb5

Please sign in to comment.