diff --git a/src/base/DataType.php b/src/base/DataType.php index a08c6997..f231ed3f 100644 --- a/src/base/DataType.php +++ b/src/base/DataType.php @@ -53,7 +53,7 @@ public function setupPaginationUrl($array, $feed): void // if the feed provides a root relative URL, make it whole again based on the feed. if ($url && UrlHelper::isRootRelativeUrl($url)) { - $url = UrlHelper::hostInfo($feed->feedUrl) . $url; + $url = UrlHelper::hostInfo($feed->getFeedUrl()) . $url; } // Replace the mapping value with the actual URL diff --git a/src/base/Field.php b/src/base/Field.php index 6aab418f..e61fdd3c 100644 --- a/src/base/Field.php +++ b/src/base/Field.php @@ -8,6 +8,7 @@ use craft\base\Component; use craft\errors\ElementNotFoundException; use craft\feedme\helpers\DataHelper; +use craft\feedme\models\FeedModel; use craft\feedme\Plugin; use craft\helpers\Json; use Throwable; @@ -46,7 +47,7 @@ abstract class Field extends Component public mixed $field = null; /** - * @var + * @var FeedModel|array */ public mixed $feed = null; @@ -167,7 +168,7 @@ protected function populateElementFields($elementIds, $nodeKey = null): void // Arrayed content doesn't provide defaults because it's unable to determine how many items it _should_ return // This also checks if there was any data that corresponds on the same array index/level as our element - $value = Hash::get($fieldValue, $nodeKey, $default); + $value = Hash::get($fieldValue, $nodeKey ?? 0, $default); if ($value) { $fieldData[$elementId][$fieldHandle] = $value; @@ -176,9 +177,16 @@ protected function populateElementFields($elementIds, $nodeKey = null): void } // Now, for each element, we need to save the contents - foreach ($fieldData as $elementId => $fieldContent) { + foreach ($fieldData as $elementId => $elementData) { $element = $elementsService->getElementById($elementId, null, Hash::get($this->feed, 'siteId')); + $fieldHandles = collect($element->getFieldLayout()->getCustomFields())->pluck('handle')->all(); + $attributeKeys = array_diff(array_keys($elementData), $fieldHandles); + + $attributeContent = collect($elementData)->only($attributeKeys)->all(); + $element->setAttributes($attributeContent); + + $fieldContent = collect($elementData)->except($attributeKeys)->all(); $element->setFieldValues($fieldContent); Plugin::debug([ @@ -191,7 +199,7 @@ protected function populateElementFields($elementIds, $nodeKey = null): void Plugin::error('`{handle}` - Unable to save sub-field: `{e}`.', ['e' => Json::encode($element->getErrors()), 'handle' => $this->fieldHandle]); } - Plugin::info('`{handle}` - Processed {name} [`#{id}`]({url}) sub-fields with content: `{content}`.', ['name' => $element::displayName(), 'id' => $elementId, 'url' => $element->cpEditUrl, 'handle' => $this->fieldHandle, 'content' => Json::encode($fieldContent)]); + Plugin::info('`{handle}` - Processed {name} [`#{id}`]({url}) sub-fields with content: `{content}`.', ['name' => $element::displayName(), 'id' => $elementId, 'url' => $element->cpEditUrl, 'handle' => $this->fieldHandle, 'content' => Json::encode($elementData)]); } } diff --git a/src/console/controllers/FeedsController.php b/src/console/controllers/FeedsController.php index be29ded4..c8193d2f 100644 --- a/src/console/controllers/FeedsController.php +++ b/src/console/controllers/FeedsController.php @@ -49,7 +49,11 @@ public function options($actionID): array $options[] = 'limit'; $options[] = 'offset'; $options[] = 'continueOnError'; - $options[] = 'all'; + + if ($actionID !== 'queue') { + $options[] = 'all'; + } + return $options; } @@ -117,4 +121,30 @@ protected function queueFeed($feed, $limit = null, $offset = null, bool $continu $this->stdout('done' . PHP_EOL, Console::FG_GREEN); } + + /** + * Execute a feed without the queue + */ + public function actionExecute(string $feedId=null): int + { + $queue = new \yii\queue\sync\Queue; + Plugin::getInstance()->queue = $queue; + + $config = [ + 'feed' => Plugin::getInstance()?->getFeeds()->getFeedById($feedId), + 'limit' => $this->limit, + 'offset' => $this->offset, + 'continueOnError' => $this->continueOnError, + ]; + + // Push the first job in to the queue + $job = new FeedImport($config); + $queue->push($job); + + // Run the queue, subsequent pages will be pushed in at the + // end of an existing page + $queue->run(); + + return 1; + } } diff --git a/src/controllers/FeedsController.php b/src/controllers/FeedsController.php index d57fd026..d257f885 100644 --- a/src/controllers/FeedsController.php +++ b/src/controllers/FeedsController.php @@ -39,7 +39,7 @@ class FeedsController extends Controller */ public function actionFeedsIndex(): Response { - $variables['feeds'] = Plugin::$plugin->feeds->getFeeds(); + $variables['feeds'] = Plugin::$plugin->feeds->getFeeds('name asc'); return $this->renderTemplate('feed-me/feeds/index', $variables); } diff --git a/src/elements/Entry.php b/src/elements/Entry.php index 4463fe58..0b7c0de8 100644 --- a/src/elements/Entry.php +++ b/src/elements/Entry.php @@ -124,9 +124,10 @@ public function getQuery($settings, array $params = []): mixed */ public function setModel($settings): ElementInterface { - $this->element = new EntryElement(); - $this->element->sectionId = $settings['elementGroup'][EntryElement::class]['section']; - $this->element->typeId = $settings['elementGroup'][EntryElement::class]['entryType']; + $this->element = new EntryElement([ + 'sectionId' => $settings['elementGroup'][EntryElement::class]['section'], + 'typeId' => $settings['elementGroup'][EntryElement::class]['entryType'], + ]); $section = Craft::$app->getSections()->getSectionById($this->element->sectionId); $siteId = Hash::get($settings, 'siteId'); diff --git a/src/fields/Matrix.php b/src/fields/Matrix.php index dd231357..bfd1ce59 100644 --- a/src/fields/Matrix.php +++ b/src/fields/Matrix.php @@ -170,13 +170,10 @@ public function parseField(): mixed $index = 1; $resultBlocks = []; foreach ($expanded as $blockData) { - // all the fields are empty and setEmptyValues is off, ignore the block - if ( - !empty(array_filter( - $blockData['fields'], - fn($value) => (is_string($value) && !empty($value)) || (is_array($value) && !empty(array_filter($value))) - )) - ) { + $setEmptyValues = $this->feed['setEmptyValues'] ?? false; + $isNotEmpty = collect($blockData['fields'])->filter(fn ($value) => ! empty($value))->isNotEmpty(); + + if ($setEmptyValues || (! $setEmptyValues && $isNotEmpty)) { $resultBlocks['new' . $index++] = $blockData; } } diff --git a/src/helpers/AssetHelper.php b/src/helpers/AssetHelper.php index 1c213bcd..b9d6d980 100644 --- a/src/helpers/AssetHelper.php +++ b/src/helpers/AssetHelper.php @@ -2,6 +2,7 @@ namespace craft\feedme\helpers; +use Aws\S3\Exception\S3Exception; use Cake\Utility\Hash; use Craft; use craft\base\ElementInterface; @@ -100,39 +101,60 @@ public static function fetchRemoteImage(array $urls, $fieldInfo, $feed, $field = // Download each image. Note we've already checked if there's an existing asset and if the // user has set to use that instead, so we're good to proceed. foreach ($urls as $url) { - try { - $filename = $newFilename ? AssetsHelper::prepareAssetName($newFilename, false) : self::getRemoteUrlFilename($url); - - $fetchedImage = $tempFeedMePath . $filename; - - // But also check if we've downloaded this recently, use the copy in the temp directory - $cachedImage = FileHelper::findFiles($tempFeedMePath, [ - 'only' => [$filename], - 'recursive' => false, - ]); - - Plugin::info('Fetching remote image `{i}` - `{j}`', ['i' => $url, 'j' => $filename]); - - if (!$cachedImage) { - self::downloadFile($url, $fetchedImage); - } else { - $fetchedImage = $cachedImage[0]; + $try = 0; + $maxTries = 5; + while($try++ < $maxTries) { + try { + $filename = $newFilename ? AssetsHelper::prepareAssetName($newFilename, false) : self::getRemoteUrlFilename($url); + + $fetchedImage = $tempFeedMePath . $filename; + + // But also check if we've downloaded this recently, use the copy in the temp directory + $cachedImage = FileHelper::findFiles($tempFeedMePath, [ + 'only' => [$filename], + 'recursive' => false, + ]); + + Plugin::info('Fetching remote image `{i}` - `{j}`', ['i' => $url, 'j' => $filename]); + + if (!$cachedImage) { + self::downloadFile($url, $fetchedImage); + } else { + $fetchedImage = $cachedImage[0]; + } + + $result = self::createAsset($fetchedImage, $filename, $folderId, $field, $element, $conflict, Hash::get($feed, 'updateSearchIndexes')); + + if ($result) { + $uploadedAssets[] = $result; + } else { + Plugin::error('Failed to create asset from `{i}`', ['i' => $url]); + } + + break; + } catch (Throwable $e) { + if ($try < $maxTries) { + $prev = $e; + while ($prev) { + if (get_class($prev) === \Aws\S3\Exception\S3Exception::class && $prev->getStatusCode() === 400) { + Plugin::info('`{handle}` - Asset error, resetting credentials. {e}.', ['e' => $prev->getMessage(), 'handle' => $field->handle]); + Craft::$app->set('volumes', ["class" => "craft\services\Volumes"]); + Craft::$app->set('fs', ["class" => "craft\services\Fs"]); + } + $prev = $prev->getPrevious(); + } + Plugin::info('`{handle}` - Asset error, trying again: `{url}` - `{e}`.', ['url' => $url, 'e' => $e->getMessage(), 'handle' => $field->handle]); + sleep($try); + continue; + } + + if ($field) { + Plugin::error('`{handle}` - Asset error: `{url}` - `{e}` `{t}`.', ['url' => $url, 'e' => $e->getMessage(), 'handle' => $field->handle, 't' => $e->getTraceAsString()]); + } else { + Plugin::error('Asset error: `{url}` - `{e}`.', ['url' => $url, 'e' => $e->getMessage()]); + } + Craft::$app->getErrorHandler()->logException($e); } - - $result = self::createAsset($fetchedImage, $filename, $folderId, $field, $element, $conflict, Hash::get($feed, 'updateSearchIndexes')); - - if ($result) { - $uploadedAssets[] = $result; - } else { - Plugin::error('Failed to create asset from `{i}`', ['i' => $url]); - } - } catch (Throwable $e) { - if ($field) { - Plugin::error('`{handle}` - Asset error: `{url}` - `{e}`.', ['url' => $url, 'e' => $e->getMessage(), 'handle' => $field->handle]); - } else { - Plugin::error('Asset error: `{url}` - `{e}`.', ['url' => $url, 'e' => $e->getMessage()]); - } - Craft::$app->getErrorHandler()->logException($e); } } diff --git a/src/models/FeedModel.php b/src/models/FeedModel.php index 831dfd24..606bfa7a 100644 --- a/src/models/FeedModel.php +++ b/src/models/FeedModel.php @@ -10,6 +10,7 @@ use craft\feedme\base\ElementInterface; use craft\feedme\helpers\DuplicateHelper; use craft\feedme\Plugin; +use craft\helpers\App; use DateTime; /** @@ -156,6 +157,11 @@ public function __toString() return Craft::t('feed-me', $this->name); } + public function getFeedUrl(): string + { + return App::parseEnv($this->feedUrl); + } + /** * @return string */ @@ -234,10 +240,12 @@ public function getFeedMapping(bool $usePrimaryElement = true): mixed public function getNextPagination(): bool { if (!$this->paginationUrl || !filter_var($this->paginationUrl, FILTER_VALIDATE_URL)) { + Plugin::info('No paginationUrl, stopping.'); return false; } // Set the URL dynamically on the feed, then kick off processing again + Plugin::info('Next paginationUrl found. Creating new Queue job with '.$this->feedUrl); $this->feedUrl = $this->paginationUrl; return true; diff --git a/src/queue/jobs/FeedImport.php b/src/queue/jobs/FeedImport.php index 0d33467b..89cf9ab6 100644 --- a/src/queue/jobs/FeedImport.php +++ b/src/queue/jobs/FeedImport.php @@ -87,6 +87,7 @@ public function execute($queue): void } $feedSettings = Plugin::$plugin->process->beforeProcessFeed($this->feed, $feedData); + Plugin::info('Processing `' . $this->feed->getFeedUrl() . '`'); $feedData = $feedSettings['feedData']; diff --git a/src/services/DataTypes.php b/src/services/DataTypes.php index 0ec71b8f..f61d82b7 100644 --- a/src/services/DataTypes.php +++ b/src/services/DataTypes.php @@ -204,27 +204,43 @@ public function getRawData($url, $feedId = null): array ]); } - try { - $client = Plugin::$plugin->service->createGuzzleClient($feedId); - $options = Plugin::$plugin->service->getRequestOptions($feedId); + $tries = 0; + $maxTries = 10; + while (++$tries <= $maxTries) { + try { + $client = Plugin::$plugin->service->createGuzzleClient($feedId); + $options = Plugin::$plugin->service->getRequestOptions($feedId); - $resp = $client->request('GET', $url, $options); - $data = (string)$resp->getBody(); + $resp = $client->request('GET', $url, $options); + $data = (string)$resp->getBody(); - // Save headers for later - $this->_headers = $resp->getHeaders(); + // Save headers for later + $this->_headers = $resp->getHeaders(); - $response = ['success' => true, 'data' => $data]; - } catch (Exception $e) { - $response = ['success' => false, 'error' => $e->getMessage()]; - Craft::$app->getErrorHandler()->logException($e); - } + $response = ['success' => true, 'data' => $data]; - return $this->_triggerEventAfterFetchFeed([ - 'url' => $url, - 'feedId' => $feedId, - 'response' => $response, - ]); + return $this->_triggerEventAfterFetchFeed([ + 'url' => $url, + 'feedId' => $feedId, + 'response' => $response, + ]); + } catch (Exception $e) { + $response = ['success' => false, 'error' => $e->getMessage()]; + Craft::$app->getErrorHandler()->logException($e); + + if ($tries === $maxTries) { + return $this->_triggerEventAfterFetchFeed([ + 'url' => $url, + 'feedId' => $feedId, + 'response' => $response, + ]); + } + else { + Plugin::info('HTTP error #' . $tries . ' at ' . $url . ', ' . $e->getMessage()); + sleep(($tries ^ 2) / 2); // wait up to 40.5 seconds on the 9th try + } + } + } } /** @@ -234,10 +250,10 @@ public function getRawData($url, $feedId = null): array */ public function getFeedData($feedModel, bool $usePrimaryElement = true): mixed { - $feedDataResponse = $feedModel->getDataType()->getFeed($feedModel->feedUrl, $feedModel, $usePrimaryElement); + $feedDataResponse = $feedModel->getDataType()->getFeed($feedModel->getFeedUrl(), $feedModel, $usePrimaryElement); $event = new FeedDataEvent([ - 'url' => $feedModel->feedUrl, + 'url' => $feedModel->getFeedUrl(), 'response' => $feedDataResponse, 'feedId' => $feedModel->id, ]); diff --git a/src/templates/_includes/elements/assets/map.html b/src/templates/_includes/elements/assets/map.html index 50b9bfb7..efd791a8 100644 --- a/src/templates/_includes/elements/assets/map.html +++ b/src/templates/_includes/elements/assets/map.html @@ -42,6 +42,13 @@ type: 'select', options: folders, }, +}, { + name: 'Alternative Text', + handle: 'alt', + instructions: 'Add descriptive text to describe the appearance and function of an image on a page.'|t('feed-me'), + default: { + type: 'text', + }, }, { name: 'Asset ID', handle: 'id', diff --git a/src/web/twig/variables/FeedMeVariable.php b/src/web/twig/variables/FeedMeVariable.php index a77436d2..f114c341 100644 --- a/src/web/twig/variables/FeedMeVariable.php +++ b/src/web/twig/variables/FeedMeVariable.php @@ -240,7 +240,11 @@ public function getElementLayoutByField($type, $field): ?array } if (($fieldLayout = Craft::$app->getFields()->getLayoutById($source->fieldLayoutId)) !== null) { - return $fieldLayout->getCustomFields(); + $customFields = $fieldLayout->getCustomFields(); + if ($field instanceof \craft\fields\Assets) { + array_unshift($customFields, new \craft\fields\PlainText(['name' => 'Alt', 'handle' => 'alt'])); + } + return $customFields; } return null;