From cac8081be4b75e13991aacd7e4f7fe05e3211a2a Mon Sep 17 00:00:00 2001 From: Maksym Leskiv Date: Thu, 1 Dec 2016 14:34:17 +0100 Subject: [PATCH 1/5] [TASK] Added code to remove feeds deleted at social media directly --- Classes/Task/CleanUpTask.php | 2 +- Classes/Utility/Task/CleanUpTaskUtility.php | 101 +++++++++++++++++++- 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/Classes/Task/CleanUpTask.php b/Classes/Task/CleanUpTask.php index 8909512..bafcf89 100644 --- a/Classes/Task/CleanUpTask.php +++ b/Classes/Task/CleanUpTask.php @@ -55,7 +55,7 @@ public function execute() { * @return string Information to display */ public function getAdditionalInformation() { - return 'Delete entries older than ' . $this->getDays() . ' days.'; + return 'Delete entries older than ' . $this->getDays() . ' days or deleted at social media feed directly.'; } /** diff --git a/Classes/Utility/Task/CleanUpTaskUtility.php b/Classes/Utility/Task/CleanUpTaskUtility.php index 1c9c84a..4cc5854 100644 --- a/Classes/Utility/Task/CleanUpTaskUtility.php +++ b/Classes/Utility/Task/CleanUpTaskUtility.php @@ -1,4 +1,10 @@ 'https://graph.facebook.com/v2.6/', + Token::INSTAGRAM => 'https://api.instagram.com/v1/', + Token::INSTAGRAM_OAUTH2 => 'https://api.instagram.com/v1/' + ); + /** * initialize */ public function __construct() { - $this->dbConnection = $GLOBALS['TYPO3_DB']; + $this->objectManager = GeneralUtility::makeInstance(ObjectManager::class); } /** @@ -66,6 +98,7 @@ public function __construct() { public function run($days) { $obsoleteEntries = $this->getObsoleteEntries($days); $this->deleteObsoleteEntries($obsoleteEntries); + $this->removeDeletedFeeds(); return TRUE; } @@ -162,4 +195,68 @@ protected function deleteObsoleteEntries($obsoleteEntries) { 'uid IN (' . ltrim($uids, ',') . ') OR deleted=1' ); } -} \ No newline at end of file + + /** + * @return void + */ + private function removeDeletedFeeds() { + $configurationRepository = $this->objectManager->get(ConfigurationRepository::class); + $feedRepository = $this->objectManager->get(FeedRepository::class); + $configurations = $configurationRepository->findAll(); + foreach ($configurations as $configuration) { + $feeds = $feedRepository->findByConfiguration($configuration); + foreach ($feeds as $feed) { + $isFeedDeleted = false; + $isFeedDeleted = $this->isFeedDeleted($feed->getExternalIdentifier(), $configuration); + if ($isFeedDeleted === true) { + $feedRepository->remove($feed); + } + } + } + $this->objectManager->get(PersistenceManager::class)->persistAll(); + } + + /** + * @param string $externalIdentifier + * @param Configuration $configuration + * @return boolean + */ + private function isFeedDeleted($externalIdentifier, Configuration $configuration) { + switch ($configuration->getToken()->getSocialType()) { + case Token::FACEBOOK: + $url = $this->apiUrls[$configuration->getToken()->getSocialType()] . $externalIdentifier . + '?access_token=' . $configuration->getToken()->getCredential('appId') . '|' . $configuration->getToken()->getCredential('appSecret'); + $data = json_decode(GeneralUtility::getUrl($url), true); + if (is_array($data)) { + if ($data["id"] == $externalIdentifier) { + return false; + } else { + return true; + } + } + break; + case Token::INSTAGRAM: + case Token::INSTAGRAM_OAUTH2: + $url = $this->apiUrls[$configuration->getToken()->getSocialType()] . 'media/' . $externalIdentifier; + $url .= $configuration->getToken()->getSocialType() === Token::INSTAGRAM ? '?client_id=' . $configuration->getToken()->getCredential('clientId') : '?access_token=' . $configuration->getToken()->getCredential('accessToken'); + $data = json_decode(GeneralUtility::getUrl($url), true); + if (is_array($data)) { + if (is_array($data["data"]) && $data["data"]["id"] == $externalIdentifier) { + return false; + } else { + return true; + } + } + break; + case Token::TWITTER: + // todo + return false; + break; + default: + throw new \UnexpectedValueException('Such social type is not valid', 1466690851); + break; + } + return false; + } + +} From bba8b0556c04dd0f13681352868193827e09d26b Mon Sep 17 00:00:00 2001 From: Maksym Leskiv Date: Thu, 1 Dec 2016 14:37:17 +0100 Subject: [PATCH 2/5] [CHORE] Removed some PhpStorm comments --- Classes/Utility/Task/CleanUpTaskUtility.php | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Classes/Utility/Task/CleanUpTaskUtility.php b/Classes/Utility/Task/CleanUpTaskUtility.php index 4cc5854..d12d4c7 100644 --- a/Classes/Utility/Task/CleanUpTaskUtility.php +++ b/Classes/Utility/Task/CleanUpTaskUtility.php @@ -1,10 +1,4 @@ Date: Fri, 23 Dec 2016 10:50:09 +0100 Subject: [PATCH 3/5] [BUGFIX] removed unused Token::INSTAGRAM --- Classes/Utility/Task/CleanUpTaskUtility.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/Classes/Utility/Task/CleanUpTaskUtility.php b/Classes/Utility/Task/CleanUpTaskUtility.php index d12d4c7..0370fe6 100644 --- a/Classes/Utility/Task/CleanUpTaskUtility.php +++ b/Classes/Utility/Task/CleanUpTaskUtility.php @@ -71,7 +71,6 @@ class CleanUpTaskUtility { */ protected $apiUrls = array( Token::FACEBOOK => 'https://graph.facebook.com/v2.6/', - Token::INSTAGRAM => 'https://api.instagram.com/v1/', Token::INSTAGRAM_OAUTH2 => 'https://api.instagram.com/v1/' ); @@ -229,7 +228,6 @@ private function isFeedDeleted($externalIdentifier, Configuration $configuration } } break; - case Token::INSTAGRAM: case Token::INSTAGRAM_OAUTH2: $url = $this->apiUrls[$configuration->getToken()->getSocialType()] . 'media/' . $externalIdentifier; $url .= $configuration->getToken()->getSocialType() === Token::INSTAGRAM ? '?client_id=' . $configuration->getToken()->getCredential('clientId') : '?access_token=' . $configuration->getToken()->getCredential('accessToken'); From 7ad7ec428ccd2c4b803cede9a302f87dca58ff06 Mon Sep 17 00:00:00 2001 From: Andriy Oprysko Date: Fri, 28 Apr 2017 16:29:32 +0300 Subject: [PATCH 4/5] [TASK] delete records if it was removed in feed --- Classes/Utility/Api/TwitterApi.php | 42 ++++-- Classes/Utility/Task/CleanUpTaskUtility.php | 142 +++++++++++++++++++- Classes/Utility/Task/ImportTaskUtility.php | 11 +- 3 files changed, 180 insertions(+), 15 deletions(-) diff --git a/Classes/Utility/Api/TwitterApi.php b/Classes/Utility/Api/TwitterApi.php index e17ee88..d41540b 100644 --- a/Classes/Utility/Api/TwitterApi.php +++ b/Classes/Utility/Api/TwitterApi.php @@ -39,7 +39,7 @@ class TwitterApi /** * path to get twitter feed */ - const API_URL = 'https://api.twitter.com/1.1/statuses/user_timeline.json'; + const API_FETCH_URL = 'https://api.twitter.com/1.1/'; /** * consumer key @@ -101,13 +101,33 @@ public function __construct( $this->oauthAccessTokenSecret = $oauthAccessTokenSecret; } + /** + * perform request to api + * + * @return array + */ + public function performFetchRequest() + { + return $this->performRequest(self::API_FETCH_URL . 'statuses/user_timeline.json'); + } + + /** + * perform request to api + * + * @return array + */ + public function performStatusesLookup() + { + return $this->performRequest(self::API_FETCH_URL . 'statuses/lookup.json'); + } + /** * perform request to api * * @return array * @throws \Exception */ - public function performRequest() + protected function performRequest($url) { if (empty($this->getFields)) { throw new \Exception('Get fields could not be empty', 1463139019); @@ -117,11 +137,11 @@ public function performRequest() /** @var RequestUtility $requestUtility */ $requestUtility = GeneralUtility::makeInstance( RequestUtility::class, - self::API_URL, + $url, RequestUtility::METHOD_GET ); $requestUtility->setGetParameters($this->getGetFields()); - $requestUtility->setHeaders(['Authorization' => $this->getAuthHeader()]); + $requestUtility->setHeaders(['Authorization' => $this->getAuthHeader($url)]); $response = $requestUtility->send(); if (!empty($response)) { @@ -134,9 +154,10 @@ public function performRequest() /** * Get Authorization header * + * @param string $url * @return string */ - protected function getAuthHeader() + protected function getAuthHeader($url) { $oauth = [ 'oauth_consumer_key' => $this->consumerKey, @@ -147,7 +168,7 @@ protected function getAuthHeader() 'oauth_version' => '1.0' ]; - $sigBase = $this->buildSigBase(array_merge($oauth, $this->getGetFields())); + $sigBase = $this->buildSigBase(array_merge($oauth, $this->getGetFields()), $url); $sigKey = rawurlencode($this->consumerSecret) . '&' . rawurlencode($this->oauthAccessTokenSecret); $oauth['oauth_signature'] = base64_encode(hash_hmac('sha1', $sigBase, $sigKey, true)); @@ -168,19 +189,20 @@ protected function getAuthHeader() * Private method to generate the base string * * @param array $oauth - * + * @param string $url * @return string Built base string */ - private function buildSigBase($oauth) + private function buildSigBase($oauth, $url) { ksort($oauth); $urlParts = []; foreach ($oauth as $key => $value) { - $urlParts[] = $key . '=' . $value; + $urlParts[] = $key . '=' . rawurlencode($value); } - return 'GET' . '&' . rawurlencode(self::API_URL) . '&' . rawurlencode(implode('&', $urlParts)); + return 'GET' . '&' . rawurlencode($url) . '&' + . rawurlencode(implode('&', $urlParts)); } /** diff --git a/Classes/Utility/Task/CleanUpTaskUtility.php b/Classes/Utility/Task/CleanUpTaskUtility.php index 681f88b..14f1367 100644 --- a/Classes/Utility/Task/CleanUpTaskUtility.php +++ b/Classes/Utility/Task/CleanUpTaskUtility.php @@ -26,6 +26,11 @@ * * This copyright notice MUST APPEAR in all copies of the script! ***************************************************************/ + +use Pixelant\PxaSocialFeed\Domain\Model\Configuration; +use Pixelant\PxaSocialFeed\Domain\Model\Token; +use Pixelant\PxaSocialFeed\Domain\Repository\ConfigurationRepository; +use Pixelant\PxaSocialFeed\Utility\Api\TwitterApi; use TYPO3\CMS\Core\Utility\GeneralUtility; /** @@ -164,9 +169,144 @@ protected function deleteObsoleteEntries($obsoleteEntries) $uids .= ',' . $obsoleteEntry['uid']; } + $where = ''; + if (!empty($uids)) { + $where .= 'uid IN (' . ltrim($uids, ',') . ') OR '; + } + $where .= 'deleted=1'; + $this->dbConnection->exec_DELETEquery( self::TABLE_FEED, - 'uid IN (' . ltrim($uids, ',') . ') OR deleted=1' + $where ); } + + /** + * Remove feed entries which were not found + * + * @return void + */ + private function removeDeletedFeeds() + { + $configurations = $this->objectManager->get(ConfigurationRepository::class)->findAll(); + $feedsToRemove = []; + + /** @var Configuration $configuration */ + foreach ($configurations as $configuration) { + $feeds = $this->dbConnection->exec_SELECTgetRows( + 'uid,external_identifier', + self::TABLE_FEED, + 'configuration=' . $configuration->getUid() + ); + + if ($configuration->getToken()->getSocialType() !== Token::TWITTER) { + foreach ($feeds as $feed) { + if ($this->isFeedDeleted($feed['external_identifier'], $configuration)) { + $feedsToRemove[] = $feed; + } + } + } else { + $this->checkTwitterFeeds($configuration, $feeds, $feedsToRemove); + } + } + + $this->deleteObsoleteEntries($feedsToRemove); + } + + /** + * @param string $externalIdentifier + * @param Configuration $configuration + * @return boolean + */ + private function isFeedDeleted($externalIdentifier, Configuration $configuration) + { + switch ($configuration->getToken()->getSocialType()) { + case Token::FACEBOOK: + $url = sprintf( + ImportTaskUtility::FACEBOOK_API_URL . '%s?access_token=%s|%s', + $externalIdentifier, + $configuration->getToken()->getCredential('appId'), + $configuration->getToken()->getCredential('appSecret') + ); + + $data = json_decode(GeneralUtility::getUrl($url), true); + + return !(isset($data['id']) && $data['id'] == $externalIdentifier); + break; + case Token::INSTAGRAM_OAUTH2: + $url = sprintf( + ImportTaskUtility::INSTAGRAM_API_URL . 'media/%s?access_token=%s', + $externalIdentifier, + $configuration->getToken()->getCredential('accessToken') + ); + + $data = json_decode(GeneralUtility::getUrl($url), true); + + return !(isset($data['data']['id']) && $data['data']['id'] == $externalIdentifier); + break; + default: + throw new \UnexpectedValueException('Such social type is not valid', 1466690851); + break; + } + } + + /** + * @param Configuration $configuration + * @param array $twitterFeeds + * @param array &$feedsToRemove + */ + private function checkTwitterFeeds(Configuration $configuration, $twitterFeeds, &$feedsToRemove) + { + if (!empty($twitterFeeds)) { + do { + // twitter limit + $feedsList = array_slice($twitterFeeds, 0, 99); + $twitterFeeds = array_slice($twitterFeeds, 99); + + $fields = [ + 'id' => $this->getListOfArrayField('external_identifier', $feedsList), + 'include_entities' => 'false', + 'trim_user' => 1, + 'map' => 'false' + ]; + + /** @var TwitterApi $twitterApi */ + $twitterApi = GeneralUtility::makeInstance( + TwitterApi::class, + $configuration->getToken()->getCredential('consumerKey'), + $configuration->getToken()->getCredential('consumerSecret'), + $configuration->getToken()->getCredential('accessToken'), + $configuration->getToken()->getCredential('accessTokenSecret') + ); + + $data = $twitterApi->setGetFields($fields)->performStatusesLookup(); + + $availableItems = $this->getListOfArrayField('id_str', $data); + + foreach ($feedsList as $feedListItem) { + if (!GeneralUtility::inList($availableItems, $feedListItem['external_identifier'])) { + $feedsToRemove[] = $feedListItem; + } + } + + } while (count($twitterFeeds) > 0); + } + } + + /** + * Get list of array field + * + * @param string $field + * @param array $listArray + * @return string + */ + private function getListOfArrayField($field, $listArray = []) + { + $list = []; + foreach ($listArray as $item) { + $list[] = $item[$field]; + } + + return implode(',', $list); + } } diff --git a/Classes/Utility/Task/ImportTaskUtility.php b/Classes/Utility/Task/ImportTaskUtility.php index f755cba..4a27bb6 100644 --- a/Classes/Utility/Task/ImportTaskUtility.php +++ b/Classes/Utility/Task/ImportTaskUtility.php @@ -39,6 +39,9 @@ class ImportTaskUtility { + const FACEBOOK_API_URL = 'https://graph.facebook.com/v2.9/'; + + const INSTAGRAM_API_URL = 'https://api.instagram.com/v1/'; /** * objectManager @@ -89,7 +92,7 @@ public function run($configurationsUids) //getting data array from facebook graph api json result // @codingStandardsIgnoreStart $url = sprintf( - 'https://graph.facebook.com/v2.6/%s/posts/?fields=likes.summary(true).limit(0),message,attachments,created_time,updated_time&limit=%d&access_token=%s|%s', + self::FACEBOOK_API_URL . '%s/posts/?fields=likes.summary(true).limit(0),message,attachments,created_time,updated_time&limit=%d&access_token=%s|%s', $configuration->getSocialId(), $configuration->getFeedsLimit(), $configuration->getToken()->getCredential('appId'), @@ -112,7 +115,7 @@ public function run($configurationsUids) case Token::INSTAGRAM_OAUTH2: // getting data array from instagram api json result //predefine a format of request string; - $urlFormat = 'https://api.instagram.com/v1/%s/%s/media/recent/?access_token=%s&count=%d'; + $urlFormat = self::INSTAGRAM_API_URL . '%s/%s/media/recent/?access_token=%s&count=%d'; // hashtag used in configuration (leading '#' symbol): preparing values for 'tag' API call if (GeneralUtility::isFirstPartOfStr($configuration->getSocialId(), '#')) { @@ -153,7 +156,7 @@ public function run($configurationsUids) 'include_rts' => 1 ]; - /** @var \Pixelant\PxaSocialFeed\Utility\Api\TwitterApi $twitterApi */ + /** @var TwitterApi $twitterApi */ $twitterApi = GeneralUtility::makeInstance( TwitterApi::class, $configuration->getToken()->getCredential('consumerKey'), @@ -162,7 +165,7 @@ public function run($configurationsUids) $configuration->getToken()->getCredential('accessTokenSecret') ); - $data = $twitterApi->setGetFields($fields)->performRequest(); + $data = $twitterApi->setGetFields($fields)->performFetchRequest(); if (is_array($data)) { $this->saveTwitterFeed($data, $configuration); From 68e38b813fbf3a078138ff2fd6f9cf81a8a16b08 Mon Sep 17 00:00:00 2001 From: Andriy Oprysko Date: Fri, 28 Apr 2017 16:43:25 +0300 Subject: [PATCH 5/5] [TASK] raise version --- ext_emconf.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext_emconf.php b/ext_emconf.php index 2baebff..f747456 100644 --- a/ext_emconf.php +++ b/ext_emconf.php @@ -15,7 +15,7 @@ 'uploadfolder' => '0', 'createDirs' => '', 'clearCacheOnLoad' => 0, - 'version' => '1.4.0', + 'version' => '1.5.0', 'constraints' => [ 'depends' => [ 'typo3' => '7.6.0-8.9.99',