From 95ceba51a5607a551b55cab692ed0fe69c720f17 Mon Sep 17 00:00:00 2001 From: Ash Date: Mon, 29 Jan 2024 20:01:31 +0800 Subject: [PATCH] refactor(greader): incrementally fetch the unread items by last sync time --- .../reader/domain/repository/ArticleDao.kt | 46 ++- .../domain/service/AbstractRssRepository.kt | 20 +- .../reader/domain/service/AccountService.kt | 8 +- .../reader/domain/service/FeverRssService.kt | 45 +-- .../domain/service/GoogleReaderRssService.kt | 331 +++++++++++------- .../reader/infrastructure/html/Readability.kt | 28 ++ .../reader/infrastructure/rss/RssHelper.kt | 16 +- .../rss/provider/greader/GoogleReaderAPI.kt | 54 ++- .../rss/provider/greader/GoogleReaderDTO.kt | 1 + 9 files changed, 358 insertions(+), 191 deletions(-) create mode 100644 app/src/main/java/me/ash/reader/infrastructure/html/Readability.kt diff --git a/app/src/main/java/me/ash/reader/domain/repository/ArticleDao.kt b/app/src/main/java/me/ash/reader/domain/repository/ArticleDao.kt index 00a9b4ccb..2173269ec 100644 --- a/app/src/main/java/me/ash/reader/domain/repository/ArticleDao.kt +++ b/app/src/main/java/me/ash/reader/domain/repository/ArticleDao.kt @@ -1,17 +1,59 @@ package me.ash.reader.domain.repository import androidx.paging.PagingSource -import androidx.room.* +import androidx.room.Dao +import androidx.room.Insert +import androidx.room.OnConflictStrategy +import androidx.room.Query +import androidx.room.RewriteQueriesToDropUnusedColumns +import androidx.room.Transaction +import androidx.room.Update import kotlinx.coroutines.flow.Flow import me.ash.reader.domain.model.article.Article import me.ash.reader.domain.model.article.ArticleMeta import me.ash.reader.domain.model.article.ArticleWithFeed import me.ash.reader.domain.model.feed.ImportantNum -import java.util.* +import java.util.Date @Dao interface ArticleDao { + @Transaction + @RewriteQueriesToDropUnusedColumns + @Query( + """ + SELECT count(1) + FROM article + WHERE feedId = :feedId + AND isStarred = :isStarred + AND accountId = :accountId + """ + ) + fun countByFeedIdWhenIsStarred( + accountId: Int, + feedId: String, + isStarred: Boolean, + ): Int + + @Transaction + @RewriteQueriesToDropUnusedColumns + @Query( + """ + SELECT count(1) + FROM article AS a + LEFT JOIN feed AS b ON b.id = a.feedId + LEFT JOIN `group` AS c ON c.id = b.groupId + WHERE c.id = :groupId + AND a.isStarred = :isStarred + AND a.accountId = :accountId + """ + ) + fun countByGroupIdWhenIsStarred( + accountId: Int, + groupId: String, + isStarred: Boolean, + ): Int + @Transaction @Query( """ diff --git a/app/src/main/java/me/ash/reader/domain/service/AbstractRssRepository.kt b/app/src/main/java/me/ash/reader/domain/service/AbstractRssRepository.kt index 414b1360d..b76055c1f 100644 --- a/app/src/main/java/me/ash/reader/domain/service/AbstractRssRepository.kt +++ b/app/src/main/java/me/ash/reader/domain/service/AbstractRssRepository.kt @@ -31,7 +31,8 @@ import me.ash.reader.infrastructure.rss.RssHelper import me.ash.reader.ui.ext.currentAccountId import me.ash.reader.ui.ext.decodeHTML import me.ash.reader.ui.ext.spacerDollar -import java.util.* +import java.util.Date +import java.util.UUID abstract class AbstractRssRepository( private val context: Context, @@ -311,13 +312,24 @@ abstract class AbstractRssRepository( feedDao.update(feed) } - open suspend fun deleteGroup(group: Group) { + open suspend fun deleteGroup(group: Group, onlyDeleteNoStarred: Boolean? = false) { + val accountId = context.currentAccountId + if (onlyDeleteNoStarred == true + && articleDao.countByGroupIdWhenIsStarred(accountId, group.id, true) > 0 + ) { + return + } deleteArticles(group = group) - feedDao.deleteByGroupId(context.currentAccountId, group.id) + feedDao.deleteByGroupId(accountId, group.id) groupDao.delete(group) } - open suspend fun deleteFeed(feed: Feed) { + open suspend fun deleteFeed(feed: Feed, onlyDeleteNoStarred: Boolean? = false) { + if (onlyDeleteNoStarred == true + && articleDao.countByFeedIdWhenIsStarred(context.currentAccountId, feed.id, true) > 0 + ) { + return + } deleteArticles(feed = feed) feedDao.delete(feed) } diff --git a/app/src/main/java/me/ash/reader/domain/service/AccountService.kt b/app/src/main/java/me/ash/reader/domain/service/AccountService.kt index 7e1943479..22d370ecf 100644 --- a/app/src/main/java/me/ash/reader/domain/service/AccountService.kt +++ b/app/src/main/java/me/ash/reader/domain/service/AccountService.kt @@ -12,7 +12,12 @@ import me.ash.reader.domain.repository.AccountDao import me.ash.reader.domain.repository.ArticleDao import me.ash.reader.domain.repository.FeedDao import me.ash.reader.domain.repository.GroupDao -import me.ash.reader.ui.ext.* +import me.ash.reader.ui.ext.DataStoreKeys +import me.ash.reader.ui.ext.currentAccountId +import me.ash.reader.ui.ext.dataStore +import me.ash.reader.ui.ext.getDefaultGroupId +import me.ash.reader.ui.ext.put +import me.ash.reader.ui.ext.showToast import javax.inject.Inject class AccountService @Inject constructor( @@ -72,6 +77,7 @@ class AccountService @Inject constructor( Looper.loop() return } + rssService.get().cancelSync() accountDao.queryById(accountId)?.let { articleDao.deleteByAccountId(accountId) feedDao.deleteByAccountId(accountId) diff --git a/app/src/main/java/me/ash/reader/domain/service/FeverRssService.kt b/app/src/main/java/me/ash/reader/domain/service/FeverRssService.kt index a2bf84fd2..abe464035 100644 --- a/app/src/main/java/me/ash/reader/domain/service/FeverRssService.kt +++ b/app/src/main/java/me/ash/reader/domain/service/FeverRssService.kt @@ -25,13 +25,18 @@ import me.ash.reader.infrastructure.android.NotificationHelper import me.ash.reader.infrastructure.di.DefaultDispatcher import me.ash.reader.infrastructure.di.IODispatcher import me.ash.reader.infrastructure.di.MainDispatcher +import me.ash.reader.infrastructure.html.Readability import me.ash.reader.infrastructure.rss.RssHelper import me.ash.reader.infrastructure.rss.provider.fever.FeverAPI import me.ash.reader.infrastructure.rss.provider.fever.FeverDTO -import me.ash.reader.ui.ext.* -import net.dankito.readability4j.extended.Readability4JExtended -import java.util.* +import me.ash.reader.ui.ext.currentAccountId +import me.ash.reader.ui.ext.decodeHTML +import me.ash.reader.ui.ext.dollarLast +import me.ash.reader.ui.ext.showToast +import me.ash.reader.ui.ext.spacerDollar +import java.util.Date import javax.inject.Inject +import kotlin.collections.set class FeverRssService @Inject constructor( @ApplicationContext @@ -96,11 +101,11 @@ class FeverRssService @Inject constructor( throw Exception("Unsupported") } - override suspend fun deleteGroup(group: Group) { + override suspend fun deleteGroup(group: Group, onlyDeleteNoStarred: Boolean?) { throw Exception("Unsupported") } - override suspend fun deleteFeed(feed: Feed) { + override suspend fun deleteFeed(feed: Feed, onlyDeleteNoStarred: Boolean?) { throw Exception("Unsupported") } @@ -144,12 +149,6 @@ class FeverRssService @Inject constructor( ) } ?: emptyList() groupDao.insertOrUpdate(groups) - val groupIds = groups.map { it.id } - groupDao.queryAll(accountId).forEach { - if (!groupIds.contains(it.id)) { - super.deleteGroup(it) - } - } // 2. Fetch the Fever feeds val feedsBody = feverAPI.getFeeds() @@ -161,11 +160,6 @@ class FeverRssService @Inject constructor( } } } - feedDao.queryAll(accountId).forEach { - if (!feedsGroupsMap.contains(it.id.dollarLast())) { - super.deleteFeed(it) - } - } // Fetch the Fever favicons val faviconsById = feverAPI.getFavicons().favicons?.associateBy { it.id } ?: emptyMap() @@ -201,10 +195,7 @@ class FeverRssService @Inject constructor( title = it.title.decodeHTML() ?: context.getString(R.string.empty), author = it.author, rawDescription = it.html ?: "", - shortDescription = (Readability4JExtended("", it.html ?: "") - .parse().textContent ?: "") - .take(110) - .trim(), + shortDescription = Readability.parseToText(it.html, it.url).take(110), fullContent = it.html, img = rssHelper.findImg(it.html ?: ""), link = it.url ?: "", @@ -241,6 +232,20 @@ class FeverRssService @Inject constructor( } } + // Remove orphaned groups and feeds, after synchronizing the starred/un-starred + val groupIds = groups.map { it.id } + groupDao.queryAll(accountId).forEach { + if (!groupIds.contains(it.id)) { + super.deleteGroup(it, true) + } + } + + feedDao.queryAll(accountId).forEach { + if (!feedsGroupsMap.contains(it.id.dollarLast())) { + super.deleteFeed(it, true) + } + } + Log.i("RLog", "onCompletion: ${System.currentTimeMillis() - preTime}") accountDao.update(account.apply { diff --git a/app/src/main/java/me/ash/reader/domain/service/GoogleReaderRssService.kt b/app/src/main/java/me/ash/reader/domain/service/GoogleReaderRssService.kt index 651ed63f1..508c7e28a 100644 --- a/app/src/main/java/me/ash/reader/domain/service/GoogleReaderRssService.kt +++ b/app/src/main/java/me/ash/reader/domain/service/GoogleReaderRssService.kt @@ -25,15 +25,20 @@ import me.ash.reader.infrastructure.android.NotificationHelper import me.ash.reader.infrastructure.di.DefaultDispatcher import me.ash.reader.infrastructure.di.IODispatcher import me.ash.reader.infrastructure.di.MainDispatcher +import me.ash.reader.infrastructure.html.Readability import me.ash.reader.infrastructure.rss.RssHelper import me.ash.reader.infrastructure.rss.provider.greader.GoogleReaderAPI import me.ash.reader.infrastructure.rss.provider.greader.GoogleReaderAPI.Companion.ofCategoryStreamIdToId import me.ash.reader.infrastructure.rss.provider.greader.GoogleReaderAPI.Companion.ofFeedStreamIdToId import me.ash.reader.infrastructure.rss.provider.greader.GoogleReaderAPI.Companion.ofItemStreamIdToId import me.ash.reader.infrastructure.rss.provider.greader.GoogleReaderDTO -import me.ash.reader.ui.ext.* -import net.dankito.readability4j.extended.Readability4JExtended -import java.util.* +import me.ash.reader.ui.ext.currentAccountId +import me.ash.reader.ui.ext.decodeHTML +import me.ash.reader.ui.ext.dollarLast +import me.ash.reader.ui.ext.showToast +import me.ash.reader.ui.ext.spacerDollar +import java.util.Calendar +import java.util.Date import javax.inject.Inject class GoogleReaderRssService @Inject constructor( @@ -164,168 +169,226 @@ class GoogleReaderRssService @Inject constructor( super.renameFeed(feed) } - override suspend fun deleteGroup(group: Group) { + override suspend fun deleteGroup(group: Group, onlyDeleteNoStarred: Boolean?) { feedDao.queryByGroupId(context.currentAccountId, group.id) .forEach { deleteFeed(it) } getGoogleReaderAPI().disableTag(group.id.dollarLast()) - super.deleteGroup(group) + super.deleteGroup(group, false) } - override suspend fun deleteFeed(feed: Feed) { + override suspend fun deleteFeed(feed: Feed, onlyDeleteNoStarred: Boolean?) { getGoogleReaderAPI().subscriptionEdit( action = "unsubscribe", destFeedId = feed.id.dollarLast() ) - super.deleteFeed(feed) + super.deleteFeed(feed, false) } /** * Google Reader API synchronous processing with object's ID to ensure idempotence * and handle foreign key relationships such as read status, starred status, etc. * - * 1. Fetch list of feeds and folders. - * 2. Fetch list of tags (it contains folders too, so you need to remove folders found in previous call to get - * tags). - * 3. Fetch ids of unread items (user can easily have 1000000 unread items so, please, add a limit on how many - * articles you sync, 25000 could be a good default, customizable limit is even better). - * 4. Fetch ids of starred items (100k starred items are possible, so, please, limit them too, 10-25k limit is a - * good default). - * 5. Fetch tagged item ids by passing s=user/-/label/TagName parameter. - * 6. Remove items that are no longer in unread/starred/tagged ids lists from your local database. - * 7. Fetch contents of items missing in database. - * 8. Mark/unmark items read/starred/tagged in you app comparing local state and ids you've got from the Google Reader API. - * Use edit-tag to sync read/starred/tagged status from your app to Google Reader API. + * 1. /reader/api/0/tag/list + * - Full list of categories/folders and tags/labels - and for InnoReader compatibility, + * including the number of unread items in each tags/labels. * + * 2. /reader/api/0/subscription/list + * - Full list of subscriptions/feeds, including their category/folder. + * - This is where you get a distinction between categories/folders and tags/labels. + * + * 3. /reader/api/0/stream/contents/user/-/state/com.google/reading-list + * (with some filters in parameter to exclude read items with xt, + * and get only the new ones with ot, cf. log below) + * - List of new unread items and their content + * - The response contains among other things the read/unread state, + * the starred/not-starred state, and the tags/labels for each entry. + * - Since this request is very expensive for the client, the network, and the server, + * it is important to use the filters appropriately. + * - If there is no new item since the last synchronisation, the response should be empty, + * and therefore efficient. + * + * 4. /reader/api/0/stream/items/ids + * (with a filter in parameter to exclude read items with xt) + * - Longer list of unread items IDs + * - This allows updating the read/unread status of the local cache of articles - assuming + * the ones not in the list are read. + * + * 5. /reader/api/0/stream/contents/user/-/state/com.google/starred + * (with some filters in parameter to exclude read items with xt, + * and get only the new ones with ot) + * - List of new unread starred items and their content + * - If there is no new unread starred item since the last synchronisation, + * the response should be empty, and therefore efficient + * - This is a bit redundant with request 3 and 6, + * but with the advantage of being able to retrieve a larger amount of unread starred items. + * + * 6. /reader/api/0/stream/contents/user/-/state/com.google/starred + * (with some other filters, which includes read starred items) + * - List of starred items (also read ones) and their content. + * + * 7. /reader/api/0/stream/items/ids + * (with a filter to get only starred ones) + * - Longer list of starred items IDs + * - This allows updating the starred/non-starred status of + * the local cache of articles - assuming the ones not in the list are not starred + * - Similar than request 4 but for the starred status. + * + * @link https://github.com/FreshRSS/FreshRSS/issues/2566#issuecomment-541317776 * @link https://github.com/bazqux/bazqux-api?tab=readme-ov-file * @link https://github.com/theoldreader/api */ - override suspend fun sync(coroutineWorker: CoroutineWorker): ListenableWorker.Result = supervisorScope { - coroutineWorker.setProgress(SyncWorker.setIsSyncing(true)) - - try { - val preTime = System.currentTimeMillis() - val accountId = context.currentAccountId - val account = accountDao.queryById(accountId)!! - val googleReaderAPI = getGoogleReaderAPI() - val groupIds = mutableSetOf() - val feedIds = mutableSetOf() - val lastUpdateAt = Calendar.getInstance().apply { - // if (account.updateAt != null) { - // time = account.updateAt!! - // add(Calendar.HOUR, -1) - // } else { - time = Date() - add(Calendar.MONTH, -1) - // } - }.time.time / 1000 - - // 1. Fetch list of feeds and folders - googleReaderAPI.getSubscriptionList() - .subscriptions.groupBy { it.categories?.first() } - .forEach { (category, feeds) -> - val groupId = accountId.spacerDollar(category?.id?.ofCategoryStreamIdToId()!!) - - // Handle folders - groupDao.insert( - Group( - id = groupId, - name = category.label!!, - accountId = accountId, - ) - ) - groupIds.add(groupId) - - // Handle feeds - feedDao.insertOrUpdate( - feeds.map { - val feedId = accountId.spacerDollar(it.id?.ofFeedStreamIdToId()!!) - Feed( - id = feedId, - name = it.title.decodeHTML() ?: context.getString(R.string.empty), - url = it.url!!, - groupId = groupId, + override suspend fun sync(coroutineWorker: CoroutineWorker): ListenableWorker.Result = + supervisorScope { + coroutineWorker.setProgress(SyncWorker.setIsSyncing(true)) + + try { + val preTime = System.currentTimeMillis() + val accountId = context.currentAccountId + val account = accountDao.queryById(accountId)!! + val googleReaderAPI = getGoogleReaderAPI() + val groupIds = mutableSetOf() + val feedIds = mutableSetOf() + val lastUpdateAt = Calendar.getInstance().apply { + if (account.updateAt != null) { + time = account.updateAt!! + add(Calendar.HOUR, -1) + } else { + time = Date(preTime) + add(Calendar.MONTH, -1) + } + }.time.time / 1000 + + // 1. Fetch tags (not supported yet) + + // 2. Fetch folder and subscription list + googleReaderAPI.getSubscriptionList() + .subscriptions.groupBy { it.categories?.first() } + .forEach { (category, feeds) -> + val groupId = + accountId.spacerDollar(category?.id?.ofCategoryStreamIdToId()!!) + + // Handle folders + groupDao.insertOrUpdate( + listOf(Group( + id = groupId, + name = category.label!!, accountId = accountId, - icon = it.iconUrl - ).also { - feedIds.add(feedId) + )) + ) + groupIds.add(groupId) + + // Handle feeds + feedDao.insertOrUpdate( + feeds.map { + val feedId = accountId.spacerDollar(it.id?.ofFeedStreamIdToId()!!) + Feed( + id = feedId, + name = it.title.decodeHTML() + ?: context.getString(R.string.empty), + url = it.url!!, + groupId = groupId, + accountId = accountId, + icon = it.iconUrl + ).also { + feedIds.add(feedId) + } } - } - ) + ) + } - // Handle empty icon for feeds - val noIconFeeds = feedDao.queryNoIcon(accountId) - Log.i("RLog", "sync: $noIconFeeds") - noIconFeeds.forEach { - it.icon = rssHelper.queryRssIconLink(it.url) + // Handle empty icon for feeds + feedDao.queryNoIcon(accountId).let { + it.forEach { feed -> + feed.icon = rssHelper.queryRssIconLink(feed.url) } - feedDao.update(*noIconFeeds.toTypedArray()) + feedDao.update(*it.toTypedArray()) } - // Remove orphaned groups and feeds - groupDao.queryAll(accountId) - .filter { it.id !in groupIds } - .forEach { super.deleteGroup(it) } - feedDao.queryAll(accountId) - .filter { it.id !in feedIds } - .forEach { super.deleteFeed(it) } - - // 3. Fetch ids of unread items - val unreadItems = googleReaderAPI.getUnreadItemIds().itemRefs - val unreadIds = unreadItems ?.map { it.id } - fetchItemsContents(unreadItems, googleReaderAPI, accountId, feedIds, unreadIds, listOf()) - - // 4. Fetch ids of starred items - val starredItems = googleReaderAPI.getStarredItemIds().itemRefs - val starredIds = starredItems?.map { it.id } - fetchItemsContents(starredItems, googleReaderAPI, accountId, feedIds, unreadIds, starredIds) - - // 5. Fetch ids of read items since last month - val readItems = googleReaderAPI.getReadItemIds(lastUpdateAt).itemRefs - - // 6. Fetch items contents for ids - fetchItemsContents(readItems, googleReaderAPI, accountId, feedIds, unreadIds, starredIds) - - // 7. Mark/unmark items read/starred/tagged in you app comparing - // local state and ids you've got from the GoogleReader - val articlesMeta = articleDao.queryMetadataAll(accountId) - for (meta: ArticleMeta in articlesMeta) { - val articleId = meta.id.dollarLast() - val shouldBeUnread = unreadIds?.contains(articleId) - val shouldBeStarred = starredIds?.contains(articleId) - if (meta.isUnread != shouldBeUnread) { - articleDao.markAsReadByArticleId(accountId, meta.id, shouldBeUnread ?: true) + // 2. Fetch latest unread item contents since last sync + var unreadIds = fetchItemIdsAndContinue { + googleReaderAPI.getUnreadItemIds(since = lastUpdateAt, continuationId = it) } - if (meta.isStarred != shouldBeStarred) { - articleDao.markAsStarredByArticleId(accountId, meta.id, shouldBeStarred ?: false) + fetchItemsContents( + itemIds = unreadIds, + googleReaderAPI = googleReaderAPI, + accountId = accountId, + feedIds = feedIds, + unreadIds = unreadIds, + starredIds = listOf()) + + // 3. Fetch all starred item contents + val starredIds = fetchItemIdsAndContinue { + googleReaderAPI.getStarredItemIds(continuationId = it) + } + fetchItemsContents( + itemIds = starredIds, + googleReaderAPI = googleReaderAPI, + accountId = accountId, + feedIds = feedIds, + unreadIds = unreadIds, + starredIds = starredIds + ) + + // 4. Mark/unmarked items read/starred (/tagged) + // Fetch all unread item id list + unreadIds = fetchItemIdsAndContinue { + googleReaderAPI.getUnreadItemIds(continuationId = it) + } + val articlesMeta = articleDao.queryMetadataAll(accountId) + for (meta: ArticleMeta in articlesMeta) { + val articleId = meta.id.dollarLast() + val shouldBeRead = !unreadIds.contains(articleId) + val shouldBeUnStarred = !starredIds.contains(articleId) + if (meta.isUnread && shouldBeRead) { + articleDao.markAsReadByArticleId(accountId, meta.id, true) + } + if (meta.isStarred && shouldBeUnStarred) { + articleDao.markAsStarredByArticleId(accountId, meta.id, false) + } } - } - Log.i("RLog", "onCompletion: ${System.currentTimeMillis() - preTime}") - accountDao.update(account.apply { - updateAt = Date() - readItems?.takeIf { it.isNotEmpty() }?.first()?.id?.let { - lastArticleId = accountId.spacerDollar(it) + // 5. Remove orphaned groups and feeds, after synchronizing the starred/un-starred + groupDao.queryAll(accountId) + .filter { it.id !in groupIds } + .forEach { super.deleteGroup(it, true) } + feedDao.queryAll(accountId) + .filter { it.id !in feedIds } + .forEach { super.deleteFeed(it, true) } + + // 6. Record the time of this synchronization + Log.i("RLog", "onCompletion: ${System.currentTimeMillis() - preTime}") + accountDao.update(account.apply { + updateAt = Date(preTime) + }) + ListenableWorker.Result.success(SyncWorker.setIsSyncing(false)) + } catch (e: Exception) { + Log.e("RLog", "On sync exception: ${e.message}", e) + withContext(mainDispatcher) { + context.showToast(e.message) } - }) - ListenableWorker.Result.success(SyncWorker.setIsSyncing(false)) - } catch (e: Exception) { - Log.e("RLog", "On sync exception: ${e.message}", e) - withContext(mainDispatcher) { - context.showToast(e.message) + ListenableWorker.Result.failure(SyncWorker.setIsSyncing(false)) } - ListenableWorker.Result.failure(SyncWorker.setIsSyncing(false)) } + + private suspend fun fetchItemIdsAndContinue(getItemIdsFunc: suspend (continuationId: String?) -> GoogleReaderDTO.ItemIds): MutableList { + var result = getItemIdsFunc(null) + val ids = result.itemRefs?.mapNotNull { it.id }?.toMutableList() ?: return mutableListOf() + while (result.continuation != null) { + result = getItemIdsFunc(result.continuation) + result.itemRefs?.mapNotNull { it.id }?.let { ids.addAll(it) } + } + return ids } private suspend fun fetchItemsContents( - readIds: List?, + itemIds: List?, googleReaderAPI: GoogleReaderAPI, accountId: Int, feedIds: MutableSet, - unreadIds: List?, + unreadIds: List?, starredIds: List?, ) { - readIds?.map { it.id!! }?.chunked(100)?.forEach { chunkedIds -> + itemIds?.chunked(100)?.forEach { chunkedIds -> articleDao.insert( *googleReaderAPI.getItemsContents(chunkedIds).items?.map { val articleId = it.id!!.ofItemStreamIdToId() @@ -335,15 +398,11 @@ class GoogleReaderRssService @Inject constructor( title = it.title.decodeHTML() ?: context.getString(R.string.empty), author = it.author, rawDescription = it.summary?.content ?: "", - shortDescription = (Readability4JExtended("", it.summary?.content ?: "") - .parse().textContent ?: "") - .take(110) - .trim(), + shortDescription = Readability + .parseToText(it.summary?.content, findArticleURL(it)).take(110), fullContent = it.summary?.content ?: "", img = rssHelper.findImg(it.summary?.content ?: ""), - link = it.canonical?.first()?.href - ?: it.alternate?.first()?.href - ?: it.origin?.htmlUrl ?: "", + link = findArticleURL(it), feedId = accountId.spacerDollar(it.origin?.streamId?.ofFeedStreamIdToId() ?: feedIds.first()), accountId = accountId, @@ -356,6 +415,10 @@ class GoogleReaderRssService @Inject constructor( } } + private fun findArticleURL(it: GoogleReaderDTO.Item) = it.canonical?.first()?.href + ?: it.alternate?.first()?.href + ?: it.origin?.htmlUrl ?: "" + override suspend fun markAsRead( groupId: String?, feedId: String?, @@ -397,7 +460,7 @@ class GoogleReaderRssService @Inject constructor( } if (markList.isNotEmpty()) googleReaderAPI.editTag( itemIds = markList, - mark = if (!isUnread) GoogleReaderAPI.Stream.READ.tag else null, + mark = if (!isUnread) GoogleReaderAPI.Stream.READ.tag else null, unmark = if (isUnread) GoogleReaderAPI.Stream.READ.tag else null, ) } diff --git a/app/src/main/java/me/ash/reader/infrastructure/html/Readability.kt b/app/src/main/java/me/ash/reader/infrastructure/html/Readability.kt new file mode 100644 index 000000000..31c37c573 --- /dev/null +++ b/app/src/main/java/me/ash/reader/infrastructure/html/Readability.kt @@ -0,0 +1,28 @@ +package me.ash.reader.infrastructure.html + +import android.util.Log +import net.dankito.readability4j.extended.Readability4JExtended +import org.jsoup.nodes.Element + +object Readability { + + fun parseToText(htmlContent: String?, uri: String?): String { + htmlContent ?: return "" + return try { + Readability4JExtended(uri ?: "", htmlContent).parse().textContent?.trim() ?: "" + } catch (e: Exception) { + Log.e("RLog", "Readability.parseToText '$uri' is error: ", e) + "" + } + } + + fun parseToElement(htmlContent: String?, uri: String?): Element? { + htmlContent ?: return null + return try { + Readability4JExtended(uri ?: "", htmlContent).parse().articleContent + } catch (e: Exception) { + Log.e("RLog", "Readability.parseToElement '$uri' is error: ", e) + null + } + } +} diff --git a/app/src/main/java/me/ash/reader/infrastructure/rss/RssHelper.kt b/app/src/main/java/me/ash/reader/infrastructure/rss/RssHelper.kt index 98c51577e..a2db73f73 100644 --- a/app/src/main/java/me/ash/reader/infrastructure/rss/RssHelper.kt +++ b/app/src/main/java/me/ash/reader/infrastructure/rss/RssHelper.kt @@ -15,10 +15,10 @@ import me.ash.reader.domain.model.article.Article import me.ash.reader.domain.model.feed.Feed import me.ash.reader.domain.repository.FeedDao import me.ash.reader.infrastructure.di.IODispatcher +import me.ash.reader.infrastructure.html.Readability import me.ash.reader.ui.ext.currentAccountId import me.ash.reader.ui.ext.decodeHTML import me.ash.reader.ui.ext.spacerDollar -import net.dankito.readability4j.extended.Readability4JExtended import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.executeAsync @@ -53,17 +53,14 @@ class RssHelper @Inject constructor( return withContext(ioDispatcher) { val response = response(okHttpClient, link) val content = response.body.string() - val readability4J = Readability4JExtended(link, content) - val articleContent = readability4J.parse().articleContent - if (articleContent == null) { - "" - } else { + val articleContent = Readability.parseToElement(content, link) + articleContent?.run { val h1Element = articleContent.selectFirst("h1") if (h1Element != null && h1Element.hasText() && h1Element.text() == title) { h1Element.remove() } articleContent.toString() - } + } ?: "" } } @@ -115,10 +112,7 @@ class RssHelper @Inject constructor( title = syndEntry.title.decodeHTML() ?: feed.name, author = syndEntry.author, rawDescription = (content ?: desc) ?: "", - shortDescription = (Readability4JExtended("", desc ?: content ?: "") - .parse().textContent ?: "") - .take(110) - .trim(), + shortDescription = Readability.parseToText(desc ?: content, syndEntry.link).take(110), fullContent = content, img = findImg((content ?: desc) ?: ""), link = syndEntry.link ?: "", diff --git a/app/src/main/java/me/ash/reader/infrastructure/rss/provider/greader/GoogleReaderAPI.kt b/app/src/main/java/me/ash/reader/infrastructure/rss/provider/greader/GoogleReaderAPI.kt index c638e35ab..6a614ae96 100644 --- a/app/src/main/java/me/ash/reader/infrastructure/rss/provider/greader/GoogleReaderAPI.kt +++ b/app/src/main/java/me/ash/reader/infrastructure/rss/provider/greader/GoogleReaderAPI.kt @@ -3,7 +3,6 @@ package me.ash.reader.infrastructure.rss.provider.greader import me.ash.reader.infrastructure.di.USER_AGENT_STRING import me.ash.reader.infrastructure.rss.provider.ProviderAPI import okhttp3.FormBody -import okhttp3.Headers.Companion.toHeaders import okhttp3.Request import okhttp3.executeAsync import java.util.concurrent.ConcurrentHashMap @@ -42,11 +41,12 @@ class GoogleReaderAPI private constructor( val clResponse = client.newCall( Request.Builder() .url("${serverUrl}accounts/ClientLogin") + .header("User-Agent", USER_AGENT_STRING) .post(FormBody.Builder() .add("output", "json") .add("Email", username) .add("Passwd", password) - .add("client", USER_AGENT_STRING) + .add("client", "ReadYou") .add("accountType", "HOSTED_OR_GOOGLE") .add("service", "reader") .build()) @@ -127,8 +127,9 @@ class GoogleReaderAPI private constructor( val response = client.newCall( Request.Builder() - .url("${serverUrl}${query}?output=json${params?.joinToString(separator = "") { "&${it.first}=${it.second}" } ?: ""}") - .header("Authorization", "GoogleLogin auth=${authData.clientLoginToken}") + .url("$serverUrl$query?output=json${params?.joinToString(separator = "") { "&${it.first}=${it.second}" } ?: ""}") + .addHeader("Authorization", "GoogleLogin auth=${authData.clientLoginToken}") + .addHeader("User-Agent", USER_AGENT_STRING) .get() .build()) .executeAsync() @@ -160,11 +161,10 @@ class GoogleReaderAPI private constructor( } val response = client.newCall( Request.Builder() - .url("${serverUrl}${query}?output=json${params?.joinToString(separator = "") { "&${it.first}=${it.second}" } ?: ""}") - .headers(mapOf( - "Authorization" to "GoogleLogin auth=${authData.clientLoginToken}", - "Content-Type" to "application/x-www-form-urlencoded", - ).toHeaders()) + .url("$serverUrl$query?output=json${params?.joinToString(separator = "") { "&${it.first}=${it.second}" } ?: ""}") + .addHeader("Authorization", "GoogleLogin auth=${authData.clientLoginToken}") + .addHeader("Content-Type", "application/x-www-form-urlencoded") + .addHeader("User-Agent", USER_AGENT_STRING) .post(FormBody.Builder() .apply { form?.forEach { add(it.first, it.second) } @@ -191,33 +191,49 @@ class GoogleReaderAPI private constructor( suspend fun getSubscriptionList(): GoogleReaderDTO.SubscriptionList = retryableGetRequest("reader/api/0/subscription/list") - suspend fun getReadItemIds(since: Long): GoogleReaderDTO.ItemIds = + suspend fun getReadItemIds( + since: Long, + limit: String? = MAXIMUM_ITEMS_LIMIT, + continuationId: String? = null, + ): GoogleReaderDTO.ItemIds = retryableGetRequest( query = "reader/api/0/stream/items/ids", - params = listOf( - Pair("s", Stream.READ.tag), - Pair("ot", since.toString()), - Pair("n", MAXIMUM_ITEMS_LIMIT), - )) + params = mutableListOf>().apply { + add(Pair("s", Stream.READ.tag)) + add(Pair("ot", since.toString())) + limit?.let { add(Pair("n", limit)) } + continuationId?.let { add(Pair("c", continuationId)) } + } + ) - suspend fun getUnreadItemIds(since: Long? = null): GoogleReaderDTO.ItemIds = + suspend fun getUnreadItemIds( + since: Long? = null, + limit: String? = MAXIMUM_ITEMS_LIMIT, + continuationId: String? = null, + ): GoogleReaderDTO.ItemIds = retryableGetRequest( query = "reader/api/0/stream/items/ids", params = mutableListOf>().apply { add(Pair("s", Stream.ALL_ITEMS.tag)) add(Pair("xt", Stream.READ.tag)) - add(Pair("n", MAXIMUM_ITEMS_LIMIT)) + limit?.let { add(Pair("n", limit)) } since?.let { add(Pair("ot", since.toString())) } + continuationId?.let { add(Pair("c", continuationId)) } } ) - suspend fun getStarredItemIds(since: Long? = null): GoogleReaderDTO.ItemIds = + suspend fun getStarredItemIds( + since: Long? = null, + limit: String? = MAXIMUM_ITEMS_LIMIT, + continuationId: String? = null, + ): GoogleReaderDTO.ItemIds = retryableGetRequest( query = "reader/api/0/stream/items/ids", params = mutableListOf>().apply { add(Pair("s", Stream.STARRED.tag)) - add(Pair("n", MAXIMUM_ITEMS_LIMIT)) + limit?.let { add(Pair("n", limit)) } since?.let { add(Pair("ot", since.toString())) } + continuationId?.let { add(Pair("c", continuationId)) } } ) diff --git a/app/src/main/java/me/ash/reader/infrastructure/rss/provider/greader/GoogleReaderDTO.kt b/app/src/main/java/me/ash/reader/infrastructure/rss/provider/greader/GoogleReaderDTO.kt index feaf06198..c233c4860 100644 --- a/app/src/main/java/me/ash/reader/infrastructure/rss/provider/greader/GoogleReaderDTO.kt +++ b/app/src/main/java/me/ash/reader/infrastructure/rss/provider/greader/GoogleReaderDTO.kt @@ -110,6 +110,7 @@ object GoogleReaderDTO { */ data class ItemIds( val itemRefs: List?, + val continuation: String?, ) /**