diff --git a/include/dfm-base/utils/infocache.h b/include/dfm-base/utils/infocache.h index 2d392c9abd..ae9f784448 100644 --- a/include/dfm-base/utils/infocache.h +++ b/include/dfm-base/utils/infocache.h @@ -30,15 +30,28 @@ class CacheWorker : public QObject public Q_SLOTS: void cacheInfo(const QUrl url, const FileInfoPointer info); void removeCaches(const QList urls); - void updateInfoTime(const QUrl url); - void dealRemoveInfo(); - void removeInfosTime(const QList urls); void disconnectWatcher(const QMap infos); private: explicit CacheWorker(QObject *parent = nullptr); }; +// 计算和统计fileinfo、filewatcher超过缓存数量和超过时间移除cache的处理 +class TimeToUpdateCache : public QObject +{ + Q_OBJECT + friend class InfoCacheController; + +public: + ~TimeToUpdateCache() override; +public Q_SLOTS: + void updateInfoTime(const QUrl url); + void dealRemoveInfo(); + void updateWatcherTime(const QList &urls, const bool add); +private: + explicit TimeToUpdateCache(QObject *parent = nullptr); +}; + class InfoCachePrivate; class InfoCache : public QObject { @@ -48,6 +61,7 @@ class InfoCache : public QObject QScopedPointer d; friend class CacheWorker; friend class InfoCacheController; + friend class TimeToUpdateCache; public: virtual ~InfoCache() override; @@ -56,7 +70,6 @@ class InfoCache : public QObject void cacheRemoveCaches(const QList &key); void cacheDisconnectWatcher(const QMap infos); void cacheUpdateInfoTime(const QUrl url); - void cacheRemoveInfosTime(const QList urls); private: explicit InfoCache(QObject *parent = nullptr); @@ -68,14 +81,18 @@ class InfoCache : public QObject void cacheInfo(const QUrl url, const FileInfoPointer info); void disconnectWatcher(const QMap infos); void removeCaches(const QList urls); - void updateSortTimeWorker(const QUrl url); + bool updateSortTimeWorker(const QUrl url); void timeRemoveCache(); void removeInfosTimeWorker(const QList urls); + void updateSortTimeWatcherWorker(const QList &urls, const bool add); private Q_SLOTS: void fileAttributeChanged(const QUrl url); void removeCache(const QUrl url); void refreshFileInfo(const QUrl &url); +private: + void addWatcherTimeInfo(const QList &urls); + void removeWatcherTimeInfo(const QList &urls); }; class InfoCacheController : public QObject @@ -85,6 +102,8 @@ class InfoCacheController : public QObject QSharedPointer thread { nullptr }; QSharedPointer worker { nullptr }; QSharedPointer removeTimer { nullptr }; // 移除缓存的 + QSharedPointer threadUpdate { nullptr }; + QSharedPointer workerUpdate { nullptr }; public: virtual ~InfoCacheController() override; static InfoCacheController &instance(); diff --git a/include/dfm-base/utils/threadcontainer.h b/include/dfm-base/utils/threadcontainer.h index ea576a6604..dd91f39558 100644 --- a/include/dfm-base/utils/threadcontainer.h +++ b/include/dfm-base/utils/threadcontainer.h @@ -374,5 +374,137 @@ class DThreadMap QMap myMap; // 当前的QMap QMutex mutable mutex; // 当前的锁 }; + +template +class DThreadHash +{ +public: + DThreadHash() + : myHash() + { + } + /*! + * \brief insert 插入一个模板类型到map + * + * \param Key 模板类引用key + * + * \param Value 模板类引用value + * + * \return + */ + inline void insert(const DKey &key, const DValue &value) + { + QMutexLocker lk(&mutex); + myHash.insert(key, value); + } + /*! + * \brief remove 从map中移除所有的模板类型 + * + * \param Key 模板类引用key + * + * \return + */ + inline void remove(const DKey &key) + { + QMutexLocker lk(&mutex); + myHash.remove(key); + } + /*! + * \brief contains map中是否包含key对应的键值对 + * + * Key 模板类引用key + * + * \return bool 是否包含key对应的键值对 + */ + inline DValue value(const DKey &key) + { + QMutexLocker lk(&mutex); + return myHash.value(key); + } + /*! + * \brief contains map中是否包含key对应的键值对 + * + * Key 模板类引用key + * + * \return bool 是否包含key对应的键值对 + */ + inline bool contains(const DKey &key) + { + QMutexLocker lk(&mutex); + return myHash.contains(key); + } + /*! + * \brief begin 当前map的开始迭代器 + * + * \param null + * + * \return QMap::iterator 当前map的开始迭代器 + */ + inline typename QMap::iterator begin() + { + QMutexLocker lk(&mutex); + return myHash.begin(); + } + /*! + * \brief begin 当前map的结束迭代器 + * + * \param null + * + * \return QMap::iterator 当前map的结束迭代器 + */ + inline typename QMap::iterator end() + { + QMutexLocker lk(&mutex); + return myHash.end(); + } + /*! + * \brief erase 去掉map中当前的迭代器 + * + * \param QMap::iterator map的迭代器 + * + * \return QMap::iterator 当前迭代器的下一个迭代器 + */ + inline typename QMap::iterator erase(typename QMap::iterator it) + { + QMutexLocker lk(&mutex); + return myHash.erase(it); + } + /*! + * \brief count map的总个数 + * + * \return int map的总个数 + */ + inline int count() + { + QMutexLocker lk(&mutex); + return myHash.count(); + } + /*! + * \brief clear 清理整个map + * + * \return + */ + inline void clear() + { + QMutexLocker lk(&mutex); + return myHash.clear(); + } + + inline QList keys() + { + QMutexLocker lk(&mutex); + return myHash.keys(); + } + + inline QHash hash() const + { + QMutexLocker lk(&mutex); + return myHash; + } + +private: + QHash myHash; // 当前的QMap + QMutex mutable mutex; // 当前的锁 +}; } #endif // THREADCONTAINER_H diff --git a/include/dfm-base/utils/watchercache.h b/include/dfm-base/utils/watchercache.h index f2d278a0f3..dc5ffddcf0 100644 --- a/include/dfm-base/utils/watchercache.h +++ b/include/dfm-base/utils/watchercache.h @@ -23,6 +23,7 @@ class WatcherCache : public QObject friend InfoCache; Q_SIGNALS: void fileDelete(const QUrl &url); + void updateWatcherTime(const QList &url, const bool add); public: static WatcherCache &instance(); @@ -31,7 +32,7 @@ class WatcherCache : public QObject QSharedPointer getCacheWatcher(const QUrl &url); void cacheWatcher(const QUrl &url, const QSharedPointer &watcher); - void removeCacheWatcher(const QUrl &url); + void removeCacheWatcher(const QUrl &url, const bool isEmit = true); void removeCacheWatcherByParent(const QUrl &parent); bool cacheDisable(const QString &scheme); void setCacheDisbale(const QString &scheme, bool disbale = true); diff --git a/src/dfm-base/utils/infocache.cpp b/src/dfm-base/utils/infocache.cpp index ce3bc3f76b..60a25262ab 100644 --- a/src/dfm-base/utils/infocache.cpp +++ b/src/dfm-base/utils/infocache.cpp @@ -11,6 +11,8 @@ // cache file total count static constexpr int kCacheFileinfoCount = 20000; +// cache file watcher total count +static constexpr int kCacheFileWatcherCount = 5000; // rotation training time static constexpr int kRotationTrainingTime = (60 * 1000); // remove cache time limit @@ -192,17 +194,18 @@ void InfoCache::cacheInfo(const QUrl url, const FileInfoPointer info) * * \return */ -void InfoCache::updateSortTimeWorker(const QUrl url) +bool InfoCache::updateSortTimeWorker(const QUrl url) { Q_D(InfoCache); if (d->cacheWorkerStoped) - return; + return false; auto time = QDateTime::currentMSecsSinceEpoch(); auto key = QString::number(time) + QString("-") + url.toString(); - if (d->urlTimeSortMap.contains(url)) - d->timeToUrlMap.remove(d->urlTimeSortMap.value(url)); + if (d->urlTimeSortHash.contains(url)) + d->timeToUrlMap.remove(d->urlTimeSortHash.value(url)); d->timeToUrlMap.insert(key, url); - d->urlTimeSortMap.insert(url, key); + d->urlTimeSortHash.insert(url, key); + return d->timeToUrlMap.count() > kCacheFileinfoCount; } void InfoCache::stop() @@ -241,8 +244,6 @@ void InfoCache::removeCaches(const QList urls) // 断开监视器监视 if (infos.size() > 0) emit cacheDisconnectWatcher(infos); - // 移除时间队列 - emit cacheRemoveInfosTime(urls); // 设置读取主缓存和插入到主缓存中 d->status = kCacheMain; @@ -293,6 +294,49 @@ void InfoCache::refreshFileInfo(const QUrl &url) if (info) info->updateAttributes(); } + +void InfoCache::addWatcherTimeInfo(const QList &urls) +{ + if (d->cacheWorkerStoped) + return; + + auto time = QDateTime::currentMSecsSinceEpoch(); + for (const auto &url : urls) { + if (d->cacheWorkerStoped) + return; + auto key = QString::number(time) + QString("-") + url.toString(); + if (d->urlTimeSortWatcherHash.contains(url)) + d->timeToUrlWatcherMap.remove(d->urlTimeSortWatcherHash.value(url)); + d->timeToUrlWatcherMap.insert(key, url); + d->urlTimeSortWatcherHash.insert(url, key); + } + + if (d->timeToUrlWatcherMap.count() <= kCacheFileWatcherCount) + return; + + // 超出限制移除先进入的watcher + auto it = d->timeToUrlWatcherMap.begin(); + while (d->urlTimeSortWatcherHash.size() > kCacheFileWatcherCount + && it != d->timeToUrlWatcherMap.end()) { + if (d->cacheWorkerStoped) + return; + auto url = it.value(); + d->urlTimeSortWatcherHash.remove(url); + WatcherCache::instance().removeCacheWatcher(url, false); + it = d->timeToUrlWatcherMap.erase(it); + } +} + +void InfoCache::removeWatcherTimeInfo(const QList &urls) +{ + for (const auto &url : urls) { + if (d->cacheWorkerStoped) + return; + if (d->urlTimeSortWatcherHash.contains(url)) { + d->timeToUrlWatcherMap.remove(d->urlTimeSortWatcherHash.take(url)); + } + } +} /*! * \brief timeRemoveCache 定时检查哪些fileinfo要移除 * @@ -302,7 +346,7 @@ void InfoCache::timeRemoveCache() { Q_D(InfoCache); // 取出哪些url的时间超出了u - qint64 delCount = d->urlTimeSortMap.size() < kCacheFileinfoCount ? 0 : d->urlTimeSortMap.size() - kCacheFileinfoCount; + qint64 delCount = d->urlTimeSortHash.size() < kCacheFileinfoCount ? 0 : d->urlTimeSortHash.size() - kCacheFileinfoCount; QList delList; foreach (const auto time, d->timeToUrlMap.uniqueKeys()) { if (d->cacheWorkerStoped) @@ -319,6 +363,11 @@ void InfoCache::timeRemoveCache() delList.append(d->timeToUrlMap.values(time)); } + for (auto url : delList) { + if (d->urlTimeSortHash.contains(url)) { + d->timeToUrlMap.remove(d->urlTimeSortHash.take(url)); + } + } // 发送异步消息 告诉移除线程创建移除线程移除,考虑是否是使用线程一直还是使用临时线程(使用临时线程) if (delList.size() > 0 && !d->cacheWorkerStoped) emit cacheRemoveCaches(delList); @@ -327,12 +376,23 @@ void InfoCache::timeRemoveCache() void InfoCache::removeInfosTimeWorker(const QList urls) { for (auto url : urls) { - if (d->urlTimeSortMap.contains(url)) { - d->timeToUrlMap.remove(d->urlTimeSortMap.take(url)); + if (d->urlTimeSortHash.contains(url)) { + d->timeToUrlMap.remove(d->urlTimeSortHash.take(url)); } } } +void InfoCache::updateSortTimeWatcherWorker(const QList &urls, const bool add) +{ + Q_ASSERT(qApp->thread() != QThread::currentThread()); + Q_D(InfoCache); + + if (add) + return addWatcherTimeInfo(urls); + + removeInfosTimeWorker(urls); +} + void InfoCache::fileAttributeChanged(const QUrl url) { refreshFileInfo(url); @@ -359,24 +419,6 @@ void CacheWorker::removeCaches(const QList urls) InfoCache::instance().removeCaches(urls); } -void CacheWorker::updateInfoTime(const QUrl url) -{ - Q_ASSERT(qApp->thread() != QThread::currentThread()); - InfoCache::instance().updateSortTimeWorker(url); -} - -void CacheWorker::dealRemoveInfo() -{ - Q_ASSERT(qApp->thread() != QThread::currentThread()); - InfoCache::instance().timeRemoveCache(); -} - -void CacheWorker::removeInfosTime(const QList urls) -{ - Q_ASSERT(qApp->thread() != QThread::currentThread()); - InfoCache::instance().removeInfosTimeWorker(urls); -} - void CacheWorker::disconnectWatcher(const QMap infos) { Q_ASSERT(qApp->thread() != QThread::currentThread()); @@ -386,9 +428,11 @@ void CacheWorker::disconnectWatcher(const QMap infos) InfoCacheController::~InfoCacheController() { removeTimer->stop(); - thread->quit(); InfoCache::instance().stop(); + thread->quit(); thread->wait(); + threadUpdate->quit(); + threadUpdate->wait(); } InfoCacheController &InfoCacheController::instance() @@ -414,6 +458,8 @@ FileInfoPointer InfoCacheController::getCacheInfo(const QUrl &url) InfoCacheController::InfoCacheController(QObject *parent) : QObject(parent), thread(new QThread), worker(new CacheWorker), removeTimer(new QTimer) + , threadUpdate(new QThread) + , workerUpdate(new TimeToUpdateCache) { init(); } @@ -421,19 +467,54 @@ InfoCacheController::InfoCacheController(QObject *parent) void InfoCacheController::init() { removeTimer->moveToThread(qApp->thread()); - connect(removeTimer.data(), &QTimer::timeout, worker.data(), &CacheWorker::dealRemoveInfo, Qt::QueuedConnection); + connect(removeTimer.data(), &QTimer::timeout, workerUpdate.data(), + &TimeToUpdateCache::dealRemoveInfo, Qt::QueuedConnection); + connect(&InfoCache::instance(), &InfoCache::cacheUpdateInfoTime, workerUpdate.data(), + &TimeToUpdateCache::updateInfoTime, Qt::QueuedConnection); connect(this, &InfoCacheController::cacheFileInfo, worker.data(), &CacheWorker::cacheInfo, Qt::QueuedConnection); connect(this, &InfoCacheController::removeCacheFileInfo, worker.data(), &CacheWorker::removeCaches, Qt::QueuedConnection); connect(&InfoCache::instance(), &InfoCache::cacheRemoveCaches, worker.data(), &CacheWorker::removeCaches, Qt::QueuedConnection); - connect(&InfoCache::instance(), &InfoCache::cacheRemoveInfosTime, worker.data(), &CacheWorker::removeInfosTime, Qt::QueuedConnection); connect(&InfoCache::instance(), &InfoCache::cacheDisconnectWatcher, worker.data(), &CacheWorker::disconnectWatcher, Qt::QueuedConnection); + connect(&WatcherCache::instance(), &WatcherCache::updateWatcherTime, + workerUpdate.data(), &TimeToUpdateCache::updateWatcherTime, Qt::QueuedConnection); worker->moveToThread(thread.data()); thread->start(); + workerUpdate->moveToThread(threadUpdate.data()); + threadUpdate->start(); removeTimer->setInterval(kRotationTrainingTime); removeTimer->start(); } +TimeToUpdateCache::~TimeToUpdateCache() +{ + +} + +void TimeToUpdateCache::updateInfoTime(const QUrl url) +{ + Q_ASSERT(qApp->thread() != QThread::currentThread()); + if (InfoCache::instance().updateSortTimeWorker(url)) + InfoCache::instance().timeRemoveCache(); +} + +void TimeToUpdateCache::dealRemoveInfo() +{ + Q_ASSERT(qApp->thread() != QThread::currentThread()); + InfoCache::instance().timeRemoveCache(); +} + +void TimeToUpdateCache::updateWatcherTime(const QList &urls, const bool add) +{ + Q_ASSERT(qApp->thread() != QThread::currentThread()); + InfoCache::instance().updateSortTimeWatcherWorker(urls, add); +} + +TimeToUpdateCache::TimeToUpdateCache(QObject *parent) : QObject (parent) +{ + +} + } //开线程移除缓存和同步缓存操作 diff --git a/src/dfm-base/utils/private/infocache_p.h b/src/dfm-base/utils/private/infocache_p.h index 11bd54d138..2557cd725f 100644 --- a/src/dfm-base/utils/private/infocache_p.h +++ b/src/dfm-base/utils/private/infocache_p.h @@ -31,9 +31,12 @@ class InfoCachePrivate QReadWriteLock copyLock; // 时间排序url,利用map的有序性,来处理时间到了要移除的url - QMap urlTimeSortMap; + QHash urlTimeSortHash; QMap timeToUrlMap; + // 时间排序url,利用map的有序性,来处理时间到了要移除的url + QHash urlTimeSortWatcherHash; + QMap timeToUrlWatcherMap; std::atomic_bool cacheWorkerStoped { false }; public: diff --git a/src/dfm-base/utils/private/watchercache_p.h b/src/dfm-base/utils/private/watchercache_p.h index b413694e41..3103fe743e 100644 --- a/src/dfm-base/utils/private/watchercache_p.h +++ b/src/dfm-base/utils/private/watchercache_p.h @@ -16,7 +16,7 @@ class WatcherCachePrivate { friend class WatcherCache; WatcherCache *const q; - DThreadMap> watchers; + DThreadHash> watchers; DThreadList disableCahceSchemes; public: diff --git a/src/dfm-base/utils/watchercache.cpp b/src/dfm-base/utils/watchercache.cpp index 7810c2edbd..5427a068bf 100644 --- a/src/dfm-base/utils/watchercache.cpp +++ b/src/dfm-base/utils/watchercache.cpp @@ -56,7 +56,7 @@ WatcherCache &WatcherCache::instance() QSharedPointer WatcherCache::getCacheWatcher(const QUrl &url) { Q_D(WatcherCache); - + emit updateWatcherTime({url}, true); return d->watchers.value(url); } /*! @@ -75,8 +75,11 @@ QSharedPointer WatcherCache::getCacheWatcher(const QUrl &ur void WatcherCache::cacheWatcher(const QUrl &url, const QSharedPointer &watcher) { Q_D(WatcherCache); + if (watcher.isNull()) + return; connect(watcher.data(), &AbstractFileWatcher::fileDeleted, this, &WatcherCache::fileDelete); d->watchers.insert(url, watcher); + emit updateWatcherTime({url}, true); } /*! * \brief removCacheWatcher 根据Url移除当前缓存的watcher @@ -91,11 +94,13 @@ void WatcherCache::cacheWatcher(const QUrl &url, const QSharedPointerwatchers.remove(url); + if (isEmit) + emit updateWatcherTime({url}, false); } void WatcherCache::removeCacheWatcherByParent(const QUrl &parent) @@ -105,10 +110,15 @@ void WatcherCache::removeCacheWatcherByParent(const QUrl &parent) Q_D(WatcherCache); auto keys = d->watchers.keys(); + QList removeUrls; for (const auto &url : keys) { - if (url.scheme() == parent.scheme() && url.path().startsWith(parent.path())) + if (url.scheme() == parent.scheme() && url.path().startsWith(parent.path())) { d->watchers.remove(url); + removeUrls.append(url); + } } + + emit updateWatcherTime(removeUrls, false); } bool WatcherCache::cacheDisable(const QString &scheme)