[network] Implement a smart cache size

src/app/options/qgsoptions.cpp

@@ -482,14 +482,14 @@ QgsOptions::QgsOptions( QWidget *parent, Qt::WindowFlags fl, const QList<QgsOpti
   connect( mRemoveUrlPushButton, &QAbstractButton::clicked, this, &QgsOptions::removeNoProxyUrl );
 
   // cache settings
-  mCacheDirectory->setText( mSettings->value( QStringLiteral( "cache/directory" ) ).toString() );
+  mCacheDirectory->setText( QgsSettingsRegistryCore::settingsNetworkCacheDirectory->value() );
   mCacheDirectory->setPlaceholderText( QStandardPaths::writableLocation( QStandardPaths::CacheLocation ) );
   mCacheSize->setMinimum( 0 );
   mCacheSize->setMaximum( std::numeric_limits<int>::max() );
-  mCacheSize->setSingleStep( 1024 );
-  qint64 cacheSize = mSettings->value( QStringLiteral( "cache/size" ), 256 * 1024 * 1024 ).toLongLong();
-  mCacheSize->setValue( static_cast<int>( cacheSize / 1024 ) );
-  mCacheSize->setClearValue( 50 * 1024 );
+  mCacheSize->setSingleStep( 50 );
+  qint64 cacheSize = QgsSettingsRegistryCore::settingsNetworkCacheSize->value();
+  mCacheSize->setValue( static_cast<int>( cacheSize / 1024 / 1024 ) );
+  mCacheSize->setClearValue( 0 );
   connect( mBrowseCacheDirectory, &QAbstractButton::clicked, this, &QgsOptions::browseCacheDirectory );
   connect( mClearCache, &QAbstractButton::clicked, this, &QgsOptions::clearCache );
 
@@ -1579,11 +1579,11 @@ void QgsOptions::saveOptions()
   mSettings->setValue( QStringLiteral( "proxy/proxyType" ), mProxyTypeComboBox->currentText() );
 
   if ( !mCacheDirectory->text().isEmpty() )
-    mSettings->setValue( QStringLiteral( "cache/directory" ), mCacheDirectory->text() );
+    QgsSettingsRegistryCore::settingsNetworkCacheDirectory->setValue( mCacheDirectory->text() );
   else
-    mSettings->remove( QStringLiteral( "cache/directory" ) );
+    QgsSettingsRegistryCore::settingsNetworkCacheDirectory->remove();
 
-  mSettings->setValue( QStringLiteral( "cache/size" ), QVariant::fromValue( mCacheSize->value() * 1024LL ) );
+  QgsSettingsRegistryCore::settingsNetworkCacheSize->setValue( mCacheSize->value() * 1024LL * 1024LL );
 
   //url with no proxy at all
   QStringList noProxyUrls;
@@ -2260,7 +2260,7 @@ void QgsOptions::clearCache()
   QgsNetworkAccessManager::instance()->cache()->clear();
 
   // Clear WFS XSD cache used by OGR GMLAS driver
-  QString cacheDirectory = mSettings->value( QStringLiteral( "cache/directory" ) ).toString();
+  QString cacheDirectory = QgsSettingsRegistryCore::settingsNetworkCacheDirectory->value();
   if ( cacheDirectory.isEmpty() )
     cacheDirectory = QStandardPaths::writableLocation( QStandardPaths::CacheLocation );
   if ( !cacheDirectory.endsWith( QDir::separator() ) )

src/core/network/qgsnetworkaccessmanager.cpp

@@ -24,6 +24,7 @@
 #include "qgsapplication.h"
 #include "qgsmessagelog.h"
 #include "qgssettings.h"
+#include "qgssettingsregistrycore.h"
 #include "qgslogger.h"
 #include "qgis.h"
 #include "qgsnetworkdiskcache.h"
@@ -749,12 +750,13 @@ void QgsNetworkAccessManager::setupDefaultProxyAndCache( Qt::ConnectionType conn
   if ( !newcache )
     newcache = new QgsNetworkDiskCache( this );
 
-  QString cacheDirectory = settings.value( QStringLiteral( "cache/directory" ) ).toString();
+  QString cacheDirectory = QgsSettingsRegistryCore::settingsNetworkCacheDirectory->value();
   if ( cacheDirectory.isEmpty() )
     cacheDirectory = QStandardPaths::writableLocation( QStandardPaths::CacheLocation );
-  const qint64 cacheSize = settings.value( QStringLiteral( "cache/size" ), 256 * 1024 * 1024 ).toLongLong();
   newcache->setCacheDirectory( cacheDirectory );
+  qint64 cacheSize = QgsSettingsRegistryCore::settingsNetworkCacheSize->value();
   newcache->setMaximumCacheSize( cacheSize );
+
   QgsDebugMsgLevel( QStringLiteral( "cacheDirectory: %1" ).arg( newcache->cacheDirectory() ), 4 );
   QgsDebugMsgLevel( QStringLiteral( "maximumCacheSize: %1" ).arg( newcache->maximumCacheSize() ), 4 );
 

src/core/network/qgsnetworkdiskcache.cpp

@@ -18,6 +18,9 @@
 
 #include "qgsnetworkdiskcache.h"
 
+#include <QStorageInfo>
+#include <mutex>
+
 ///@cond PRIVATE
 ExpirableNetworkDiskCache QgsNetworkDiskCache::sDiskCache;
 ///@endcond
@@ -49,6 +52,13 @@ qint64 QgsNetworkDiskCache::maximumCacheSize() const
 void QgsNetworkDiskCache::setMaximumCacheSize( qint64 size )
 {
   const QMutexLocker lock( &sDiskCacheMutex );
+
+  if ( size == 0 )
+  {
+    // Calculate maximum cache size based on available free space
+    size = smartCacheSize( sDiskCache.cacheDirectory() );
+  }
+
   sDiskCache.setMaximumCacheSize( size );
 }
 
@@ -111,3 +121,76 @@ void QgsNetworkDiskCache::clear()
   const QMutexLocker lock( &sDiskCacheMutex );
   return sDiskCache.clear();
 }
+
+qint64 QgsNetworkDiskCache::smartCacheSize( const QString &cacheDir )
+{
+  static qint64 cacheSize = 0;
+  static std::once_flag initialized;
+  std::call_once( initialized, [ = ]
+  {
+    std::function<qint64( const QString & )> dirSize;
+    dirSize = [&dirSize]( const QString & dirPath ) -> qint64
+    {
+      qint64 size = 0;
+      QDir dir( dirPath );
+
+      const QStringList filePaths = dir.entryList( QDir::Files | QDir::System | QDir::Hidden );
+      for ( const QString &filePath : filePaths )
+      {
+        QFileInfo fi( dir, filePath );
+        size += fi.size();
+      }
+
+      const QStringList childDirPaths = dir.entryList( QDir::Dirs | QDir::NoDotAndDotDot | QDir::System | QDir::Hidden | QDir::NoSymLinks );
+      for ( const QString &childDirPath : childDirPaths )
+      {
+        size += dirSize( dirPath + QDir::separator() + childDirPath );
+      }
+
+      return size;
+    };
+
+    qint64 bytesFree;
+    QStorageInfo storageInfo( cacheDir );
+    bytesFree = storageInfo.bytesFree() + dirSize( cacheDir );
+
+    // NOLINTBEGIN(bugprone-narrowing-conversions)
+    // Logic taken from Firefox's smart cache size handling
+    qint64 available10MB = bytesFree / 1024 / ( 1024LL * 10 );
+    qint64 cacheSize10MB = 0;
+    if ( available10MB > 2500 )
+    {
+      // Cap the cache size to 1GB
+      cacheSize10MB = 100;
+    }
+    else
+    {
+      if ( available10MB > 700 )
+      {
+        // Add 2.5% of the free space above 7GB
+        cacheSize10MB += ( available10MB - 700 ) * 0.025;
+        available10MB = 700;
+      }
+      if ( available10MB > 50 )
+      {
+        // Add 7.5% of free space between 500MB to 7GB
+        cacheSize10MB += ( available10MB - 50 ) * 0.075;
+        available10MB = 50;
+      }
+
+#if defined( Q_OS_ANDROID )
+      // On Android, smaller/older devices may have very little storage
+
+      // Add 16% of free space up to 500 MB
+      cacheSize10MB += std::max( 2LL, static_cast<qint64>( available10MB * 0.16 ) );
+#else
+      // Add 30% of free space up to 500 MB
+      cacheSize10MB += std::max( 5LL, static_cast<qint64>( available10MB * 0.30 ) );
+#endif
+    }
+    cacheSize = cacheSize10MB * 10 * 1024 * 1024;
+    // NOLINTEND(bugprone-narrowing-conversions)
+  } );
+
+  return cacheSize;
+}

src/core/network/qgsnetworkdiskcache.h

@@ -20,6 +20,8 @@
 
 #define SIP_NO_FILE
 
+#include "qgis_core.h"
+
 #include <QNetworkDiskCache>
 #include <QMutex>
 
@@ -45,7 +47,7 @@ class ExpirableNetworkDiskCache : public QNetworkDiskCache
  *
  * \note not available in Python bindings
  */
-class QgsNetworkDiskCache : public QNetworkDiskCache
+class CORE_EXPORT QgsNetworkDiskCache : public QNetworkDiskCache
 {
     Q_OBJECT
 
@@ -87,6 +89,12 @@ class QgsNetworkDiskCache : public QNetworkDiskCache
     //! \see QNetworkDiskCache::fileMetaData()
     QNetworkCacheMetaData fileMetaData( const QString &fileName ) const;
 
+    /**
+     * Returns a smart cache size, in bytes, based on available free space
+     * \since QGIS 3.40
+     */
+    static qint64 smartCacheSize( const QString &path );
+
   public slots:
     //! \see QNetworkDiskCache::clear()
     void clear() override;

src/core/network/qgsrangerequestcache.cpp

@@ -16,6 +16,7 @@
  ***************************************************************************/
 
 #include "qgsrangerequestcache.h"
+#include "qgsnetworkdiskcache.h"
 
 #include <QtDebug>
 #include <QFile>
@@ -75,6 +76,12 @@ bool QgsRangeRequestCache::setCacheDirectory( const QString &path )
 
 void QgsRangeRequestCache::setCacheSize( qint64 cacheSize )
 {
+  if ( cacheSize == 0 )
+  {
+    // Calculate maximum cache size based on available free space
+    cacheSize = QgsNetworkDiskCache::smartCacheSize( mCacheDir );
+  }
+
   mMaxDataSize = cacheSize;
   expire();
 }

src/core/qgstiledownloadmanager.cpp

@@ -21,6 +21,8 @@
 #include "qgsnetworkaccessmanager.h"
 #include "qgsrangerequestcache.h"
 #include "qgssettings.h"
+#include "qgssettingsregistrycore.h"
+#include "qgssettingsentryimpl.h"
 
 #include <QElapsedTimer>
 #include <QNetworkReply>
@@ -190,17 +192,16 @@ QgsTileDownloadManager::QgsTileDownloadManager()
   mRangesCache.reset( new QgsRangeRequestCache );
 
   const QgsSettings settings;
-  QString cacheDirectory = settings.value( QStringLiteral( "cache/directory" ) ).toString();
+  QString cacheDirectory = QgsSettingsRegistryCore::settingsNetworkCacheDirectory->value();
   if ( cacheDirectory.isEmpty() )
     cacheDirectory = QStandardPaths::writableLocation( QStandardPaths::CacheLocation );
   if ( !cacheDirectory.endsWith( QDir::separator() ) )
   {
     cacheDirectory.push_back( QDir::separator() );
   }
   cacheDirectory += QLatin1String( "http-ranges" );
-  const qint64 cacheSize = settings.value( QStringLiteral( "cache/size" ), 256 * 1024 * 1024 ).toLongLong();
-
   mRangesCache->setCacheDirectory( cacheDirectory );
+  qint64 cacheSize = QgsSettingsRegistryCore::settingsNetworkCacheSize->value();
   mRangesCache->setCacheSize( cacheSize );
 }
 

src/core/settings/qgssettingsregistrycore.cpp

@@ -115,6 +115,10 @@ const QgsSettingsEntryInteger *QgsSettingsRegistryCore::settingsLayerParallelLoa
 
 const QgsSettingsEntryBool *QgsSettingsRegistryCore::settingsLayerParallelLoading = new QgsSettingsEntryBool( QStringLiteral( "provider-parallel-loading" ), QgsSettingsTree::sTreeCore, true, QStringLiteral( "Load layers in parallel (only available for some providers (GDAL and PostgreSQL)" ), Qgis::SettingsOption() );
 
+const QgsSettingsEntryString *QgsSettingsRegistryCore::settingsNetworkCacheDirectory = new QgsSettingsEntryString( QStringLiteral( "directory" ), QgsSettingsTree::sTreeNetworkCache, QString(), QStringLiteral( "Network disk cache directory" ) );
+
+const QgsSettingsEntryInteger64 *QgsSettingsRegistryCore::settingsNetworkCacheSize = new QgsSettingsEntryInteger64( QStringLiteral( "size-bytes" ), QgsSettingsTree::sTreeNetworkCache, 0, QStringLiteral( "Network disk cache size in bytes (0 = automatic size)" ) );
+
 QgsSettingsRegistryCore::QgsSettingsRegistryCore()
   : QgsSettingsRegistry()
 {

src/core/settings/qgssettingsregistrycore.h

@@ -26,6 +26,7 @@ class QgsSettingsEntryBool;
 class QgsSettingsEntryColor;
 class QgsSettingsEntryDouble;
 class QgsSettingsEntryInteger;
+class QgsSettingsEntryInteger64;
 class QgsSettingsEntryString;
 class QgsSettingsEntryStringList;
 template<class T> class QgsSettingsEntryEnumFlag;
@@ -165,6 +166,12 @@ class CORE_EXPORT QgsSettingsRegistryCore : public QgsSettingsRegistry
     //! Settings entry whether layer are loading in parallel
     static const QgsSettingsEntryBool *settingsLayerParallelLoading;
 
+    //! Settings entry network cache directory
+    static const QgsSettingsEntryString *settingsNetworkCacheDirectory;
+
+    //! Settings entry network cache directory
+    static const QgsSettingsEntryInteger64 *settingsNetworkCacheSize;
+
   private:
     friend class QgsApplication;
 

src/core/settings/qgssettingstree.h

@@ -63,6 +63,7 @@ class CORE_EXPORT QgsSettingsTree
     static inline QgsSettingsTreeNode *sTreeWms = treeRoot()->createChildNode( QStringLiteral( "wms" ) );
     static inline QgsSettingsTreeNode *sTreeMeasure = treeRoot()->createChildNode( QStringLiteral( "measure" ) );
     static inline QgsSettingsTreeNode *sTreeAnnotations = treeRoot()->createChildNode( QStringLiteral( "annotations" ) );
+    static inline QgsSettingsTreeNode *sTreeNetworkCache = treeRoot()->createChildNode( QStringLiteral( "cache" ) );
 
 #endif
 

src/server/qgsserver.cpp

@@ -31,6 +31,7 @@
 #include "qgslogger.h"
 #include "qgsmapserviceexception.h"
 #include "qgsnetworkaccessmanager.h"
+#include "qgsnetworkdiskcache.h"
 #include "qgsserverlogger.h"
 #include "qgsserverrequest.h"
 #include "qgsfilterresponsedecorator.h"
@@ -88,9 +89,9 @@ void QgsServer::setupNetworkAccessManager()
   const QSettings settings;
   QgsNetworkAccessManager *nam = QgsNetworkAccessManager::instance();
   QNetworkDiskCache *cache = new QNetworkDiskCache( nullptr );
-  const qint64 cacheSize = sSettings()->cacheSize();
   const QString cacheDirectory = sSettings()->cacheDirectory();
   cache->setCacheDirectory( cacheDirectory );
+  qint64 cacheSize = sSettings()->cacheSize();
   cache->setMaximumCacheSize( cacheSize );
   QgsMessageLog::logMessage( QStringLiteral( "cacheDirectory: %1" ).arg( cache->cacheDirectory() ), QStringLiteral( "Server" ), Qgis::MessageLevel::Info );
   QgsMessageLog::logMessage( QStringLiteral( "maximumCacheSize: %1" ).arg( cache->maximumCacheSize() ), QStringLiteral( "Server" ), Qgis::MessageLevel::Info );

src/server/qgsserversettings.cpp

@@ -123,10 +123,10 @@ void QgsServerSettings::initSettings()
   // cache size
   const Setting sCacheSize = { QgsServerSettingsEnv::QGIS_SERVER_CACHE_SIZE,
                                QgsServerSettingsEnv::DEFAULT_VALUE,
-                               QStringLiteral( "Specify the cache size" ),
-                               QStringLiteral( "/cache/size" ),
+                               QStringLiteral( "Specify the cache size (0 = automatic size)" ),
+                               QStringLiteral( "/cache/size-bytes" ),
                                QMetaType::Type::LongLong,
-                               QVariant( 256 * 1024 * 1024 ),
+                               0,
                                QVariant()
                              };
   mSettings[ sCacheSize.envVar ] = sCacheSize;

src/ui/qgsoptionsbase.ui

@@ -4862,7 +4862,7 @@ The bigger the number, the faster zooming with the mouse wheel will be.</string>
                       <item row="4" column="0">
                        <widget class="QLabel" name="label_11">
                         <property name="text">
-                         <string>Size [KiB]</string>
+                         <string>Size</string>
                         </property>
                        </widget>
                       </item>
@@ -4877,7 +4877,32 @@ The bigger the number, the faster zooming with the mouse wheel will be.</string>
                        <widget class="QLineEdit" name="mCacheDirectory"/>
                       </item>
                       <item row="4" column="1">
-                       <widget class="QgsSpinBox" name="mCacheSize"/>
+                       <widget class="QgsSpinBox" name="mCacheSize">
+                        <property name="toolTip">
+                         <string>Specify the cache size in megabytes. Clear the value to enable smart cache size, which sets the maximum cache size based on available space.</string>
+                        </property>
+                        <property name="suffix">
+                         <string> MB</string>
+                        </property>
+                        <property name="minimum">
+                         <number>0</number>
+                        </property>
+                        <property name="maximum">
+                         <number>10000</number>
+                        </property>
+                        <property name="singleStep">
+                         <number>100</number>
+                        </property>
+                        <property name="clearValue">
+                         <number>0</number>
+                        </property>
+                        <property name="showClearButton" stdset="0">
+                         <bool>true</bool>
+                        </property>
+                        <property name="specialValueText">
+                         <string>Smart cache size</string>
+                        </property>
+                       </widget>
                       </item>
                       <item row="0" column="2">
                        <widget class="QToolButton" name="mBrowseCacheDirectory">

tests/src/python/test_qgsserver_settings.py

@@ -20,7 +20,7 @@
 
 from utilities import unitTestDataPath
 
-DEFAULT_CACHE_SIZE = 256 * 1024 * 1024
+DEFAULT_CACHE_SIZE = 0  # Smart cache size
 
 
 class TestQgsServerSettings(unittest.TestCase):

tests/testdata/qgis_server_settings/conf0.ini

@@ -1,6 +1,6 @@
 [cache]
 directory=
-size=@Variant(\0\0\0\x81\0\0\0\0\x3 \0\0)
+size-bytes=@Variant(\0\0\0\x81\0\0\0\0\x3 \0\0)
 
 [qgis]
 parallel_rendering=true

tests/testdata/qgis_server_settings/conf1.ini

@@ -1,6 +1,6 @@
 [cache]
 directory=/tmp/mycache
-size=@Variant(\0\0\0\x81\0\0\0\0\x3 \0\0)
+size-bytes=@Variant(\0\0\0\x81\0\0\0\0\x3 \0\0)
 
 [qgis]
 parallel_rendering=false