fix: make number of concurrent downloads configurable

go-shiori · Aug 28, 2024 · 4210c93 · 4210c93
1 parent 9a113af
commit 4210c93
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 4 deletions.
diff --git a/docs/Configuration.md b/docs/Configuration.md
@@ -47,9 +47,10 @@ Most configuration can be set directly using environment variables or flags. The
 
 The `StorageConfig` struct contains settings related to storage.
 
-| Environment variable | Default       | Required | Description                             |
-| -------------------- | ------------- | -------- | --------------------------------------- |
-| `SHIORI_DIR`         | (current dir) | No       | Directory where Shiori stores its data. |
+| Environment variable | Default           | Required | Description                              |
+| -------------------- | ----------------- | -------- | ---------------------------------------- |
+| `SHIORI_DIR`         | (current dir)     | No       | Directory where Shiori stores its data.  |
+| `SHIORI_MAX_PAR_DL`  | (num logicl CPU)  | No       | Number of parallel articles to download. |
 
 #### The data Directory
 

diff --git a/internal/cmd/update.go b/internal/cmd/update.go
@@ -132,7 +132,7 @@ func updateHandler(cmd *cobra.Command, args []string) {
 		chDone := make(chan struct{})
 		chProblem := make(chan int, 10)
 		chMessage := make(chan interface{}, 10)
-		semaphore := make(chan struct{}, 10)
+		semaphore := make(chan struct{}, cfg.Storage.MaxParDl)
 
 		cInfo.Println("Downloading article(s)...")
 

diff --git a/internal/config/config.go b/internal/config/config.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"time"
 
@@ -86,6 +87,7 @@ type DatabaseConfig struct {
 
 type StorageConfig struct {
 	DataDir string `env:"DIR"` // Using DIR to be backwards compatible with the old config
+	MaxParDl int `env:"MAX_PAR_DL,default=-1"`
 }
 
 type Config struct {
@@ -109,6 +111,10 @@ func (c Config) SetDefaults(logger *logrus.Logger, portableMode bool) {
 		}
 	}
 
+	if c.Storage.MaxParDl == -1 {
+		c.Storage.MaxParDl = runtime.NumCPU()
+	}
+
 	// Set default database url if not set
 	if c.Database.DBMS == "" && c.Database.URL == "" {
 		c.Database.URL = fmt.Sprintf("sqlite:///%s", filepath.Join(c.Storage.DataDir, "shiori.db"))
@@ -124,6 +130,7 @@ func (c *Config) DebugConfiguration(logger *logrus.Logger) {
 	logger.Debugf(" SHIORI_DATABASE_URL: %s", c.Database.URL)
 	logger.Debugf(" SHIORI_DBMS: %s", c.Database.DBMS)
 	logger.Debugf(" SHIORI_DIR: %s", c.Storage.DataDir)
+	logger.Debugf(" SHIORI_MAX_PAR_DL: %d", c.Storage.MaxParDl)
 	logger.Debugf(" SHIORI_HTTP_ENABLED: %t", c.Http.Enabled)
 	logger.Debugf(" SHIORI_HTTP_PORT: %d", c.Http.Port)
 	logger.Debugf(" SHIORI_HTTP_ADDRESS: %s", c.Http.Address)