fix: make number of concurrent downloads configurable

go-shiori · Sep 14, 2024 · 555936b · 555936b
1 parent 2bcb890
commit 555936b
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 12 deletions.
diff --git a/docs/Configuration.md b/docs/Configuration.md
@@ -47,9 +47,10 @@ Most configuration can be set directly using environment variables or flags. The
 
 The `StorageConfig` struct contains settings related to storage.
 
-| Environment variable | Default       | Required | Description                             |
-| -------------------- | ------------- | -------- | --------------------------------------- |
-| `SHIORI_DIR`         | (current dir) | No       | Directory where Shiori stores its data. |
+| Environment variable | Default           | Required | Description                              |
+| -------------------- | ----------------- | -------- | ---------------------------------------- |
+| `SHIORI_DIR`         | (current dir)     | No       | Directory where Shiori stores its data.  |
+| `SHIORI_MAX_PAR_DL`  | (num logicl CPU)  | No       | Number of parallel articles to download. |
 
 #### The data Directory
 

diff --git a/internal/cmd/check.go b/internal/cmd/check.go
@@ -29,7 +29,7 @@ func checkCmd() *cobra.Command {
 }
 
 func checkHandler(cmd *cobra.Command, args []string) {
-	_, deps := initShiori(cmd.Context(), cmd)
+	cfg, deps := initShiori(cmd.Context(), cmd)
 
 	// Parse flags
 	skipConfirm, _ := cmd.Flags().GetBool("yes")
@@ -69,9 +69,9 @@ func checkHandler(cmd *cobra.Command, args []string) {
 
 	wg := sync.WaitGroup{}
 	chDone := make(chan struct{})
-	chProblem := make(chan int, 10)
-	chMessage := make(chan interface{}, 10)
-	semaphore := make(chan struct{}, 10)
+	chProblem := make(chan int, cfg.Storage.MaxParDl)
+	chMessage := make(chan interface{}, cfg.Storage.MaxParDl)
+	semaphore := make(chan struct{}, cfg.Storage.MaxParDl)
 
 	for i, book := range bookmarks {
 		wg.Add(1)

diff --git a/internal/cmd/update.go b/internal/cmd/update.go
@@ -130,9 +130,9 @@ func updateHandler(cmd *cobra.Command, args []string) {
 		mx := sync.RWMutex{}
 		wg := sync.WaitGroup{}
 		chDone := make(chan struct{})
-		chProblem := make(chan int, 10)
-		chMessage := make(chan interface{}, 10)
-		semaphore := make(chan struct{}, 10)
+		chProblem := make(chan int, cfg.Storage.MaxParDl)
+		chMessage := make(chan interface{}, cfg.Storage.MaxParDl)
+		semaphore := make(chan struct{}, cfg.Storage.MaxParDl)
 
 		cInfo.Println("Downloading article(s)...")
 

diff --git a/internal/config/config.go b/internal/config/config.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"time"
 
@@ -86,6 +87,7 @@ type DatabaseConfig struct {
 
 type StorageConfig struct {
 	DataDir string `env:"DIR"` // Using DIR to be backwards compatible with the old config
+	MaxParDl int `env:"MAX_PAR_DL,default=-1"`
 }
 
 type Config struct {
@@ -109,6 +111,10 @@ func (c Config) SetDefaults(logger *logrus.Logger, portableMode bool) {
 		}
 	}
 
+	if c.Storage.MaxParDl == -1 {
+		c.Storage.MaxParDl = runtime.NumCPU()
+	}
+
 	// Set default database url if not set
 	if c.Database.DBMS == "" && c.Database.URL == "" {
 		c.Database.URL = fmt.Sprintf("sqlite:///%s", filepath.Join(c.Storage.DataDir, "shiori.db"))
@@ -124,6 +130,7 @@ func (c *Config) DebugConfiguration(logger *logrus.Logger) {
 	logger.Debugf(" SHIORI_DATABASE_URL: %s", c.Database.URL)
 	logger.Debugf(" SHIORI_DBMS: %s", c.Database.DBMS)
 	logger.Debugf(" SHIORI_DIR: %s", c.Storage.DataDir)
+	logger.Debugf(" SHIORI_MAX_PAR_DL: %d", c.Storage.MaxParDl)
 	logger.Debugf(" SHIORI_HTTP_ENABLED: %t", c.Http.Enabled)
 	logger.Debugf(" SHIORI_HTTP_PORT: %d", c.Http.Port)
 	logger.Debugf(" SHIORI_HTTP_ADDRESS: %s", c.Http.Address)

diff --git a/internal/http/routes/api/v1/bookmarks.go b/internal/http/routes/api/v1/bookmarks.go
@@ -162,12 +162,13 @@ func (r *BookmarksAPIRoutes) updateCache(c *gin.Context) {
 	}
 	// TODO: limit request to 20
 
+	cfg := r.deps.Config
 	// Fetch data from internet
 	mx := sync.RWMutex{}
 	wg := sync.WaitGroup{}
 	chDone := make(chan struct{})
-	chProblem := make(chan int, 10)
-	semaphore := make(chan struct{}, 10)
+	chProblem := make(chan int, cfg.Storage.MaxParDl)
+	semaphore := make(chan struct{}, cfg.Storage.MaxParDl)
 
 	for i, book := range bookmarks {
 		wg.Add(1)