diff --git a/docs/Configuration.md b/docs/Configuration.md index fdaf4e7e9..a2868da9f 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -47,9 +47,10 @@ Most configuration can be set directly using environment variables or flags. The The `StorageConfig` struct contains settings related to storage. -| Environment variable | Default | Required | Description | -| -------------------- | ------------- | -------- | --------------------------------------- | -| `SHIORI_DIR` | (current dir) | No | Directory where Shiori stores its data. | +| Environment variable | Default | Required | Description | +| -------------------- | ----------------- | -------- | ---------------------------------------- | +| `SHIORI_DIR` | (current dir) | No | Directory where Shiori stores its data. | +| `SHIORI_MAX_PAR_DL` | (num logicl CPU) | No | Number of parallel articles to download. | #### The data Directory diff --git a/internal/cmd/check.go b/internal/cmd/check.go index 0f41f3f35..61e271e17 100644 --- a/internal/cmd/check.go +++ b/internal/cmd/check.go @@ -29,7 +29,7 @@ func checkCmd() *cobra.Command { } func checkHandler(cmd *cobra.Command, args []string) { - _, deps := initShiori(cmd.Context(), cmd) + cfg, deps := initShiori(cmd.Context(), cmd) // Parse flags skipConfirm, _ := cmd.Flags().GetBool("yes") @@ -69,9 +69,9 @@ func checkHandler(cmd *cobra.Command, args []string) { wg := sync.WaitGroup{} chDone := make(chan struct{}) - chProblem := make(chan int, 10) - chMessage := make(chan interface{}, 10) - semaphore := make(chan struct{}, 10) + chProblem := make(chan int, cfg.Storage.MaxParDl) + chMessage := make(chan interface{}, cfg.Storage.MaxParDl) + semaphore := make(chan struct{}, cfg.Storage.MaxParDl) for i, book := range bookmarks { wg.Add(1) diff --git a/internal/cmd/update.go b/internal/cmd/update.go index 918ea2a1a..e53600df6 100644 --- a/internal/cmd/update.go +++ b/internal/cmd/update.go @@ -130,9 +130,9 @@ func updateHandler(cmd *cobra.Command, args []string) { mx := sync.RWMutex{} wg := sync.WaitGroup{} chDone := make(chan struct{}) - chProblem := make(chan int, 10) - chMessage := make(chan interface{}, 10) - semaphore := make(chan struct{}, 10) + chProblem := make(chan int, cfg.Storage.MaxParDl) + chMessage := make(chan interface{}, cfg.Storage.MaxParDl) + semaphore := make(chan struct{}, cfg.Storage.MaxParDl) cInfo.Println("Downloading article(s)...") diff --git a/internal/config/config.go b/internal/config/config.go index 390b642d6..3506261f6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strings" "time" @@ -86,6 +87,7 @@ type DatabaseConfig struct { type StorageConfig struct { DataDir string `env:"DIR"` // Using DIR to be backwards compatible with the old config + MaxParDl int `env:"MAX_PAR_DL,default=-1"` } type Config struct { @@ -109,6 +111,10 @@ func (c Config) SetDefaults(logger *logrus.Logger, portableMode bool) { } } + if c.Storage.MaxParDl == -1 { + c.Storage.MaxParDl = runtime.NumCPU() + } + // Set default database url if not set if c.Database.DBMS == "" && c.Database.URL == "" { c.Database.URL = fmt.Sprintf("sqlite:///%s", filepath.Join(c.Storage.DataDir, "shiori.db")) @@ -124,6 +130,7 @@ func (c *Config) DebugConfiguration(logger *logrus.Logger) { logger.Debugf(" SHIORI_DATABASE_URL: %s", c.Database.URL) logger.Debugf(" SHIORI_DBMS: %s", c.Database.DBMS) logger.Debugf(" SHIORI_DIR: %s", c.Storage.DataDir) + logger.Debugf(" SHIORI_MAX_PAR_DL: %d", c.Storage.MaxParDl) logger.Debugf(" SHIORI_HTTP_ENABLED: %t", c.Http.Enabled) logger.Debugf(" SHIORI_HTTP_PORT: %d", c.Http.Port) logger.Debugf(" SHIORI_HTTP_ADDRESS: %s", c.Http.Address) diff --git a/internal/http/routes/api/v1/bookmarks.go b/internal/http/routes/api/v1/bookmarks.go index a95ec538b..657419c75 100644 --- a/internal/http/routes/api/v1/bookmarks.go +++ b/internal/http/routes/api/v1/bookmarks.go @@ -162,12 +162,13 @@ func (r *BookmarksAPIRoutes) updateCache(c *gin.Context) { } // TODO: limit request to 20 + cfg := r.deps.Config // Fetch data from internet mx := sync.RWMutex{} wg := sync.WaitGroup{} chDone := make(chan struct{}) - chProblem := make(chan int, 10) - semaphore := make(chan struct{}, 10) + chProblem := make(chan int, cfg.Storage.MaxParDl) + semaphore := make(chan struct{}, cfg.Storage.MaxParDl) for i, book := range bookmarks { wg.Add(1)