chore: Better abstract secondary storage - add channel stream for sec…

…ondary insertions
Layr-Labs · Oct 12, 2024 · a598791 · a598791
1 parent ab6b939
commit a598791
Showing 1 changed file with 6 additions and 4 deletions.
diff --git a/server/load_store.go b/server/load_store.go
@@ -124,10 +124,12 @@ func LoadStoreRouter(ctx context.Context, cfg CLIConfig, log log.Logger) (store.
 		return nil, err
 	}
 
-	if secondary.Enabled() { // only spin-up go routine if secondary storage is enabled
-		// NOTE: the number of workers is set to 1 for now but it should be possible to increase
-		log.Debug("Starting secondary stream processing routine")
-		go secondary.StreamProcess(ctx)
+	if secondary.Enabled() { // only spin-up go routines if secondary storage is enabled
+		log.Debug("Starting secondary stream processing routines")
+
+		for i := 0; i < 10; i++ {
+			go secondary.StreamProcess(ctx)
+		}
 	}
 
 	log.Info("Creating storage router", "eigenda backend type", eigenDA != nil, "s3 backend type", s3Store != nil)