Merge branch 'main' of github.com:janheinrichmerker/trec-biogen

trec_biogen/jobs/index_pubmed_full_text.py

@@ -138,6 +138,7 @@ def get_full_text_batch(input_batch: DataBatch) -> DataBatch:
         get_full_text_batch,
         batch_size=10,
         num_cpus=0.25,
+        concurrency=6,
     )
     if dry_run:
         data = data.filter(lambda x: x["doc"]["full_text"] is not None)