Set server side cursors (compat. 1/4+)

alephdata · Jul 21, 2023 · 4874aa8 · 4874aa8
1 parent 5edf94e
commit 4874aa8
Showing 1 changed file with 2 additions and 5 deletions.
diff --git a/followthemoney/mapping/sql.py b/followthemoney/mapping/sql.py
@@ -55,10 +55,7 @@ def __init__(self, query: "QueryMapping", data: Dict[str, Any]) -> None:
         if database is None:
             raise InvalidMapping("No database in SQL mapping!")
         self.database_uri = cast(str, os.path.expandvars(database))
-        kwargs = {}
-        if self.database_uri.lower().startswith("postgres"):
-            kwargs["server_side_cursors"] = True
-        self.engine = create_engine(self.database_uri, poolclass=NullPool, **kwargs)  # type: ignore
+        self.engine = create_engine(self.database_uri, poolclass=NullPool)  # type: ignore
         self.meta = MetaData()
 
         tables = keys_values(data, "table", "tables")
@@ -104,7 +101,7 @@ def records(self) -> Generator[Record, None, None]:
         q = self.compose_query()
         log.info("Query: %s", q)
         with self.engine.connect() as conn:
-            rp = conn.execute(q)
+            rp = conn.execution_options(stream_results=True).execute(q)
             while True:
                 rows = rp.fetchmany(size=DATA_PAGE)
                 if not len(rows):