updated engine to pyogrio to accelerate saving GPKG/GeoJSON, added on…

…e error catching

erde/io/geojson.py

@@ -53,7 +53,7 @@ def write_df(cls, df, path, path_match, *args, driver=None, **kwargs):
 		if os.path.exists(path):
 			os.unlink(path)
 
-		df.to_file(path, driver=driver, encoding='utf-8')
+		df.to_file(path, driver=driver, encoding='utf-8', engine='pyogrio')
 
 
 driver = GeoJsonDriver

erde/io/gpkg.py

@@ -12,6 +12,7 @@
 FIONA_DRIVER = 'GPKG'
 PATH_REGEXP = r'^(?P<file_path>(?:.*/)?(?P<file_name>(?:.*/)?(?P<file_own_name>.*)\.(?P<extension>gpkg)))(?:\:(?P<layer_name>[a-z0-9_-]+))?$'
 
+
 class GpkgReader(BaseReader):
 	fiona_driver = FIONA_DRIVER
 	source_regexp = PATH_REGEXP
@@ -227,7 +228,7 @@ def write_df(cls, df, path, path_match, *args, **kwargs):
 			if layer_name in fiona.listlayers(filepath):
 				fiona.remove(filepath, FIONA_DRIVER, layer_name)
 
-		df.to_file(filepath, driver=FIONA_DRIVER, encoding='utf-8', layer=layer_name)
+		df.to_file(filepath, driver=FIONA_DRIVER, encoding='utf-8', layer=layer_name, engine='pyogrio')
 
 
 driver = GpkgDriver

erde/op/table.py

@@ -1,4 +1,4 @@
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ProcessPoolExecutor, as_completed
 from erde import CONFIG, autocli, write_stream, utils
 from functools import partial
 from itertools import product
@@ -174,16 +174,20 @@ def table_route(sources, destinations, router, max_table_size=2_000, threads=10,
 
 	_route_partial = partial(_route_chunk, host_url=host_url, annotations=annotations, extra_params=extra_params)
 
-	with tqdm(total=total_rows * total_cols, desc='Table routing', disable=(not pbar)) as t, ThreadPoolExecutor(max_workers=threads) as tpe:
+	with tqdm(total=total_rows * total_cols, desc='Table routing', disable=(not pbar)) as t, ProcessPoolExecutor(max_workers=threads) as ppe:
 		combos = list(product(range(0, total_rows, rows), range(0, total_cols, cols)))
 		slices = ((sources[s:s + rows], destinations[d:d + cols], s, d) for s, d in combos)
 
 		# process/thread/an instance of executor
-		for df in tpe.map(_route_partial, slices):
-			df['source'] = df['source'].map(sources_indices)
-			df['destination'] = df['destination'].map(destinations_indices)
-			yield df
-			t.update(len(df))
+		jobs = {ppe.submit(_route_partial, s): s for s in slices}
+		for j in as_completed(jobs):
+			try:
+				df = j.result()
+			except Exception as exc:
+				print(f'generated an exception: {exc}')
+			else:
+				t.update(len(df))
+				yield df
 
 
 @autocli

examples/8_satimg/test_areas.geojson

@@ -4,6 +4,7 @@
 "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } },
 "features": [
 { "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 83.070385, 54.826626 ], [ 83.114869, 54.822552 ], [ 83.121383, 54.841848 ], [ 83.106493, 54.849350 ], [ 83.097559, 54.857172 ], [ 83.086578, 54.855457 ], [ 83.071874, 54.846457 ], [ 83.070385, 54.826626 ] ] ] } },
-{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 83.102957, 54.815715 ], [ 83.107936, 54.812444 ], [ 83.143276, 54.820407 ], [ 83.119231, 54.831243 ], [ 83.108971, 54.822197 ], [ 83.107645, 54.818738 ], [ 83.102957, 54.815715 ] ] ] } }
+{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 83.102957, 54.815715 ], [ 83.107936, 54.812444 ], [ 83.143276, 54.820407 ], [ 83.119231, 54.831243 ], [ 83.108971, 54.822197 ], [ 83.107645, 54.818738 ], [ 83.102957, 54.815715 ] ] ] } },
+{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 82.588183390529238, 55.051069607475547 ], [ 82.49716850409672, 55.090393319673474 ], [ 82.516711577870566, 55.15999409601222 ], [ 82.769654789857881, 55.200485433724566 ], [ 82.842243349589339, 55.063222522118274 ], [ 82.588183390529238, 55.051069607475547 ] ] ] } }
 ]
 }

requirements.txt

@@ -3,8 +3,9 @@ geopandas
 ipdb
 matplotlib
 polyline
+pygeos --no-binary=pygeos
+pyogrio
 requests
+shapely --no-binary=shapely
 tqdm
 yaargh
-shapely --no-binary=shapely
-pygeos --no-binary=pygeos