From ce7fb9d94a5d26ad5e5a3296a589feb5d84d308d Mon Sep 17 00:00:00 2001 From: ingalls Date: Tue, 5 Sep 2023 16:48:44 +0100 Subject: [PATCH 1/2] Render GeoJSON --- openaddr/process_one.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/openaddr/process_one.py b/openaddr/process_one.py index ed59c8b6..10cfd2d3 100644 --- a/openaddr/process_one.py +++ b/openaddr/process_one.py @@ -56,7 +56,7 @@ def boolstr(value): raise ValueError(repr(value)) def process(source, destination, layer, layersource, - do_geojsonld, do_preview, do_mbtiles, do_pmtiles, + do_preview, do_mbtiles, do_pmtiles, mapbox_key=None, extras=dict()): ''' Process a single source and destination, return path to JSON state file. @@ -163,13 +163,12 @@ def process(source, destination, layer, layersource, else: _L.info('Preview image in {}'.format(preview_path)) - if do_geojsonld: - geojsonld_path = render_geojsonld(conform_result.path, temp_dir) + geojsonld_path = render_geojsonld(conform_result.path, temp_dir) - if not geojsonld_path: - _L.warning('No GeoJSON-LD generated') - else: - _L.info('GeoJSON-LD file in {}'.format(geojsonld_path)) + if not geojsonld_path: + _L.warning('No GeoJSON-LD generated') + else: + _L.info('GeoJSON-LD file in {}'.format(geojsonld_path)) if do_mbtiles: if not geojsonld_path: @@ -475,13 +474,13 @@ def write_state(source, layer, data_source_name, skipped, destination, log_handl parser.add_argument('-ls', '--layersource', help='Source within a given layer to pull from', dest='layersource', default='') -parser.add_argument('--skip-geojsonld', help="Don't generate GeoJSON-LD alongside CSV output", - action='store_const', dest='generate_geojsonld', - const=False, default=False) - parser.add_argument('--skip-preview', help="Don't render a map preview", action='store_const', dest='render_preview', - const=False, default=False) + const=False, default=True) + +parser.add_argument('--render-preview', help="Render a map preview", + action='store_const', dest='render_preview', + const=True, default=True) parser.add_argument('--mapbox-key', dest='mapbox_key', help='Mapbox API Key. See: https://mapbox.com/') @@ -524,18 +523,12 @@ def main(): _L.error('Mapbox key is required to generate preview') return 1 - # Can't generate preview without GeoJSON-LD - if args.render_preview and not args.generate_geojsonld: - _L.error('GeoJSON-LD is required to generate preview') - return 1 - # Allow CSV files with very long fields csv.field_size_limit(sys.maxsize) try: processed_path = process(args.source, args.destination, args.layer, args.layersource, - args.generate_geojsonld, args.render_preview, args.render_preview, args.render_preview, From df48dbb362ff599b50dea52eeecd6b7dc4e99ffb Mon Sep 17 00:00:00 2001 From: ingalls Date: Tue, 5 Sep 2023 17:06:08 +0100 Subject: [PATCH 2/2] Update process() args --- openaddr/tests/__init__.py | 86 +++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index d7d36999..b89fa43f 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -352,7 +352,7 @@ def test_single_ac_local(self): preview_ren.side_effect = touch_second_arg_file slippymap_gen.side_effect = touch_first_arg_file render_geojsonld.side_effect = return_path_in_second_arg_dir - state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, True, mapbox_key='mapbox-XXXX') + state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, mapbox_key='mapbox-XXXX') self.assertTrue(slippymap_gen.mock_calls[0][1][0].endswith('.mbtiles')) self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) @@ -402,7 +402,7 @@ def test_single_ac(self): preview_ren.side_effect = touch_second_arg_file slippymap_gen.side_effect = touch_first_arg_file render_geojsonld.side_effect = return_path_in_second_arg_dir - state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, True, mapbox_key='mapbox-XXXX') + state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, mapbox_key='mapbox-XXXX') self.assertTrue(slippymap_gen.mock_calls[0][1][0].endswith('.mbtiles')) self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) @@ -451,7 +451,7 @@ def test_single_ac_mixedcase(self): preview_ren.side_effect = touch_second_arg_file slippymap_gen.side_effect = touch_first_arg_file render_geojsonld.side_effect = return_path_in_second_arg_dir - state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, True, mapbox_key='mapbox-XXXX') + state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, mapbox_key='mapbox-XXXX') self.assertTrue(slippymap_gen.mock_calls[0][1][0].endswith('.mbtiles')) self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) @@ -497,7 +497,7 @@ def test_single_sf(self): preview_ren.side_effect = touch_second_arg_file slippymap_gen.side_effect = touch_first_arg_file render_geojsonld.side_effect = return_path_in_second_arg_dir - state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, True, mapbox_key='mapbox-XXXX') + state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, mapbox_key='mapbox-XXXX') self.assertTrue(slippymap_gen.mock_calls[0][1][0].endswith('.mbtiles')) self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) @@ -547,7 +547,7 @@ def test_single_car(self): preview_ren.side_effect = touch_second_arg_file slippymap_gen.side_effect = touch_first_arg_file render_geojsonld.side_effect = return_path_in_second_arg_dir - state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, True, mapbox_key='mapbox-XXXX') + state_path = process_one.process(source, self.testdir, "addresses", "default", True, True, True, mapbox_key='mapbox-XXXX') self.assertTrue(slippymap_gen.mock_calls[0][1][0].endswith('.mbtiles')) self.assertTrue(slippymap_gen.mock_calls[0][1][1].endswith('.geojson')) @@ -583,7 +583,7 @@ def test_single_car_cached(self): source = join(self.src_dir, 'us-ca-carson-cached.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -603,7 +603,7 @@ def test_single_car_old_cached(self): source = join(self.src_dir, 'us-ca-carson-old-cached.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -623,7 +623,7 @@ def test_single_tx_runnels(self): source = join(self.src_dir, 'us/tx/runnels.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -642,7 +642,7 @@ def test_single_oak(self): source = join(self.src_dir, 'us-ca-oakland.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -661,7 +661,7 @@ def test_single_oak_skip(self): source = join(self.src_dir, 'us-ca-oakland-skip.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -680,7 +680,7 @@ def test_single_berk(self): source = join(self.src_dir, 'us-ca-berkeley.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -699,7 +699,7 @@ def test_single_berk_404(self): source = join(self.src_dir, 'us-ca-berkeley-404.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -716,7 +716,7 @@ def test_single_berk_apn(self): source = join(self.src_dir, 'us-ca-berkeley-apn.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -749,7 +749,7 @@ def test_single_pl_ds(self): source = join(self.src_dir, 'pl-dolnoslaskie.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -765,7 +765,7 @@ def test_single_pl_l(self): source = join(self.src_dir, 'pl-lodzkie.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -795,7 +795,7 @@ def test_single_jp_fukushima2(self): source = join(self.src_dir, 'jp-fukushima2.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -825,7 +825,7 @@ def test_single_utah(self): source = join(self.src_dir, 'us-ut.json') with mock.patch('openaddr.util.request_ftp_file', new=self.response_content_ftp): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -839,7 +839,7 @@ def test_single_iceland(self): source = join(self.src_dir, 'iceland.json') with mock.patch('openaddr.util.request_ftp_file', new=self.response_content_ftp): - state_path = process_one.process(source, self.testdir, "addresses", "default", True, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -872,7 +872,7 @@ def test_single_fr_paris(self): source = join(self.src_dir, 'fr-paris.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -895,7 +895,7 @@ def test_single_fr_lareunion(self): raise Exception('Could not find a usable fr/la-réunion.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -909,7 +909,7 @@ def test_single_va_statewide(self): source = join(self.src_dir, 'us/va/statewide.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -924,7 +924,7 @@ def test_single_oh_trumbull(self): source = join(self.src_dir, 'us/oh/trumbull.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -939,7 +939,7 @@ def test_single_ks_brown(self): source = join(self.src_dir, 'us/ks/brown_county.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -954,7 +954,7 @@ def test_single_pa_lancaster(self): source = join(self.src_dir, 'us/pa/lancaster.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -982,7 +982,7 @@ def test_single_ua_kharkiv(self): source = join(self.src_dir, 'ua-63-city_of_kharkiv.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -996,7 +996,7 @@ def test_single_pa_bucks(self): source = join(self.src_dir, 'us/pa/bucks.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1024,7 +1024,7 @@ def test_single_nm_washington(self): source = join(self.src_dir, 'us/nm/washington.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1054,7 +1054,7 @@ def test_single_tx_waco(self): with HTTMock(self.response_content): ofs = csv.field_size_limit() csv.field_size_limit(1) - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) csv.field_size_limit(ofs) with open(state_path) as file: @@ -1068,7 +1068,7 @@ def test_single_tx_waco(self): with HTTMock(self.response_content): ofs = csv.field_size_limit() csv.field_size_limit(sys.maxsize) - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) csv.field_size_limit(ofs) with open(state_path) as file: @@ -1100,7 +1100,7 @@ def test_single_wy_park(self): source = join(self.src_dir, 'us-wy-park.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1127,7 +1127,7 @@ def test_single_ny_orange(self): source = join(self.src_dir, 'us-ny-orange.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1154,7 +1154,7 @@ def test_single_de_berlin(self): source = join(self.src_dir, 'de/berlin.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1179,7 +1179,7 @@ def test_single_us_or_portland(self): source = join(self.src_dir, 'us/or/portland.json') with mock.patch('openaddr.util.request_ftp_file', new=self.response_content_ftp): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1208,7 +1208,7 @@ def test_single_nl_countrywide(self): source = join(self.src_dir, 'nl/countrywide.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1233,7 +1233,7 @@ def test_single_be_wa_brussels(self): source = join(self.src_dir, 'be/wa/brussels-fr.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1274,7 +1274,7 @@ def test_single_it_52_statewide(self): source = join(self.src_dir, 'it-52-statewide.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1300,7 +1300,7 @@ def test_single_us_nj_statewide(self): source = join(self.src_dir, 'us/nj/statewide.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1326,7 +1326,7 @@ def test_single_cz_countrywide(self): source = join(self.src_dir, 'cz-countrywide-bad-tests.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1341,7 +1341,7 @@ def test_single_or_curry(self): source = join(self.src_dir, 'us-or-curry.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1356,7 +1356,7 @@ def test_single_mi_grand_traverse(self): source = join(self.src_dir, 'us-mi-grand_traverse.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1371,7 +1371,7 @@ def test_single_lake_man_gdb(self): source = join(self.src_dir, 'lake-man-gdb.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1403,7 +1403,7 @@ def test_single_lake_man_gdb_nested(self): source = join(self.src_dir, 'lake-man-gdb-nested.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file))) @@ -1435,7 +1435,7 @@ def test_single_lake_man_gdb_nested_nodir(self): source = join(self.src_dir, 'lake-man-gdb-nested-nodir.json') with HTTMock(self.response_content): - state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False, False) + state_path = process_one.process(source, self.testdir, "addresses", "default", False, False, False) with open(state_path) as file: state = dict(zip(*json.load(file)))