From 06714880aec0056eb7fc3d6373cb5a551c41c89d Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Mon, 7 Nov 2016 10:58:13 -0600 Subject: [PATCH 1/2] optional geo support --- census/__init__.py | 9 +- census/geo.py | 230 ++++++++++++++++++++++++++++++++++++ census/tests/test_census.py | 5 +- setup.py | 1 + 4 files changed, 242 insertions(+), 3 deletions(-) create mode 100644 census/geo.py diff --git a/census/__init__.py b/census/__init__.py index d439cb3..ab3434b 100644 --- a/census/__init__.py +++ b/census/__init__.py @@ -1,2 +1,9 @@ -from census.core import (Census, ALL, CensusException, +from census.core import (ALL, CensusException, UnsupportedYearException, __version__) +try: + import pyesridump, shapely +except ImportError: + from census.core import Census +else: + from census.geo import Census + diff --git a/census/geo.py b/census/geo.py new file mode 100644 index 0000000..1cf4021 --- /dev/null +++ b/census/geo.py @@ -0,0 +1,230 @@ +import json +import sys +import logging + +import esridump +import shapely.geometry +import shapely.geos + +import .core as core +from .core import supported_years + +try: # Python 2.7+ + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + +logging.getLogger(__name__).addHandler(NullHandler()) + +GEO_URLS = { + 'tracts' : { + 1990 : 'https://gis.uspatial.umn.edu/arcgis/rest/services/nhgis/Census_Tracts_1910_2014/MapServer/8', + 2000 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/Census2010/tigerWMS_Census2000/MapServer/6', + 2010 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_Census2010/MapServer/14', + 2011 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_Census2010/MapServer/14', + 2012 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_Census2010/MapServer/14', + 2013 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_ACS2013/MapServer/8', + 2014 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_ACS2014/MapServer/8', + 2015 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_ACS2015/MapServer/8', + 2016 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_ACS2015/MapServer/8'}, + 'block groups' : { + 2000 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/Census2010/tigerWMS_Census2000/MapServer/8', + 2010 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_Census2010/MapServer/16', + 2011 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_Census2010/MapServer/16', + 2012 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_Census2010/MapServer/16', + 2013 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_ACS2013/MapServer/10', + 2014 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_ACS2014/MapServer/10', + 2015 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_ACS2015/MapServer/10', + 2016 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_ACS2015/MapServer/10'}, + 'blocks' : { + 2000 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/Census2010/tigerWMS_Census2000/MapServer/10', + 2010 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_Current/MapServer/12'}, + 'incorporated places' : { + 1990 : 'https://gis.uspatial.umn.edu/arcgis/rest/services/nhgis/Places_1980_2014/MapServer/1', + 2000 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/Census2010/tigerWMS_Census2000/MapServer/24', + 2010 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_Census2010/MapServer/34', + 2011 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_Census2010/MapServer/34', + 2012 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_Census2010/MapServer/34', + 2013 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_ACS2013/MapServer/26', + 2014 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_ACS2014/MapServer/26', + 2015 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_ACS2015/MapServer/26', + 2016 : 'https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/tigerWMS_ACS2016/MapServer/26'} +} + + +class AreaFilter(object): + def __init__(self, geojson_geometry, sub_geography_url): + self.geo = shapely.geometry.shape(geojson_geometry) + + geo_query_args = {'geometry': ','.join(str(x) for x in self.geo.bounds), + 'geometryType': 'esriGeometryEnvelope', + 'spatialRel': 'esriSpatialRelEnvelopeIntersects', + 'inSR' : '4326', + 'geometryPrecision' : 9, + 'orderByFields': 'OID'} + self.area_dumper = esridump.EsriDumper(sub_geography_url, + extra_query_args = geo_query_args) + + def __iter__(self): + for area in self.area_dumper: + area_geo = shapely.geometry.shape(area['geometry']) + if self.geo.intersects(area_geo): + try: + intersection = self.geo.intersection(area_geo) + except shapely.geos.TopologicalError: + intersection = self.geo.buffer(0).intersection(area_geo.buffer(0)) + if intersection.area/area_geo.area > 0.1: + yield area + +class GeoClient(core.Client): + @supported_years(2014, 2013, 2012, 2011, 2010, 2000) + def geo_tract(self, fields, geojson_geometry, year=None): + if year is None: + year = self.default_year + + filtered_tracts = AreaFilter(geojson_geometry, + GEO_URLS['tracts'][self.default_year]) + + for tract in filtered_tracts: + context = {'state' : tract['properties']['STATE'], + 'county' : tract['properties']['COUNTY']} + within = 'state:{state} county:{county}'.format(**context) + + tract_id = tract['properties']['TRACT'] + result = self.get(fields, + {'for': 'tract:{}'.format(tract_id), + 'in' : within}, year) + + if result: + result, = result + else: + result = {} + + yield tract, result + + @supported_years(2014, 2013, 2012, 2011, 2010, 2000) + def geo_blockgroup(self, fields, geojson_geometry, year=None): + if year is None: + year = self.default_year + + filtered_block_groups = AreaFilter(geojson_geometry, + GEO_URLS['block groups'][year]) + + for block_group in filtered_block_groups: + context = {'state' : block_group['properties']['STATE'], + 'county' : block_group['properties']['COUNTY'], + 'tract' : block_group['properties']['TRACT']} + within = 'state:{state} county:{county} tract:{tract}'.format(**context) + + block_group_id = block_group['properties']['BLKGRP'] + + result = self.get(fields, + {'for': 'block group:{}'.format(block_group_id), + 'in' : within}, year) + + if result: + result, = result + else: + result = {} + + yield block_group, result + + + def _state_place_area(self, method, fields, state, place, year=None, return_geometry=False): + if year is None: + year = self.default_year + + search_query = "PLACE='{}' AND STATE={}".format(place, state) + place_dumper = esridump.EsriDumper(GEO_URLS['incorporated places'][year], + extra_query_args = {'where' : search_query, + 'orderByFields': 'OID'}) + + place = next(iter(place_dumper)) + logging.info(place['properties']['NAME']) + place_geojson = place['geometry'] + + areas = method(fields, place_geojson, year) + + features = [] + for i, (feature, result) in enumerate(areas): + if return_geometry: + feature['properties'].update(result) + features.append(feature) + else: + features.append(result) + if i % 100 == 0: + logging.info('{} features'.format(i)) + + if return_geometry: + return {'type': "FeatureCollection", 'features': features} + else: + return features + + +class ACS5Client(core.ACS5Client, GeoClient): + + @supported_years(2014, 2013, 2012, 2011, 2010) + def state_place_tract(self, *args, **kwargs): + return self._state_place_area(self.geo_tract, *args, **kwargs) + + @supported_years(2014, 2013, 2012, 2011, 2010) + def state_place_blockgroup(self, *args, **kwargs): + return self._state_place_area(self.geo_blockgroup, *args, **kwargs) + +class SF1Client(core.SF1Client, GeoClient): + @supported_years(2010, 2000, 1990) + def state_place_tract(self, *args, **kwargs): + return self._state_place_area(self.geo_tract, *args, **kwargs) + + @supported_years(2010, 2000) + def state_place_blockgroup(self, *args, **kwargs): + return self._state_place_area(self.geo_blockgroup, *args, **kwargs) + + @supported_years(2010, 2000) + def state_place_block(self, *args, **kwargs): + return self._state_place_area(self.geo_block, *args, **kwargs) + + @supported_years(2010, 2000) + def geo_block(self, fields, geojson_geometry, year): + if year is None: + year = self.default_year + + filtered_blocks = AreaFilter(geojson_geometry, + GEO_URLS['blocks'][year]) + + for block in filtered_blocks: + context = {'state' : block['properties']['STATE'], + 'county' : block['properties']['COUNTY'], + 'tract' : block['properties']['TRACT']} + within = 'state:{state} county:{county} tract:{tract}'.format(**context) + + block_id = block['properties']['BLOCK'] + result = self.get(fields, + {'for': 'block:{}'.format(block_id), + 'in' : within}, year) + + if result: + result, = result + else: + result = {} + + yield block, result + + +class SF3Client(core.SF3Client, GeoClient): + @supported_years(2000, 1990) + def state_place_tract(self, *args, **kwargs): + return self._state_place_area(self.geo_tract, *args, **kwargs) + + @supported_years(2000) + def state_place_blockgroup(self, *args, **kwargs): + return self._state_place_area(self.geo_blockgroup, *args, **kwargs) + +class Census(core.Census): + def __init__(self, key, year=None, session=None): + super(Census, self).__init__(key, year, session) + self.acs5 = ACS5Client(key, year, session) + self.sf1 = SF1Client(key, year, session) + self.sf3 = SF3Client(key, year, session) diff --git a/census/tests/test_census.py b/census/tests/test_census.py index 67ff12d..abfe224 100644 --- a/census/tests/test_census.py +++ b/census/tests/test_census.py @@ -5,8 +5,9 @@ import requests -from census.core import ( - Census, UnsupportedYearException, DEFINITIONS, __version__) +from census import ( + Census, UnsupportedYearException, __version__) +from census.core import DEFINITIONS KEY = os.environ.get('CENSUS_KEY', '') diff --git a/setup.py b/setup.py index 4fb0b45..5124a7a 100644 --- a/setup.py +++ b/setup.py @@ -24,4 +24,5 @@ "Programming Language :: Python", ], install_requires=['requests>=1.1.0', 'us>=0.7'], + extras_require={'geo' : ['esridump', 'shapely'], ) From bd7616c95dce33147abcb2222d3f197bc441bc05 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Mon, 7 Nov 2016 11:01:31 -0600 Subject: [PATCH 2/2] right import --- census/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/census/__init__.py b/census/__init__.py index ab3434b..ffb6221 100644 --- a/census/__init__.py +++ b/census/__init__.py @@ -1,7 +1,7 @@ from census.core import (ALL, CensusException, UnsupportedYearException, __version__) try: - import pyesridump, shapely + import esridump, shapely except ImportError: from census.core import Census else: