diff --git a/CHANGELOG.md b/CHANGELOG.md index c5337e5..524a3e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add bar charts to state data tooltips [#42](https://github.com/azavea/green-equity-demo/pull/42) - Add budget tracker [#34](https://github.com/azavea/green-equity-demo/pull/34) - Add spending category selector [#30](https://github.com/azavea/green-equity-demo/pull/30) +- Add spending and state data fetching scripts [#40](https://github.com/azavea/green-equity-demo/pull/40) ### Changed diff --git a/scripts/fetch-data b/scripts/fetch-data new file mode 100755 index 0000000..b74155a --- /dev/null +++ b/scripts/fetch-data @@ -0,0 +1,8 @@ +#!/bin/bash + +USAGE="Fetch and save spending data" + +source ./scripts/_sourced +check_for_help_flag "$@" + +./scripts/yarn run fetch-data diff --git a/scripts/update b/scripts/update index b17f6c0..76f9d8b 100755 --- a/scripts/update +++ b/scripts/update @@ -11,5 +11,8 @@ docker compose build # Update Yarn dependencies ./scripts/yarn install --frozen-lockfile +# Fetch data if it doesn't exist +./scripts/fetch-data + # Build static asset bundle for React frontend ./scripts/yarn run build diff --git a/src/app/.gitignore b/src/app/.gitignore index 4d29575..687d244 100644 --- a/src/app/.gitignore +++ b/src/app/.gitignore @@ -21,3 +21,5 @@ npm-debug.log* yarn-debug.log* yarn-error.log* + +/src/data diff --git a/src/app/dataScripts/fetchData.ts b/src/app/dataScripts/fetchData.ts new file mode 100644 index 0000000..9d6cbec --- /dev/null +++ b/src/app/dataScripts/fetchData.ts @@ -0,0 +1,17 @@ +import { mkdir } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; + +import { dataDir } from './nodeConstants'; + +import fetchPerCapitaSpendingData from './fetchPerCapitaSpendingData'; +import fetchStatesData from './fetchStatesData'; + +async function fetchData() { + if (!existsSync(dataDir)) { + await mkdir(dataDir); + } + + await Promise.all([fetchStatesData(), fetchPerCapitaSpendingData()]); +} + +fetchData(); diff --git a/src/app/dataScripts/fetchPerCapitaSpendingData.ts b/src/app/dataScripts/fetchPerCapitaSpendingData.ts new file mode 100644 index 0000000..f442695 --- /dev/null +++ b/src/app/dataScripts/fetchPerCapitaSpendingData.ts @@ -0,0 +1,59 @@ +import fs from 'node:fs/promises'; +import { existsSync } from 'node:fs'; + +import path from 'node:path'; + +import { spendingApiUrl } from '../src/constants'; +import { Category } from '../src/enums'; +import { + getAgenciesForCategory, + getDefaultSpendingByGeographyRequest, +} from '../src/util'; + +import { dataDir } from './nodeConstants'; +import httpsRequestToFile from './httpRequestToFile'; + +export default async function fetchPerCapitaSpendingData() { + console.log('Fetching per-capita spending data...'); + + await Promise.all([ + writeSpendingDataFile(), + writeSpendingDataFile(Category.CLIMATE), + writeSpendingDataFile(Category.CIVIL_WORKS), + writeSpendingDataFile(Category.TRANSPORTATION), + writeSpendingDataFile(Category.BROADBAND), + writeSpendingDataFile(Category.OTHER), + ]); +} + +async function writeSpendingDataFile(category?: Category) { + const filename = path.join(dataDir, `${category ?? 'all'}.spending.json`); + + if (existsSync(filename)) { + console.warn( + ` Skipping ${ + category ?? 'All' + } spending because the file already exists.` + ); + return; + } + + const requestBody = getDefaultSpendingByGeographyRequest(); + + if (category) { + requestBody.filters.agencies = getAgenciesForCategory(category); + } + + return fs.open(filename, 'w').then(async fileHandle => { + await httpsRequestToFile({ + url: `${spendingApiUrl}/search/spending_by_geography/`, + fileHandle, + options: { + method: 'POST', + }, + body: JSON.stringify(requestBody), + }); + + fileHandle.close(); + }); +} diff --git a/src/app/dataScripts/fetchStatesData.ts b/src/app/dataScripts/fetchStatesData.ts new file mode 100644 index 0000000..814ec8c --- /dev/null +++ b/src/app/dataScripts/fetchStatesData.ts @@ -0,0 +1,28 @@ +import fs from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import path from 'node:path'; + +import { spendingApiUrl } from '../src/constants'; + +import { dataDir } from './nodeConstants'; +import httpsRequestToFile from './httpRequestToFile'; + +export default async function fetchStatesData() { + console.log('Fetching state data...'); + + const filename = path.join(dataDir, 'states.json'); + + if (existsSync(filename)) { + console.warn(' Skipping states data because the file already exists.'); + return; + } + + await fs.open(filename, 'w').then(async fileHandle => { + await httpsRequestToFile({ + url: `${spendingApiUrl}/recipient/state/`, + fileHandle, + }); + + fileHandle.close(); + }); +} diff --git a/src/app/dataScripts/httpRequestToFile.ts b/src/app/dataScripts/httpRequestToFile.ts new file mode 100644 index 0000000..b4c89cf --- /dev/null +++ b/src/app/dataScripts/httpRequestToFile.ts @@ -0,0 +1,44 @@ +import { FileHandle } from 'node:fs/promises'; +import https from 'node:https'; + +export default function httpsRequestToFile({ + url, + fileHandle, + options = {}, + body, +}: { + url: string; + fileHandle: FileHandle; + options?: https.RequestOptions; + body?: string; +}): Promise { + return new Promise((resolve, reject) => { + const request = https.request( + url, + { + headers: { + 'Content-Type': 'application/json', + ...(options.headers ?? {}), + }, + ...options, + }, + response => { + response.on('data', data => { + fileHandle.write(data); + }); + + response.on('end', () => { + resolve(); + }); + } + ); + + request.on('error', error => reject(error)); + + if (body) { + request.write(body); + } + + request.end(); + }); +} diff --git a/src/app/dataScripts/nodeConstants.ts b/src/app/dataScripts/nodeConstants.ts new file mode 100644 index 0000000..b199422 --- /dev/null +++ b/src/app/dataScripts/nodeConstants.ts @@ -0,0 +1,3 @@ +import path from 'node:path'; + +export const dataDir = path.join(__dirname, '..', 'src', 'data'); diff --git a/src/app/package.json b/src/app/package.json index 96bec06..e264dbd 100644 --- a/src/app/package.json +++ b/src/app/package.json @@ -42,7 +42,8 @@ "scripts": { "start": "craco start", "build": "craco build", - "test": "craco test" + "test": "craco test", + "fetch-data": "ts-node dataScripts/fetchData.ts" }, "eslintConfig": { "extends": "react-app" @@ -65,6 +66,7 @@ "@types/leaflet": "^1.9.1", "babel-jest": "28.1.0", "prettier": "^2.6.2", - "prettier-loader": "^3.3.0" + "prettier-loader": "^3.3.0", + "ts-node": "^10.9.1" } } diff --git a/src/app/src/api.ts b/src/app/src/api.ts index 3eb485a..b4b9895 100644 --- a/src/app/src/api.ts +++ b/src/app/src/api.ts @@ -1,15 +1,20 @@ -import { createApi, fetchBaseQuery } from '@reduxjs/toolkit/query/react'; +import { createApi } from '@reduxjs/toolkit/query/react'; import { SpendingByGeographyRequest, SpendingByGeographyResponse, State, } from './types/api'; +import { spendingApiUrl } from './constants'; + +/* Uncomment this to use the api */ +/* import { fetchBaseQuery } from '@reduxjs/toolkit/query/react'; */ + +/* Uncomment this to use cached data */ +import fetchBaseQuery from './cachedApiQuery'; export const spendingApi = createApi({ reducerPath: 'spendingApi', - baseQuery: fetchBaseQuery({ - baseUrl: 'https://api.usaspending.gov/api/v2', - }), + baseQuery: fetchBaseQuery({ baseUrl: spendingApiUrl }), endpoints: builder => ({ getStates: builder.query({ query: () => '/recipient/state/', diff --git a/src/app/src/cachedApiQuery.ts b/src/app/src/cachedApiQuery.ts new file mode 100644 index 0000000..3e15800 --- /dev/null +++ b/src/app/src/cachedApiQuery.ts @@ -0,0 +1,92 @@ +import { FetchArgs, fetchBaseQuery } from '@reduxjs/toolkit/query/react'; + +import { + SpendingByGeographyRequest, + SpendingByGeographyResponse, +} from './types/api'; +import { getCategoryForAgencies } from './util'; +import { Category } from './enums'; + +import states from './data/states.json'; +import allSpending from './data/all.spending.json'; +import broadbandSpending from './data/Broadband.spending.json'; +import civilWorksSpending from './data/Civil Works.spending.json'; +import climateSpending from './data/Climate.spending.json'; +import otherSpending from './data/Other.spending.json'; +import transportationSpending from './data/Transportation.spending.json'; + +const cachedApiQuery: typeof fetchBaseQuery = _ => { + return (stringOrArgs, api) => { + const args = isFetchArgs(stringOrArgs) + ? stringOrArgs + : { url: stringOrArgs }; + + switch (args.url) { + case '/recipient/state/': + return wrapIntoData(getStates()); + case '/search/spending_by_geography/': + return wrapIntoData( + getSpendingByGeography( + args.body as SpendingByGeographyRequest + ) + ); + + default: + throw new Error(`Unknown url: ${args.url}`); + } + }; +}; + +function isFetchArgs(args: string | FetchArgs): args is FetchArgs { + return typeof args !== 'string'; +} + +function wrapIntoData(response: T): { data: T } { + return { data: response }; +} + +function getStates() { + return states; +} + +function getSpendingByGeography( + request: SpendingByGeographyRequest +): SpendingByGeographyResponse { + const category = request.filters.agencies + ? getCategoryForAgencies(request.filters.agencies) + : undefined; + + const spending = category + ? getSpendingForCategory(category) + : (allSpending as SpendingByGeographyResponse); + + if (request.geo_layer_filters) { + return { + ...spending, + results: spending.results.filter(result => + request.geo_layer_filters!.includes(result.shape_code) + ), + }; + } + + return spending; +} + +function getSpendingForCategory( + category: Category +): SpendingByGeographyResponse { + switch (category) { + case Category.BROADBAND: + return broadbandSpending as SpendingByGeographyResponse; + case Category.CIVIL_WORKS: + return civilWorksSpending as SpendingByGeographyResponse; + case Category.CLIMATE: + return climateSpending as SpendingByGeographyResponse; + case Category.OTHER: + return otherSpending as SpendingByGeographyResponse; + case Category.TRANSPORTATION: + return transportationSpending as SpendingByGeographyResponse; + } +} + +export default cachedApiQuery; diff --git a/src/app/src/components/DataSandbox.tsx b/src/app/src/components/DataSandbox.tsx index 5813ee6..00ec0c1 100644 --- a/src/app/src/components/DataSandbox.tsx +++ b/src/app/src/components/DataSandbox.tsx @@ -64,7 +64,7 @@ export default function DataSandbox() { onChange={({ target: { value } }) => setStateOrTerritory(value)} size='sm' > - + {states.map(state => (