Merge pull request #40 from azavea/ms/add-spending-fetch-script

Add spending and state data fetching scripts

CHANGELOG.md

@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add bar charts to state data tooltips [#42](https://github.com/azavea/green-equity-demo/pull/42)
 - Add budget tracker [#34](https://github.com/azavea/green-equity-demo/pull/34)
 - Add spending category selector [#30](https://github.com/azavea/green-equity-demo/pull/30)
+- Add spending and state data fetching scripts [#40](https://github.com/azavea/green-equity-demo/pull/40)
 
 ### Changed
 

scripts/fetch-data

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+USAGE="Fetch and save spending data"
+
+source ./scripts/_sourced
+check_for_help_flag "$@"
+
+./scripts/yarn run fetch-data

scripts/update

@@ -11,5 +11,8 @@ docker compose build
 # Update Yarn dependencies
 ./scripts/yarn install --frozen-lockfile
 
+# Fetch data if it doesn't exist
+./scripts/fetch-data
+
 # Build static asset bundle for React frontend
 ./scripts/yarn run build

src/app/.gitignore

@@ -21,3 +21,5 @@
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+
+/src/data

src/app/dataScripts/fetchData.ts

@@ -0,0 +1,17 @@
+import { mkdir } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+
+import { dataDir } from './nodeConstants';
+
+import fetchPerCapitaSpendingData from './fetchPerCapitaSpendingData';
+import fetchStatesData from './fetchStatesData';
+
+async function fetchData() {
+    if (!existsSync(dataDir)) {
+        await mkdir(dataDir);
+    }
+
+    await Promise.all([fetchStatesData(), fetchPerCapitaSpendingData()]);
+}
+
+fetchData();

src/app/dataScripts/fetchPerCapitaSpendingData.ts

@@ -0,0 +1,59 @@
+import fs from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+
+import path from 'node:path';
+
+import { spendingApiUrl } from '../src/constants';
+import { Category } from '../src/enums';
+import {
+    getAgenciesForCategory,
+    getDefaultSpendingByGeographyRequest,
+} from '../src/util';
+
+import { dataDir } from './nodeConstants';
+import httpsRequestToFile from './httpRequestToFile';
+
+export default async function fetchPerCapitaSpendingData() {
+    console.log('Fetching per-capita spending data...');
+
+    await Promise.all([
+        writeSpendingDataFile(),
+        writeSpendingDataFile(Category.CLIMATE),
+        writeSpendingDataFile(Category.CIVIL_WORKS),
+        writeSpendingDataFile(Category.TRANSPORTATION),
+        writeSpendingDataFile(Category.BROADBAND),
+        writeSpendingDataFile(Category.OTHER),
+    ]);
+}
+
+async function writeSpendingDataFile(category?: Category) {
+    const filename = path.join(dataDir, `${category ?? 'all'}.spending.json`);
+
+    if (existsSync(filename)) {
+        console.warn(
+            `  Skipping ${
+                category ?? 'All'
+            } spending because the file already exists.`
+        );
+        return;
+    }
+
+    const requestBody = getDefaultSpendingByGeographyRequest();
+
+    if (category) {
+        requestBody.filters.agencies = getAgenciesForCategory(category);
+    }
+
+    return fs.open(filename, 'w').then(async fileHandle => {
+        await httpsRequestToFile({
+            url: `${spendingApiUrl}/search/spending_by_geography/`,
+            fileHandle,
+            options: {
+                method: 'POST',
+            },
+            body: JSON.stringify(requestBody),
+        });
+
+        fileHandle.close();
+    });
+}

src/app/dataScripts/fetchStatesData.ts

@@ -0,0 +1,28 @@
+import fs from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+
+import { spendingApiUrl } from '../src/constants';
+
+import { dataDir } from './nodeConstants';
+import httpsRequestToFile from './httpRequestToFile';
+
+export default async function fetchStatesData() {
+    console.log('Fetching state data...');
+
+    const filename = path.join(dataDir, 'states.json');
+
+    if (existsSync(filename)) {
+        console.warn(' Skipping states data because the file already exists.');
+        return;
+    }
+
+    await fs.open(filename, 'w').then(async fileHandle => {
+        await httpsRequestToFile({
+            url: `${spendingApiUrl}/recipient/state/`,
+            fileHandle,
+        });
+
+        fileHandle.close();
+    });
+}

src/app/dataScripts/httpRequestToFile.ts

@@ -0,0 +1,44 @@
+import { FileHandle } from 'node:fs/promises';
+import https from 'node:https';
+
+export default function httpsRequestToFile({
+    url,
+    fileHandle,
+    options = {},
+    body,
+}: {
+    url: string;
+    fileHandle: FileHandle;
+    options?: https.RequestOptions;
+    body?: string;
+}): Promise<void> {
+    return new Promise((resolve, reject) => {
+        const request = https.request(
+            url,
+            {
+                headers: {
+                    'Content-Type': 'application/json',
+                    ...(options.headers ?? {}),
+                },
+                ...options,
+            },
+            response => {
+                response.on('data', data => {
+                    fileHandle.write(data);
+                });
+
+                response.on('end', () => {
+                    resolve();
+                });
+            }
+        );
+
+        request.on('error', error => reject(error));
+
+        if (body) {
+            request.write(body);
+        }
+
+        request.end();
+    });
+}

src/app/dataScripts/nodeConstants.ts

@@ -0,0 +1,3 @@
+import path from 'node:path';
+
+export const dataDir = path.join(__dirname, '..', 'src', 'data');

src/app/package.json

@@ -42,7 +42,8 @@
   "scripts": {
     "start": "craco start",
     "build": "craco build",
-    "test": "craco test"
+    "test": "craco test",
+    "fetch-data": "ts-node dataScripts/fetchData.ts"
   },
   "eslintConfig": {
     "extends": "react-app"
@@ -65,6 +66,7 @@
     "@types/leaflet": "^1.9.1",
     "babel-jest": "28.1.0",
     "prettier": "^2.6.2",
-    "prettier-loader": "^3.3.0"
+    "prettier-loader": "^3.3.0",
+    "ts-node": "^10.9.1"
   }
 }

src/app/src/api.ts

@@ -1,15 +1,20 @@
-import { createApi, fetchBaseQuery } from '@reduxjs/toolkit/query/react';
+import { createApi } from '@reduxjs/toolkit/query/react';
 import {
     SpendingByGeographyRequest,
     SpendingByGeographyResponse,
     State,
 } from './types/api';
+import { spendingApiUrl } from './constants';
+
+/* Uncomment this to use the api */
+/* import {  fetchBaseQuery } from '@reduxjs/toolkit/query/react'; */
+
+/* Uncomment this to use cached data */
+import fetchBaseQuery from './cachedApiQuery';
 
 export const spendingApi = createApi({
     reducerPath: 'spendingApi',
-    baseQuery: fetchBaseQuery({
-        baseUrl: 'https://api.usaspending.gov/api/v2',
-    }),
+    baseQuery: fetchBaseQuery({ baseUrl: spendingApiUrl }),
     endpoints: builder => ({
         getStates: builder.query<State[], void>({
             query: () => '/recipient/state/',

src/app/src/cachedApiQuery.ts

@@ -0,0 +1,92 @@
+import { FetchArgs, fetchBaseQuery } from '@reduxjs/toolkit/query/react';
+
+import {
+    SpendingByGeographyRequest,
+    SpendingByGeographyResponse,
+} from './types/api';
+import { getCategoryForAgencies } from './util';
+import { Category } from './enums';
+
+import states from './data/states.json';
+import allSpending from './data/all.spending.json';
+import broadbandSpending from './data/Broadband.spending.json';
+import civilWorksSpending from './data/Civil Works.spending.json';
+import climateSpending from './data/Climate.spending.json';
+import otherSpending from './data/Other.spending.json';
+import transportationSpending from './data/Transportation.spending.json';
+
+const cachedApiQuery: typeof fetchBaseQuery = _ => {
+    return (stringOrArgs, api) => {
+        const args = isFetchArgs(stringOrArgs)
+            ? stringOrArgs
+            : { url: stringOrArgs };
+
+        switch (args.url) {
+            case '/recipient/state/':
+                return wrapIntoData(getStates());
+            case '/search/spending_by_geography/':
+                return wrapIntoData(
+                    getSpendingByGeography(
+                        args.body as SpendingByGeographyRequest
+                    )
+                );
+
+            default:
+                throw new Error(`Unknown url: ${args.url}`);
+        }
+    };
+};
+
+function isFetchArgs(args: string | FetchArgs): args is FetchArgs {
+    return typeof args !== 'string';
+}
+
+function wrapIntoData<T>(response: T): { data: T } {
+    return { data: response };
+}
+
+function getStates() {
+    return states;
+}
+
+function getSpendingByGeography(
+    request: SpendingByGeographyRequest
+): SpendingByGeographyResponse {
+    const category = request.filters.agencies
+        ? getCategoryForAgencies(request.filters.agencies)
+        : undefined;
+
+    const spending = category
+        ? getSpendingForCategory(category)
+        : (allSpending as SpendingByGeographyResponse);
+
+    if (request.geo_layer_filters) {
+        return {
+            ...spending,
+            results: spending.results.filter(result =>
+                request.geo_layer_filters!.includes(result.shape_code)
+            ),
+        };
+    }
+
+    return spending;
+}
+
+function getSpendingForCategory(
+    category: Category
+): SpendingByGeographyResponse {
+    switch (category) {
+        case Category.BROADBAND:
+            return broadbandSpending as SpendingByGeographyResponse;
+        case Category.CIVIL_WORKS:
+            return civilWorksSpending as SpendingByGeographyResponse;
+        case Category.CLIMATE:
+            return climateSpending as SpendingByGeographyResponse;
+        case Category.OTHER:
+            return otherSpending as SpendingByGeographyResponse;
+        case Category.TRANSPORTATION:
+            return transportationSpending as SpendingByGeographyResponse;
+    }
+}
+
+export default cachedApiQuery;

src/app/src/components/DataSandbox.tsx

@@ -64,7 +64,7 @@ export default function DataSandbox() {
                 onChange={({ target: { value } }) => setStateOrTerritory(value)}
                 size='sm'
             >
-                <option>All states</option>
+                <option value=''>All states</option>
                 {states.map(state => (
                     <option key={state.fips} value={state.code}>
                         {state.name}

src/app/src/constants.ts

@@ -1,6 +1,9 @@
-import { AmountCategory } from './types';
 import { PathOptions } from 'leaflet';
 
+import { AmountCategory } from './types';
+
+export const spendingApiUrl = 'https://api.usaspending.gov/api/v2';
+
 export const AMOUNT_CATEGORIES: AmountCategory[] = [
     {
         min: 3001,

src/app/src/util.ts

@@ -120,6 +120,26 @@ export function getAgenciesForCategory(category: Category): Agency[] {
     }
 }
 
+export function getCategoryForAgencies(agencies: Agency[]): Category {
+    const anAgencyInEachCategory: Record<string, Category> = {
+        'Bureau of Reclamation': Category.CLIMATE,
+        'Corps of Engineers - Civil Works': Category.CIVIL_WORKS,
+        'Federal Aviation Administration': Category.TRANSPORTATION,
+        'Rural Utilities Service': Category.BROADBAND,
+        'Denali Commission': Category.OTHER,
+    };
+
+    for (const [agencyName, category] of Object.entries(
+        anAgencyInEachCategory
+    )) {
+        if (agencies.some(agency => agency.name === agencyName)) {
+            return category;
+        }
+    }
+
+    throw new Error(`Category not found for this agency list ${agencies}`);
+}
+
 export function getAmountCategory(amount: number): AmountCategory {
     const category =
         AMOUNT_CATEGORIES.find(

src/app/tsconfig.json

@@ -8,6 +8,7 @@
     "module": "commonjs",
     "noUncheckedIndexedAccess": true,
     "outDir": "build",
+    "resolveJsonModule": true,
     "skipLibCheck": true,
     "strict": true,
     "target": "es2016"

src/app/yarn.lock

@@ -11772,7 +11772,7 @@ ts-easing@^0.2.0:
   resolved "https://registry.yarnpkg.com/ts-easing/-/ts-easing-0.2.0.tgz#c8a8a35025105566588d87dbda05dd7fbfa5a4ec"
   integrity sha512-Z86EW+fFFh/IFB1fqQ3/+7Zpf9t2ebOAxNI/V6Wo7r5gqiqtxmgTlQ1qbqQcjLKYeSHPTsEmvlJUDg/EuL0uHQ==
 
-ts-node@^10.7.0:
+ts-node@^10.7.0, ts-node@^10.9.1:
   version "10.9.1"
   resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.1.tgz#e73de9102958af9e1f0b168a6ff320e25adcff4b"
   integrity sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==