From a37074f1c6145c72534484c185ea2e8603d7fd0a Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Tue, 17 Oct 2023 10:24:22 -0700 Subject: [PATCH 01/36] adds more fwi functions from R library --- api/app/fire_behaviour/cffdrs.py | 70 +++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/api/app/fire_behaviour/cffdrs.py b/api/app/fire_behaviour/cffdrs.py index b03de5026..1963ae338 100644 --- a/api/app/fire_behaviour/cffdrs.py +++ b/api/app/fire_behaviour/cffdrs.py @@ -550,7 +550,73 @@ def fine_fuel_moisture_code(ffmc: float, temperature: float, relative_humidity: raise CFFDRSException("Failed to calculate ffmc") -def initial_spread_index(ffmc: float, wind_speed: float, fbpMod: bool = False): +def duff_moisture_code(dmc: float, temperature: float, relative_humidity: float, + precipitation: float, latitude: float = 55, month: int = 7, + latitude_adjust: bool = True): + """ + Computes Duff Moisture Code (DMC) by delegating to the cffdrs R package. + + R function signature: + function (dmc_yda, temp, rh, prec, lat, mon, lat.adjust = TRUE) + + :param dmc: The Duff Moisture Code (unitless) of the previous day + :type dmc: float + :param temperature: Temperature (centigrade) + :type temperature: float + :param relative_humidity: Relative humidity (%) + :type relative_humidity: float + :param precipitation: 24-hour rainfall (mm) + :type precipitation: float + :param latitude: Latitude (decimal degrees), defaults to 55 + :type latitude: float + :param month: Month of the year (1-12), defaults to 7 (July) + :type month: int, optional + :param latitude_adjust: Options for whether day length adjustments should be applied to + the calculation, defaults to True + :type latitude_adjust: bool, optional + """ + if dmc is None: + dmc = NULL + result = CFFDRS.instance().cffdrs._dmcCalc(dmc, temperature, relative_humidity, precipitation, + latitude, month, latitude_adjust) + if isinstance(result[0], float): + return result[0] + raise CFFDRSException("Failed to calculate dmc") + + +def drought_code(dc: float, temperature: float, relative_humidity: float, precipitation: float, + latitude: float = 55, month: int = 7, latitude_adjust: bool = True) -> None: + """ + Computes Drought Code (DC) by delegating to the cffdrs R package. + + :param dc: The Drought Code (unitless) of the previous day + :type dc: float + :param temperature: Temperature (centigrade) + :type temperature: float + :param relative_humidity: Relative humidity (%) + :type relative_humidity: float + :param precipitation: 24-hour rainfall (mm) + :type precipitation: float + :param latitude: Latitude (decimal degrees), defaults to 55 + :type latitude: float + :param month: Month of the year (1-12), defaults to 7 (July) + :type month: int, optional + :param latitude_adjust: Options for whether day length adjustments should be applied to + the calculation, defaults to True + :type latitude_adjust: bool, optional + :raises CFFDRSException: + :return: None + """ + if dc is None: + dc = NULL + result = CFFDRS.instance().cffdrs._dcCalc(dc, temperature, relative_humidity, precipitation, + latitude, month, latitude_adjust) + if isinstance(result[0], float): + return result[0] + raise CFFDRSException("Failed to calculate dmc") + + +def initial_spread_index(ffmc: float, wind_speed: float, fbp_mod: bool = False): """ Computes Initial Spread Index (ISI) by delegating to cffdrs R package. This is necessary when recalculating ROS/HFI for modified FFMC values. Otherwise, should be using the ISI value retrieved from WFWX. @@ -565,7 +631,7 @@ def initial_spread_index(ffmc: float, wind_speed: float, fbpMod: bool = False): """ if ffmc is None: ffmc = NULL - result = CFFDRS.instance().cffdrs._ISIcalc(ffmc=ffmc, ws=wind_speed, fbpMod=fbpMod) + result = CFFDRS.instance().cffdrs._ISIcalc(ffmc=ffmc, ws=wind_speed, fbpMod=fbp_mod) if isinstance(result[0], float): return result[0] raise CFFDRSException("Failed to calculate ISI") From d6faa20b5f1b7fa756bc1ffd559480acd7e8886c Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Tue, 17 Oct 2023 10:31:23 -0700 Subject: [PATCH 02/36] adds lat/long to WeatherIndeterminate --- api/app/schemas/morecast_v2.py | 2 ++ api/app/wildfire_one/schema_parsers.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/api/app/schemas/morecast_v2.py b/api/app/schemas/morecast_v2.py index f53755f2c..6dbca0111 100644 --- a/api/app/schemas/morecast_v2.py +++ b/api/app/schemas/morecast_v2.py @@ -119,6 +119,8 @@ class WeatherIndeterminate(BaseModel): """ Used to represent a predicted or actual value """ station_code: int station_name: str + latitude: float + longitude: float determinate: WeatherDeterminate utc_timestamp: datetime temperature: Optional[float] = None diff --git a/api/app/wildfire_one/schema_parsers.py b/api/app/wildfire_one/schema_parsers.py index 36b062ae4..cf7423154 100644 --- a/api/app/wildfire_one/schema_parsers.py +++ b/api/app/wildfire_one/schema_parsers.py @@ -83,6 +83,8 @@ async def weather_indeterminate_list_mapper(raw_dailies: Generator[dict, None, N async for raw_daily in raw_dailies: station_code = raw_daily.get('stationData').get('stationCode') station_name = raw_daily.get('stationData').get('displayLabel') + latitude = raw_daily.get('stationData').get('latitude') + longitude = raw_daily.get('stationData').get('longitude') utc_timestamp = datetime.fromtimestamp(raw_daily.get('weatherTimestamp') / 1000, tz=timezone.utc) precip = raw_daily.get('precipitation') rh = raw_daily.get('relativeHumidity') @@ -101,6 +103,8 @@ async def weather_indeterminate_list_mapper(raw_dailies: Generator[dict, None, N observed_dailies.append(WeatherIndeterminate( station_code=station_code, station_name=station_name, + latitude=latitude, + longitude=longitude, determinate=WeatherDeterminate.ACTUAL, utc_timestamp=utc_timestamp, temperature=temp, @@ -120,6 +124,8 @@ async def weather_indeterminate_list_mapper(raw_dailies: Generator[dict, None, N forecasts.append(WeatherIndeterminate( station_code=station_code, station_name=station_name, + latitude=latitude, + longitude=longitude, determinate=WeatherDeterminate.FORECAST, utc_timestamp=utc_timestamp, temperature=temp, From 95af97e5d86ba6cc4f38bb94523e57bcde2ce65b Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Tue, 17 Oct 2023 11:07:29 -0700 Subject: [PATCH 03/36] calculate forecasted fwi system values --- api/app/morecast_v2/forecasts.py | 87 +++++++++++++++++++++++++++++++- api/app/routers/morecast_v2.py | 3 ++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/api/app/morecast_v2/forecasts.py b/api/app/morecast_v2/forecasts.py index 002426b68..553ed43b4 100644 --- a/api/app/morecast_v2/forecasts.py +++ b/api/app/morecast_v2/forecasts.py @@ -1,5 +1,5 @@ -from datetime import datetime, time +from datetime import datetime, time, timedelta from urllib.parse import urljoin from app import config @@ -14,6 +14,7 @@ from app.schemas.morecast_v2 import MoreCastForecastOutput, StationDailyFromWF1, WF1ForecastRecordType, WF1PostForecast, WeatherIndeterminate from app.wildfire_one.schema_parsers import WFWXWeatherStation from app.wildfire_one.wfwx_api import get_auth_header, get_forecasts_for_stations_by_date_range, get_wfwx_stations_from_station_codes +from app.fire_behaviour import cffdrs def get_forecasts(db_session: Session, start_time: datetime, end_time: datetime, station_codes: List[int]) -> List[MoreCastForecastOutput]: @@ -105,3 +106,87 @@ def filter_for_api_forecasts(forecasts: List[WeatherIndeterminate], actuals: Lis if actual_exists(forecast, actuals): filtered_forecasts.append(forecast) return filtered_forecasts + + +def get_forecasted_fwi_values(actuals: List[WeatherIndeterminate], forecasts: List[WeatherIndeterminate]) -> List[WeatherIndeterminate]: + """ + Fills forecasts with Fire Weather Index System values by calculating based off previous actuals and subsequent forecasts. + + :param actuals: List of actual weather values + :type actuals: List[WeatherIndeterminate] + :param forecasts: List of existing forecasted values + :type forecasts: List[WeatherIndeterminate] + :return: Updated and filled forecasts + :rtype: List[WeatherIndeterminate] + """ + actuals_dict = defaultdict(dict) + for actual in actuals: + actuals_dict[actual.station_code][actual.utc_timestamp.date()] = actual + + previous_indeterminate = None + for idx, forecast in enumerate(forecasts): + if previous_indeterminate and previous_indeterminate.station_code == forecast.station_code: + updated_forecast = calculate_fwi_indices(previous_indeterminate, forecast) + forecasts[idx] = updated_forecast + else: + last_actual = actuals_dict[forecast.station_code][forecast.utc_timestamp.date() - timedelta(days=1)] + updated_forecast = calculate_fwi_indices(last_actual, forecast) + forecasts[idx] = updated_forecast + previous_indeterminate = forecast + + return forecasts + + +def calculate_fwi_indices(yesterday: WeatherIndeterminate, today: WeatherIndeterminate) -> WeatherIndeterminate: + """ + Uses CFFDRS library to calculate Fire Weather Index System values + + :param yesterday: The WeatherIndeterminate from the day before the date to calculate + :type yesterday: WeatherIndeterminate + :param today: The WeatherIndeterminate from the date to calculate + :type today: WeatherIndeterminate + :return: Updated WeatherIndeterminate + :rtype: WeatherIndeterminate + """ + + # weather params for calculation date + month_to_calculate_for = int(today.utc_timestamp.strftime('%m')) + latitude = today.latitude + temp = today.temperature + rh = today.relative_humidity + precip = today.precipitation + wind_spd = today.wind_speed + + if yesterday.fine_fuel_moisture_code: + today.fine_fuel_moisture_code = cffdrs.fine_fuel_moisture_code(ffmc=yesterday.fine_fuel_moisture_code, + temperature=temp, + relative_humidity=rh, + precipitation=precip, + wind_speed=wind_spd) + if yesterday.duff_moisture_code: + today.duff_moisture_code = cffdrs.duff_moisture_code(dmc=yesterday.duff_moisture_code, + temperature=temp, + relative_humidity=rh, + precipitation=precip, + latitude=latitude, + month=month_to_calculate_for, + latitude_adjust=True + ) + if yesterday.drought_code: + today.drought_code = cffdrs.drought_code(dc=yesterday.drought_code, + temperature=temp, + relative_humidity=rh, + precipitation=precip, + latitude=latitude, + month=month_to_calculate_for, + latitude_adjust=True + ) + if today.fine_fuel_moisture_code: + today.initial_spread_index = cffdrs.initial_spread_index(ffmc=today.fine_fuel_moisture_code, + wind_speed=today.wind_speed) + if today.duff_moisture_code and today.drought_code: + today.build_up_index = cffdrs.bui_calc(dmc=today.duff_moisture_code, dc=today.drought_code) + if today.initial_spread_index and today.build_up_index: + today.fire_weather_index = cffdrs.fire_weather_index(isi=today.initial_spread_index, bui=today.build_up_index) + + return today diff --git a/api/app/routers/morecast_v2.py b/api/app/routers/morecast_v2.py index ab1aa4aa9..96d209ff2 100644 --- a/api/app/routers/morecast_v2.py +++ b/api/app/routers/morecast_v2.py @@ -29,6 +29,7 @@ get_dailies_for_stations_and_date, get_daily_determinates_for_stations_and_date, get_wfwx_stations_from_station_codes) from app.wildfire_one.wfwx_post_api import post_forecasts +from app.morecast_v2.forecasts import get_forecasted_fwi_values logger = logging.getLogger(__name__) @@ -191,6 +192,8 @@ async def get_determinates_for_date_range(start_date: date, start_date_of_interest, end_date_of_interest, unique_station_codes) + + wf1_forecasts = get_forecasted_fwi_values(wf1_actuals, wf1_forecasts) # Find the min and max dates for actuals from wf1. These define the range of dates for which # we need to retrieve forecasts from our API database. Note that not all stations report actuals # at the same time, so every station won't necessarily have an actual for each date in the range. From ba327f072131cf518943ce3736141dc2a1da696a Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Thu, 19 Oct 2023 08:16:44 -0700 Subject: [PATCH 04/36] adds FWI actuals/forecasts to Forecast Summary tab --- .../moreCast2/components/ColumnDefBuilder.tsx | 9 +++- .../moreCast2/components/DataGridColumns.tsx | 2 +- .../components/GridComponentRenderer.tsx | 10 ++-- .../moreCast2/components/MoreCast2Column.tsx | 16 +++--- .../moreCast2/components/TabbedDataGrid.tsx | 6 ++- web/src/features/moreCast2/interfaces.ts | 21 +++++--- .../features/moreCast2/saveForecast.test.ts | 14 ++--- .../features/moreCast2/slices/dataSlice.ts | 52 ++++++++++++++----- 8 files changed, 85 insertions(+), 45 deletions(-) diff --git a/web/src/features/moreCast2/components/ColumnDefBuilder.tsx b/web/src/features/moreCast2/components/ColumnDefBuilder.tsx index de1824a07..d1a5f17c0 100644 --- a/web/src/features/moreCast2/components/ColumnDefBuilder.tsx +++ b/web/src/features/moreCast2/components/ColumnDefBuilder.tsx @@ -49,11 +49,13 @@ export class ColumnDefBuilder implements ColDefGenerator, ForecastColDefGenerato } public generateForecastColDef = (headerName?: string) => { + const isCalcField = this.field.includes('Calc') + return this.generateForecastColDefWith( `${this.field}${WeatherDeterminate.FORECAST}`, headerName ? headerName : this.headerName, this.precision, - DEFAULT_FORECAST_COLUMN_WIDTH + isCalcField ? DEFAULT_COLUMN_WIDTH : DEFAULT_FORECAST_COLUMN_WIDTH ) } @@ -101,6 +103,7 @@ export class ColumnDefBuilder implements ColDefGenerator, ForecastColDefGenerato } public generateForecastColDefWith = (field: string, headerName: string, precision: number, width?: number) => { + const isCalcField = field.includes('Calc') return { field: field, disableColumnMenu: true, @@ -111,7 +114,9 @@ export class ColumnDefBuilder implements ColDefGenerator, ForecastColDefGenerato type: 'number', width: width || 120, renderHeader: (params: GridColumnHeaderParams) => { - return this.gridComponentRenderer.renderForecastHeaderWith(params) + return isCalcField + ? this.gridComponentRenderer.renderHeaderWith(params) + : this.gridComponentRenderer.renderForecastHeaderWith(params) }, renderCell: (params: Pick) => { return this.gridComponentRenderer.renderForecastCellWith(params, field) diff --git a/web/src/features/moreCast2/components/DataGridColumns.tsx b/web/src/features/moreCast2/components/DataGridColumns.tsx index 63b76622a..ee06b8807 100644 --- a/web/src/features/moreCast2/components/DataGridColumns.tsx +++ b/web/src/features/moreCast2/components/DataGridColumns.tsx @@ -55,7 +55,7 @@ export class DataGridColumns { public static getSummaryColumns(): GridColDef[] { return MORECAST2_STATION_DATE_FIELDS.map(field => field.generateColDef()).concat( MORECAST2_FORECAST_FIELDS.map(forecastField => forecastField.generateForecastColDef()).concat( - MORECAST2_INDEX_FIELDS.map(field => field.generateColDef()) + MORECAST2_INDEX_FIELDS.map(field => field.generateForecastColDef()) ) ) } diff --git a/web/src/features/moreCast2/components/GridComponentRenderer.tsx b/web/src/features/moreCast2/components/GridComponentRenderer.tsx index 4fa1fe444..86cef0692 100644 --- a/web/src/features/moreCast2/components/GridComponentRenderer.tsx +++ b/web/src/features/moreCast2/components/GridComponentRenderer.tsx @@ -35,9 +35,7 @@ export class GridComponentRenderer { ) public getActualField = (field: string) => { - const index = field.indexOf('Forecast') - const prefix = field.slice(0, index) - const actualField = `${prefix}Actual` + const actualField = field.replace('Forecast', 'Actual') return actualField } @@ -66,12 +64,14 @@ export class GridComponentRenderer { // We need the prefix to help us grab the correct 'actual' field (eg. tempACTUAL, precipACTUAL, etc.) const actualField = this.getActualField(field) + const isCalcField = field.includes('Calc') + const isActual = !isNaN(params.row[actualField]) return ( ) diff --git a/web/src/features/moreCast2/components/MoreCast2Column.tsx b/web/src/features/moreCast2/components/MoreCast2Column.tsx index 9ee8ac2a8..a211e5e54 100644 --- a/web/src/features/moreCast2/components/MoreCast2Column.tsx +++ b/web/src/features/moreCast2/components/MoreCast2Column.tsx @@ -112,13 +112,13 @@ export const rhForecastField = new IndeterminateField('rh', 'RH', 'number', 0, t export const windDirForecastField = new IndeterminateField('windDirection', 'Wind Dir', 'number', 0, true) export const windSpeedForecastField = new IndeterminateField('windSpeed', 'Wind Speed', 'number', 1, true) export const precipForecastField = new IndeterminateField('precip', 'Precip', 'number', 1, false) -export const buiField = new IndeterminateField('bui', 'BUI', 'number', 0, false) -export const isiField = new IndeterminateField('isi', 'ISI', 'number', 1, false) -export const fwiField = new IndeterminateField('fwi', 'FWI', 'number', 0, false) -export const ffmcField = new IndeterminateField('ffmc', 'FFMC', 'number', 1, false) -export const dmcField = new IndeterminateField('dmc', 'DMC', 'number', 0, false) -export const dcField = new IndeterminateField('dc', 'DC', 'number', 0, false) -export const dgrField = new IndeterminateField('dgr', 'DGR', 'number', 0, false) +export const buiField = new IndeterminateField('buiCalc', 'BUI', 'number', 0, false) +export const isiField = new IndeterminateField('isiCalc', 'ISI', 'number', 1, false) +export const fwiField = new IndeterminateField('fwiCalc', 'FWI', 'number', 0, false) +export const ffmcField = new IndeterminateField('ffmcCalc', 'FFMC', 'number', 1, false) +export const dmcField = new IndeterminateField('dmcCalc', 'DMC', 'number', 0, false) +export const dcField = new IndeterminateField('dcCalc', 'DC', 'number', 0, false) +export const dgrField = new IndeterminateField('dgrCalc', 'DGR', 'number', 0, false) export const MORECAST2_STATION_DATE_FIELDS: ColDefGenerator[] = [ StationForecastField.getInstance(), @@ -143,7 +143,7 @@ export const MORECAST2_FORECAST_FIELDS: ForecastColDefGenerator[] = [ precipForecastField ] -export const MORECAST2_INDEX_FIELDS: ColDefGenerator[] = [ +export const MORECAST2_INDEX_FIELDS: ForecastColDefGenerator[] = [ ffmcField, dmcField, dcField, diff --git a/web/src/features/moreCast2/components/TabbedDataGrid.tsx b/web/src/features/moreCast2/components/TabbedDataGrid.tsx index 5146f58b2..60534da6c 100644 --- a/web/src/features/moreCast2/components/TabbedDataGrid.tsx +++ b/web/src/features/moreCast2/components/TabbedDataGrid.tsx @@ -74,7 +74,11 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp } | null>(null) const handleColumnHeaderClick: GridEventListener<'columnHeaderClick'> = (params, event) => { - if (!isEqual(params.colDef.field, 'stationName') && !isEqual(params.colDef.field, 'forDate')) { + if ( + !isEqual(params.colDef.field, 'stationName') && + !isEqual(params.colDef.field, 'forDate') && + !params.colDef.field.includes('Calc') + ) { setClickedColDef(params.colDef) setContextMenu(contextMenu === null ? { mouseX: event.clientX, mouseY: event.clientY } : null) } diff --git a/web/src/features/moreCast2/interfaces.ts b/web/src/features/moreCast2/interfaces.ts index 11fca7696..1c9bc7e0c 100644 --- a/web/src/features/moreCast2/interfaces.ts +++ b/web/src/features/moreCast2/interfaces.ts @@ -28,13 +28,20 @@ export interface BaseRow { export interface MoreCast2Row extends BaseRow { // Fire weather indices - ffmc: number - dmc: number - dc: number - isi: number - bui: number - fwi: number - dgr: number + ffmcCalcActual: number + dmcCalcActual: number + dcCalcActual: number + isiCalcActual: number + buiCalcActual: number + fwiCalcActual: number + dgrCalcActual: number + ffmcCalcForecast?: PredictionItem + dmcCalcForecast?: PredictionItem + dcCalcForecast?: PredictionItem + isiCalcForecast?: PredictionItem + buiCalcForecast?: PredictionItem + fwiCalcForecast?: PredictionItem + dgrCalcForecast?: PredictionItem // Forecast properties precipForecast?: PredictionItem diff --git a/web/src/features/moreCast2/saveForecast.test.ts b/web/src/features/moreCast2/saveForecast.test.ts index 9d702740e..70e242e3c 100644 --- a/web/src/features/moreCast2/saveForecast.test.ts +++ b/web/src/features/moreCast2/saveForecast.test.ts @@ -59,13 +59,13 @@ const baseRow = { windSpeedNAM_BIAS: 0, windSpeedRDPS: 0, windSpeedRDPS_BIAS: 0, - ffmc: 0, - dmc: 0, - dc: 0, - isi: 0, - bui: 0, - fwi: 0, - dgr: 0 + ffmcCalcActual: 0, + dmcCalcActual: 0, + dcCalcActual: 0, + isiCalcActual: 0, + buiCalcActual: 0, + fwiCalcActual: 0, + dgrCalcActual: 0 } const baseRowWithActuals = { diff --git a/web/src/features/moreCast2/slices/dataSlice.ts b/web/src/features/moreCast2/slices/dataSlice.ts index 87dd35472..e37cb2699 100644 --- a/web/src/features/moreCast2/slices/dataSlice.ts +++ b/web/src/features/moreCast2/slices/dataSlice.ts @@ -142,13 +142,13 @@ export const createMoreCast2Rows = ( row.tempActual = getNumberOrNaN(value.temperature) row.windDirectionActual = getNumberOrNaN(value.wind_direction) row.windSpeedActual = getNumberOrNaN(value.wind_speed) - row.ffmc = getNumberOrNaN(value.fine_fuel_moisture_code) - row.dmc = getNumberOrNaN(value.duff_moisture_code) - row.dc = getNumberOrNaN(value.drought_code) - row.isi = getNumberOrNaN(value.initial_spread_index) - row.bui = getNumberOrNaN(value.build_up_index) - row.fwi = getNumberOrNaN(value.fire_weather_index) - row.dgr = getNumberOrNaN(value.danger_rating) + row.ffmcCalcActual = getNumberOrNaN(value.fine_fuel_moisture_code) + row.dmcCalcActual = getNumberOrNaN(value.duff_moisture_code) + row.dcCalcActual = getNumberOrNaN(value.drought_code) + row.isiCalcActual = getNumberOrNaN(value.initial_spread_index) + row.buiCalcActual = getNumberOrNaN(value.build_up_index) + row.fwiCalcActual = getNumberOrNaN(value.fire_weather_index) + row.dgrCalcActual = getNumberOrNaN(value.danger_rating) break case WeatherDeterminate.FORECAST: case WeatherDeterminate.NULL: @@ -172,6 +172,30 @@ export const createMoreCast2Rows = ( choice: forecastOrNull(value.determinate), value: getNumberOrNaN(value.wind_speed) } + row.ffmcCalcForecast = { + choice: forecastOrNull(ModelChoice.NULL), + value: getNumberOrNaN(value.fine_fuel_moisture_code) + } + row.dmcCalcForecast = { + choice: forecastOrNull(ModelChoice.NULL), + value: getNumberOrNaN(value.duff_moisture_code) + } + row.dcCalcForecast = { + choice: forecastOrNull(ModelChoice.NULL), + value: getNumberOrNaN(value.drought_code) + } + row.isiCalcForecast = { + choice: forecastOrNull(ModelChoice.NULL), + value: getNumberOrNaN(value.initial_spread_index) + } + row.buiCalcForecast = { + choice: forecastOrNull(ModelChoice.NULL), + value: getNumberOrNaN(value.build_up_index) + } + row.fwiCalcForecast = { + choice: forecastOrNull(ModelChoice.NULL), + value: getNumberOrNaN(value.fire_weather_index) + } break case WeatherDeterminate.GDPS: row.precipGDPS = getNumberOrNaN(value.precipitation) @@ -503,13 +527,13 @@ const createEmptyMoreCast2Row = ( windSpeedActual: NaN, // Indices - ffmc: NaN, - dmc: NaN, - dc: NaN, - isi: NaN, - bui: NaN, - fwi: NaN, - dgr: NaN, + ffmcCalcActual: NaN, + dmcCalcActual: NaN, + dcCalcActual: NaN, + isiCalcActual: NaN, + buiCalcActual: NaN, + fwiCalcActual: NaN, + dgrCalcActual: NaN, // GDPS model predictions precipGDPS: NaN, From e17027e498db7cf3b5b69abb87977087f18aee4b Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Wed, 25 Oct 2023 12:23:45 -0700 Subject: [PATCH 05/36] Morecast 2.0 frontend - adds lat/long to Morecast rows - updates API for simulating indices --- web/src/api/moreCast2API.ts | 50 ++++++++++++++++++- web/src/features/moreCast2/interfaces.ts | 2 + .../features/moreCast2/slices/dataSlice.ts | 33 ++++++++++-- 3 files changed, 79 insertions(+), 6 deletions(-) diff --git a/web/src/api/moreCast2API.ts b/web/src/api/moreCast2API.ts index 2f6f96687..f1dee1bf8 100644 --- a/web/src/api/moreCast2API.ts +++ b/web/src/api/moreCast2API.ts @@ -1,7 +1,8 @@ import axios from 'api/axios' import { isEqual } from 'lodash' import { DateTime } from 'luxon' -import { MoreCast2ForecastRow } from 'features/moreCast2/interfaces' +import { MoreCast2ForecastRow, MoreCast2Row } from 'features/moreCast2/interfaces' +import { isForecastRowPredicate } from 'features/moreCast2/saveForecasts' export enum ModelChoice { ACTUAL = 'ACTUAL', @@ -121,6 +122,8 @@ export interface WeatherIndeterminate { id: string station_code: number station_name: string + latitude: number + longitude: number determinate: WeatherDeterminateType utc_timestamp: string precipitation: number | null @@ -149,6 +152,10 @@ export interface WeatherIndeterminateResponse { predictions: WeatherIndeterminate[] } +export interface UpdatedWeatherIndeterminateResponse { + simulatedForecasts: WeatherIndeterminate[] +} + export const ModelOptions: ModelType[] = ModelChoices.filter(choice => !isEqual(choice, ModelChoice.MANUAL)) export interface MoreCast2ForecastRecord { @@ -232,3 +239,44 @@ export async function fetchWeatherIndeterminates( return payload } + +export async function fetchCalculatedIndices( + recordsToSimulate: MoreCast2Row[] +): Promise { + const url = 'morecast-v2/simulate-indices/' + const determinatesToSimulate = mapMoreCast2RowsToIndeterminates(recordsToSimulate) + const { data } = await axios.post(url, { + simulate_records: determinatesToSimulate + }) + const response: UpdatedWeatherIndeterminateResponse = { simulatedForecasts: data } + + return response +} + +export const mapMoreCast2RowsToIndeterminates = (rows: MoreCast2Row[]): WeatherIndeterminate[] => { + const mappedIndeterminates = rows.map(r => { + const isForecast = isForecastRowPredicate(r) + return { + id: r.id, + station_code: r.stationCode, + station_name: r.stationName, + determinate: isForecast ? WeatherDeterminate.FORECAST : WeatherDeterminate.ACTUAL, + latitude: r.latitude, + longitude: r.longitude, + utc_timestamp: r.forDate.toString(), + precipitation: isForecast ? r.precipForecast!.value : r.precipActual, + relative_humidity: isForecast ? r.rhForecast!.value : r.rhActual, + temperature: isForecast ? r.tempForecast!.value : r.tempActual, + wind_direction: isForecast ? r.windDirectionForecast!.value : r.windDirectionActual, + wind_speed: isForecast ? r.windSpeedForecast!.value : r.windSpeedActual, + fine_fuel_moisture_code: isForecast ? r.ffmcCalcForecast!.value : r.ffmcCalcActual, + duff_moisture_code: isForecast ? r.dmcCalcForecast!.value : r.dmcCalcActual, + drought_code: isForecast ? r.dcCalcForecast!.value : r.dcCalcActual, + initial_spread_index: isForecast ? r.isiCalcForecast!.value : r.isiCalcActual, + build_up_index: isForecast ? r.buiCalcForecast!.value : r.buiCalcActual, + fire_weather_index: isForecast ? r.fwiCalcForecast!.value : r.fwiCalcActual, + danger_rating: isForecast ? null : r.rhActual + } + }) + return mappedIndeterminates +} diff --git a/web/src/features/moreCast2/interfaces.ts b/web/src/features/moreCast2/interfaces.ts index 1c9bc7e0c..03587acc6 100644 --- a/web/src/features/moreCast2/interfaces.ts +++ b/web/src/features/moreCast2/interfaces.ts @@ -24,6 +24,8 @@ export interface BaseRow { stationCode: number stationName: string forDate: DateTime + latitude: number + longitude: number } export interface MoreCast2Row extends BaseRow { diff --git a/web/src/features/moreCast2/slices/dataSlice.ts b/web/src/features/moreCast2/slices/dataSlice.ts index e37cb2699..921549931 100644 --- a/web/src/features/moreCast2/slices/dataSlice.ts +++ b/web/src/features/moreCast2/slices/dataSlice.ts @@ -131,7 +131,9 @@ export const createMoreCast2Rows = ( firstItem.id, firstItem.station_code, firstItem.station_name, - DateTime.fromISO(firstItem.utc_timestamp) + DateTime.fromISO(firstItem.utc_timestamp), + firstItem.latitude, + firstItem.longitude ) for (const value of values) { @@ -336,11 +338,20 @@ export const fillMissingWeatherIndeterminates = ( for (const [key, values] of Object.entries(groupedByStationCode)) { const stationCode = parseInt(key) const stationName = stationMap.get(stationCode) ?? '' + const latitude = values[0]?.latitude ?? 0 + const longitude = values[0]?.longitude ?? 0 // We expect one actual per date in our date interval if (values.length < dateInterval.length) { for (const date of dateInterval) { if (!values.some(value => isEqual(DateTime.fromISO(value.utc_timestamp), DateTime.fromISO(date)))) { - const missing = createEmptyWeatherIndeterminate(stationCode, stationName, date, determinate) + const missing = createEmptyWeatherIndeterminate( + stationCode, + stationName, + date, + determinate, + latitude, + longitude + ) weatherIndeterminates.push(missing) } } @@ -379,6 +390,8 @@ export const fillMissingPredictions = ( for (const [stationCodeAsString, weatherIndeterminatesByStationCode] of Object.entries(groupedByStationCode)) { const stationCode = parseInt(stationCodeAsString) const stationName = stationMap.get(stationCode) ?? '' + const latitude = weatherIndeterminatesByStationCode[0]?.latitude ?? 0 + const longitude = weatherIndeterminatesByStationCode[0]?.longitude ?? 0 const groupedByUtcTimestamp = createUtcTimeStampToWeatherIndeterminateGroups( weatherIndeterminatesByStationCode, dateInterval @@ -392,7 +405,9 @@ export const fillMissingPredictions = ( stationCode, stationName, utcTimestamp, - determinate + determinate, + latitude, + longitude ) allPredictions.push(missingDeterminate) } @@ -513,13 +528,17 @@ const createEmptyMoreCast2Row = ( id: string, stationCode: number, stationName: string, - forDate: DateTime + forDate: DateTime, + latitude: number, + longitude: number ): MoreCast2Row => { return { id, stationCode, stationName, forDate, + latitude, + longitude, precipActual: NaN, rhActual: NaN, tempActual: NaN, @@ -619,12 +638,16 @@ const createEmptyWeatherIndeterminate = ( station_code: number, station_name: string, utc_timestamp: string, - determinate: WeatherDeterminateType + determinate: WeatherDeterminateType, + latitude: number, + longitude: number ): WeatherIndeterminate => { return { id: '', station_code, station_name, + latitude, + longitude, determinate, utc_timestamp, precipitation: null, From 784ab604a9549639e23285da07242e35d13d3ada Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Wed, 25 Oct 2023 12:25:34 -0700 Subject: [PATCH 06/36] Morecast 2.0 backend - updates end point for simulating indices --- api/app/routers/morecast_v2.py | 50 +++++++++++++++++++++++++++++++++- api/app/schemas/morecast_v2.py | 8 ++++-- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/api/app/routers/morecast_v2.py b/api/app/routers/morecast_v2.py index 96d209ff2..65f16bdaf 100644 --- a/api/app/routers/morecast_v2.py +++ b/api/app/routers/morecast_v2.py @@ -20,7 +20,8 @@ MorecastForecastResponse, ObservedDailiesForStations, StationDailiesResponse, - WeatherIndeterminate) + WeatherIndeterminate, + SimulateIndeterminateIndices) from app.schemas.shared import StationsRequest from app.wildfire_one.schema_parsers import transform_morecastforecastoutput_to_weatherindeterminate from app.utils.time import get_hour_20_from_date, get_utc_now @@ -30,6 +31,7 @@ get_daily_determinates_for_stations_and_date, get_wfwx_stations_from_station_codes) from app.wildfire_one.wfwx_post_api import post_forecasts from app.morecast_v2.forecasts import get_forecasted_fwi_values +from app.fire_behaviour import cffdrs logger = logging.getLogger(__name__) @@ -223,3 +225,49 @@ async def get_determinates_for_date_range(start_date: date, actuals=wf1_actuals, predictions=predictions, forecasts=wf1_forecasts) + + +@router.post('/simulate-indices/', response_model=SimulateIndeterminateIndices) +async def calculate_forecasted_indices(simulate_records: SimulateIndeterminateIndices): + indeterminates = simulate_records.simulate_records + logger.info( + f'/simulate-indices/ - simulating forecast records for {indeterminates[0].station_name}') + + yesterday_record = indeterminates[0] + indeterminates_to_simulate = indeterminates[1:] + for record in indeterminates_to_simulate: + # weather params for calculation date + month_to_calculate_for = int(record.utc_timestamp.strftime('%m')) + latitude = record.latitude + temp = record.temperature + rh = record.relative_humidity + precip = record.precipitation + wind_speed = record.wind_speed + record.fine_fuel_moisture_code = cffdrs.fine_fuel_moisture_code(ffmc=yesterday_record.fine_fuel_moisture_code, + temperature=temp, + relative_humidity=rh, + precipitation=precip, + wind_speed=wind_speed) + record.duff_moisture_code = cffdrs.duff_moisture_code(dmc=yesterday_record.duff_moisture_code, + temperature=temp, + relative_humidity=rh, + precipitation=precip, + latitude=latitude, + month=month_to_calculate_for, + latitude_adjust=True + ) + record.drought_code = cffdrs.drought_code(dc=yesterday_record.drought_code, + temperature=temp, + relative_humidity=rh, + precipitation=precip, + latitude=latitude, + month=month_to_calculate_for, + latitude_adjust=True + ) + record.initial_spread_index = cffdrs.initial_spread_index(ffmc=record.fine_fuel_moisture_code, + wind_speed=record.wind_speed) + record.build_up_index = cffdrs.bui_calc(dmc=record.duff_moisture_code, dc=record.drought_code) + record.fire_weather_index = cffdrs.fire_weather_index( + isi=record.initial_spread_index, bui=record.build_up_index) + yesterday_record = record + return SimulateIndeterminateIndices(simulate_records=indeterminates_to_simulate) diff --git a/api/app/schemas/morecast_v2.py b/api/app/schemas/morecast_v2.py index 6dbca0111..8ce03e21f 100644 --- a/api/app/schemas/morecast_v2.py +++ b/api/app/schemas/morecast_v2.py @@ -119,10 +119,10 @@ class WeatherIndeterminate(BaseModel): """ Used to represent a predicted or actual value """ station_code: int station_name: str - latitude: float - longitude: float determinate: WeatherDeterminate utc_timestamp: datetime + latitude: Optional[float] = None + longitude: Optional[float] = None temperature: Optional[float] = None relative_humidity: Optional[float] = None precipitation: Optional[float] = None @@ -143,6 +143,10 @@ class IndeterminateDailiesResponse(BaseModel): forecasts: List[WeatherIndeterminate] +class SimulateIndeterminateIndices(BaseModel): + simulate_records: List[WeatherIndeterminate] + + class WF1ForecastRecordType(BaseModel): id: str = "FORECAST" displayLabel: str = "Forecast" From a4bd2666bd5f79f877180ca18d414de839e78732 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Thu, 26 Oct 2023 10:11:01 -0700 Subject: [PATCH 07/36] updates api and updates forecasts in redux store --- api/app/routers/morecast_v2.py | 7 +-- api/app/schemas/morecast_v2.py | 4 ++ web/src/api/moreCast2API.ts | 5 +-- .../components/ForecastSummaryDataGrid.tsx | 43 ++++++++++++++++++- .../features/moreCast2/slices/dataSlice.ts | 21 +++++++-- 5 files changed, 68 insertions(+), 12 deletions(-) diff --git a/api/app/routers/morecast_v2.py b/api/app/routers/morecast_v2.py index 65f16bdaf..b081a9d71 100644 --- a/api/app/routers/morecast_v2.py +++ b/api/app/routers/morecast_v2.py @@ -21,7 +21,8 @@ ObservedDailiesForStations, StationDailiesResponse, WeatherIndeterminate, - SimulateIndeterminateIndices) + SimulateIndeterminateIndices, + SimulatedWeatherIndeterminateResponse) from app.schemas.shared import StationsRequest from app.wildfire_one.schema_parsers import transform_morecastforecastoutput_to_weatherindeterminate from app.utils.time import get_hour_20_from_date, get_utc_now @@ -227,7 +228,7 @@ async def get_determinates_for_date_range(start_date: date, forecasts=wf1_forecasts) -@router.post('/simulate-indices/', response_model=SimulateIndeterminateIndices) +@router.post('/simulate-indices/', response_model=SimulatedWeatherIndeterminateResponse) async def calculate_forecasted_indices(simulate_records: SimulateIndeterminateIndices): indeterminates = simulate_records.simulate_records logger.info( @@ -270,4 +271,4 @@ async def calculate_forecasted_indices(simulate_records: SimulateIndeterminateIn record.fire_weather_index = cffdrs.fire_weather_index( isi=record.initial_spread_index, bui=record.build_up_index) yesterday_record = record - return SimulateIndeterminateIndices(simulate_records=indeterminates_to_simulate) + return (SimulatedWeatherIndeterminateResponse(simulatedForecasts=indeterminates_to_simulate)) diff --git a/api/app/schemas/morecast_v2.py b/api/app/schemas/morecast_v2.py index 8ce03e21f..99d8d899f 100644 --- a/api/app/schemas/morecast_v2.py +++ b/api/app/schemas/morecast_v2.py @@ -147,6 +147,10 @@ class SimulateIndeterminateIndices(BaseModel): simulate_records: List[WeatherIndeterminate] +class SimulatedWeatherIndeterminateResponse(BaseModel): + simulatedForecasts: List[WeatherIndeterminate] + + class WF1ForecastRecordType(BaseModel): id: str = "FORECAST" displayLabel: str = "Forecast" diff --git a/web/src/api/moreCast2API.ts b/web/src/api/moreCast2API.ts index f1dee1bf8..ec291fcc2 100644 --- a/web/src/api/moreCast2API.ts +++ b/web/src/api/moreCast2API.ts @@ -245,12 +245,11 @@ export async function fetchCalculatedIndices( ): Promise { const url = 'morecast-v2/simulate-indices/' const determinatesToSimulate = mapMoreCast2RowsToIndeterminates(recordsToSimulate) - const { data } = await axios.post(url, { + const { data } = await axios.post(url, { simulate_records: determinatesToSimulate }) - const response: UpdatedWeatherIndeterminateResponse = { simulatedForecasts: data } - return response + return data } export const mapMoreCast2RowsToIndeterminates = (rows: MoreCast2Row[]): WeatherIndeterminate[] => { diff --git a/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx b/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx index 3e135b6b4..21f638a32 100644 --- a/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx +++ b/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx @@ -1,11 +1,17 @@ import React from 'react' import { styled } from '@mui/material/styles' -import { DataGrid, GridColDef, GridEventListener } from '@mui/x-data-grid' -import { ModelChoice, ModelType } from 'api/moreCast2API' +import { DataGrid, GridColDef, GridEventListener, GridCellEditStopParams } from '@mui/x-data-grid' +import { ModelChoice, ModelType, fetchCalculatedIndices } from 'api/moreCast2API' import { MoreCast2Row } from 'features/moreCast2/interfaces' import { LinearProgress } from '@mui/material' import ApplyToColumnMenu from 'features/moreCast2/components/ApplyToColumnMenu' import { DataGridColumns } from 'features/moreCast2/components/DataGridColumns' +import { isNaN } from 'lodash' +import { rowIDHasher } from 'features/moreCast2/util' +import { validForecastPredicate } from 'features/moreCast2/saveForecasts' +import { updateWeatherIndeterminates } from 'features/moreCast2/slices/dataSlice' +import { AppDispatch } from 'app/store' +import { useDispatch } from 'react-redux' const PREFIX = 'ForecastSummaryDataGrid' @@ -33,6 +39,17 @@ interface ForecastSummaryDataGridProps { handleClose: () => void } +const getYesterdayRowID = (todayRow: MoreCast2Row): string => { + const yesterdayDate = todayRow.forDate.minus({ days: 1 }) + const yesterdayID = rowIDHasher(todayRow.stationCode, yesterdayDate) + + return yesterdayID +} + +const isActualOrValidForecastPredicate = (row: MoreCast2Row) => + validForecastPredicate(row) || + (!isNaN(row.precipActual) && !isNaN(row.rhActual) && !isNaN(row.tempActual) && !isNaN(row.windSpeedActual)) + const ForecastSummaryDataGrid = ({ loading, rows, @@ -42,6 +59,27 @@ const ForecastSummaryDataGrid = ({ handleColumnHeaderClick, handleClose }: ForecastSummaryDataGridProps) => { + const dispatch: AppDispatch = useDispatch() + const handleCellEditStop = async (params: GridCellEditStopParams) => { + const editedRow = params.row + + const mustBeFilled = [ + editedRow.tempForecast?.value, + editedRow.rhForecast?.value, + editedRow.windSpeedForecast?.value, + editedRow.precipForecast?.value + ] + for (const value of mustBeFilled) { + if (isNaN(value)) { + return editedRow + } + } + const idBeforeEditedRow = getYesterdayRowID(editedRow) + const rowsForUpdate = rows.filter(row => row.id >= idBeforeEditedRow).filter(isActualOrValidForecastPredicate) + const data = await fetchCalculatedIndices(rowsForUpdate) + dispatch(updateWeatherIndeterminates(data)) + } + return ( params.row[params.field] !== ModelChoice.ACTUAL} + onCellEditStop={handleCellEditStop} /> ) { + const updatedForecasts = addUniqueIds(action.payload.simulatedForecasts) + + state.forecasts = state.forecasts.map(forecast => { + const updatedForecast = updatedForecasts.find(item => item.id === forecast.id) + return updatedForecast ? updatedForecast : forecast + }) } } }) -export const { getWeatherIndeterminatesStart, getWeatherIndeterminatesFailed, getWeatherIndeterminatesSuccess } = - dataSlice.actions +export const { + getWeatherIndeterminatesStart, + getWeatherIndeterminatesFailed, + getWeatherIndeterminatesSuccess, + updateWeatherIndeterminates +} = dataSlice.actions export default dataSlice.reducer @@ -302,7 +315,7 @@ const forecastOrNull = (determinate: WeatherDeterminateType): ModelChoice.FORECA * @returns Returns an array of WeatherIndeterminates where each item has an ID derived * from its station_code and utc_timestamp. */ -const addUniqueIds = (items: WeatherIndeterminate[]) => { +export const addUniqueIds = (items: WeatherIndeterminate[]) => { return items.map(item => ({ ...item, id: rowIDHasher(item.station_code, DateTime.fromISO(item.utc_timestamp)) From 264b8ff9ff559435d8764ccab3455bc7abcac83b Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Thu, 26 Oct 2023 13:08:17 -0700 Subject: [PATCH 08/36] updates tests --- .../features/moreCast2/saveForecast.test.ts | 67 +++++++++++++------ .../moreCast2/slices/dataSlice.test.ts | 8 +++ 2 files changed, 54 insertions(+), 21 deletions(-) diff --git a/web/src/features/moreCast2/saveForecast.test.ts b/web/src/features/moreCast2/saveForecast.test.ts index 70e242e3c..c008e4e52 100644 --- a/web/src/features/moreCast2/saveForecast.test.ts +++ b/web/src/features/moreCast2/saveForecast.test.ts @@ -83,12 +83,16 @@ const buildCompleteForecast = ( id: string, forDate: DateTime, stationCode: number, - stationName: string + stationName: string, + latitude: number, + longitude: number ): MoreCast2Row => ({ id, forDate, stationCode, stationName, + latitude, + longitude, ...baseRow, precipForecast: { choice: ModelChoice.GDPS, value: 0 }, rhForecast: { choice: ModelChoice.GDPS, value: 0 }, @@ -101,12 +105,16 @@ const buildForecastMissingWindDirection = ( id: string, forDate: DateTime, stationCode: number, - stationName: string + stationName: string, + latitude: number, + longitude: number ): MoreCast2Row => ({ id, forDate, stationCode, stationName, + latitude, + longitude, ...baseRow, precipForecast: { choice: ModelChoice.GDPS, value: 0 }, rhForecast: { choice: ModelChoice.GDPS, value: 0 }, @@ -119,20 +127,33 @@ const buildInvalidForecast = ( id: string, forDate: DateTime, stationCode: number, - stationName: string + stationName: string, + latitude: number, + longitude: number ): MoreCast2Row => ({ id, forDate, stationCode, stationName, + latitude, + longitude, ...baseRow }) -const buildNAForecast = (id: string, forDate: DateTime, stationCode: number, stationName: string): MoreCast2Row => ({ +const buildNAForecast = ( + id: string, + forDate: DateTime, + stationCode: number, + stationName: string, + latitude: number, + longitude: number +): MoreCast2Row => ({ id, forDate, stationCode, stationName, + latitude, + longitude, ...baseRow, precipForecast: { choice: ModelChoice.NULL, value: NaN }, rhForecast: { choice: ModelChoice.NULL, value: NaN }, @@ -145,12 +166,16 @@ const buildForecastWithActuals = ( id: string, forDate: DateTime, stationCode: number, - stationName: string + stationName: string, + latitude: number, + longitude: number ): MoreCast2Row => ({ id, forDate, stationCode, stationName, + latitude, + longitude, ...baseRowWithActuals, precipForecast: { choice: ModelChoice.GDPS, value: 0 }, rhForecast: { choice: ModelChoice.GDPS, value: 0 }, @@ -164,16 +189,16 @@ describe('saveForecasts', () => { it('should return true if all forecasts fields are set', () => { expect( isForecastValid([ - buildCompleteForecast('1', mockForDate, 1, 'one'), - buildCompleteForecast('2', mockForDate, 2, 'two') + buildCompleteForecast('1', mockForDate, 1, 'one', 1, 1), + buildCompleteForecast('2', mockForDate, 2, 'two', 2, 2) ]) ).toBe(true) }) it('should return true if all forecasts fields are set except windDirectionForecast', () => { expect( isForecastValid([ - buildForecastMissingWindDirection('1', mockForDate, 1, 'one'), - buildForecastMissingWindDirection('2', mockForDate, 2, 'two') + buildForecastMissingWindDirection('1', mockForDate, 1, 'one', 1, 1), + buildForecastMissingWindDirection('2', mockForDate, 2, 'two', 2, 2) ]) ).toBe(true) }) @@ -181,47 +206,47 @@ describe('saveForecasts', () => { it('should return false if any forecasts have missing forecast fields', () => { expect( isForecastValid([ - buildCompleteForecast('1', mockForDate, 1, 'one'), - buildInvalidForecast('2', mockForDate, 2, 'two') + buildCompleteForecast('1', mockForDate, 1, 'one', 1, 1), + buildInvalidForecast('2', mockForDate, 2, 'two', 2, 2) ]) ).toBe(false) }) it('should return false if any forecasts have missing forecast fields set other than windDirectionForecast', () => { - expect(isForecastValid([buildNAForecast('1', mockForDate, 2, 'one')])).toBe(false) + expect(isForecastValid([buildNAForecast('1', mockForDate, 2, 'one', 1, 1)])).toBe(false) }) }) describe('validForecastPredicate', () => { it('should return false for a forecast with missing forecast fields', () => { - expect(validForecastPredicate(buildInvalidForecast('1', mockForDate, 1, 'one'))).toBe(false) + expect(validForecastPredicate(buildInvalidForecast('1', mockForDate, 1, 'one', 1, 1))).toBe(false) }) it('should return false for a forecast with forecasts but N/A values', () => { - expect(validForecastPredicate(buildNAForecast('1', mockForDate, 1, 'one'))).toBe(false) + expect(validForecastPredicate(buildNAForecast('1', mockForDate, 1, 'one', 1, 1))).toBe(false) }) }) describe('getRowsToSave', () => { it('should filter out invalid forecasts', () => { const res = getRowsToSave([ - buildCompleteForecast('1', mockForDate, 1, 'one'), - buildInvalidForecast('2', mockForDate, 2, 'two') + buildCompleteForecast('1', mockForDate, 1, 'one', 1, 1), + buildInvalidForecast('2', mockForDate, 2, 'two', 2, 2) ]) expect(res).toHaveLength(1) expect(res[0].id).toBe('1') }) it('should filter out N/A forecasts', () => { const res = getRowsToSave([ - buildCompleteForecast('1', mockForDate, 1, 'one'), - buildNAForecast('2', mockForDate, 2, 'two') + buildCompleteForecast('1', mockForDate, 1, 'one', 1, 1), + buildNAForecast('2', mockForDate, 2, 'two', 2, 2) ]) expect(res).toHaveLength(1) expect(res[0].id).toBe('1') }) it('should filter out rows with actuals', () => { - const forecastWithActual = buildCompleteForecast('2', mockForDate, 2, 'two') + const forecastWithActual = buildCompleteForecast('2', mockForDate, 2, 'two', 2, 2) forecastWithActual.precipActual = 1 const res = getRowsToSave([ - buildCompleteForecast('1', mockForDate, 1, 'one'), - buildForecastWithActuals('2', mockForDate, 2, 'two') + buildCompleteForecast('1', mockForDate, 1, 'one', 1, 1), + buildForecastWithActuals('2', mockForDate, 2, 'two', 2, 2) ]) expect(res).toHaveLength(1) expect(res[0].id).toBe('1') diff --git a/web/src/features/moreCast2/slices/dataSlice.test.ts b/web/src/features/moreCast2/slices/dataSlice.test.ts index fe7253316..e285cdffb 100644 --- a/web/src/features/moreCast2/slices/dataSlice.test.ts +++ b/web/src/features/moreCast2/slices/dataSlice.test.ts @@ -19,6 +19,8 @@ const FROM_DATE_STRING = '2023-04-27T20:00:00+00:00' const TO_DATE_STRING = '2023-04-28T20:00:00+00:00' const FROM_DATE_TIME = DateTime.fromISO(FROM_DATE_STRING) const TO_DATE_TIME = DateTime.fromISO(TO_DATE_STRING) +const LAT = 1.1 +const LONG = 2.2 const PRECIP = 1 const RH = 75 const TEMP = 10 @@ -52,6 +54,8 @@ const weatherIndeterminateGenerator = ( station_name, determinate, utc_timestamp, + latitude: LAT, + longitude: LONG, precipitation: precipValue ?? PRECIP, relative_humidity: RH, temperature: TEMP, @@ -104,6 +108,8 @@ describe('dataSlice', () => { station_name: 'station', determinate: WeatherDeterminate.ACTUAL, utc_timestamp: '2023-04-21', + latitude: 1.1, + longitude: 2.2, precipitation: 0.5, relative_humidity: 55, temperature: 12, @@ -126,6 +132,8 @@ describe('dataSlice', () => { station_name: 'prediction station', determinate: WeatherDeterminate.GDPS, utc_timestamp: '2023-04-22', + latitude: 1.1, + longitude: 2.2, precipitation: 1.5, relative_humidity: 75, temperature: 5, From 9b58a8fb1d97f9cea2f3fc12c5cab044381180c5 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Thu, 26 Oct 2023 13:10:50 -0700 Subject: [PATCH 09/36] docstring --- api/app/routers/morecast_v2.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/app/routers/morecast_v2.py b/api/app/routers/morecast_v2.py index b081a9d71..a6acfb0eb 100644 --- a/api/app/routers/morecast_v2.py +++ b/api/app/routers/morecast_v2.py @@ -230,6 +230,9 @@ async def get_determinates_for_date_range(start_date: date, @router.post('/simulate-indices/', response_model=SimulatedWeatherIndeterminateResponse) async def calculate_forecasted_indices(simulate_records: SimulateIndeterminateIndices): + """ + Returns forecasts with all Fire Weather Index System values calculated using the CFFDRS R library + """ indeterminates = simulate_records.simulate_records logger.info( f'/simulate-indices/ - simulating forecast records for {indeterminates[0].station_name}') From e1b5dca92f132413d6a4f4532c78d92a7a3664be Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Mon, 30 Oct 2023 10:31:15 -0700 Subject: [PATCH 10/36] forecast choice labels - stores edited rows in redux store - maps stored rows to new rows on user edit --- .../components/ForecastSummaryDataGrid.tsx | 17 +++-- .../moreCast2/components/TabbedDataGrid.tsx | 69 +++++++++++++++++-- .../features/moreCast2/slices/dataSlice.ts | 28 +++++++- 3 files changed, 97 insertions(+), 17 deletions(-) diff --git a/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx b/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx index 21f638a32..3e304f129 100644 --- a/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx +++ b/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx @@ -9,7 +9,7 @@ import { DataGridColumns } from 'features/moreCast2/components/DataGridColumns' import { isNaN } from 'lodash' import { rowIDHasher } from 'features/moreCast2/util' import { validForecastPredicate } from 'features/moreCast2/saveForecasts' -import { updateWeatherIndeterminates } from 'features/moreCast2/slices/dataSlice' +import { updateWeatherIndeterminates, storeUserEditedRows } from 'features/moreCast2/slices/dataSlice' import { AppDispatch } from 'app/store' import { useDispatch } from 'react-redux' @@ -46,9 +46,11 @@ const getYesterdayRowID = (todayRow: MoreCast2Row): string => { return yesterdayID } -const isActualOrValidForecastPredicate = (row: MoreCast2Row) => - validForecastPredicate(row) || - (!isNaN(row.precipActual) && !isNaN(row.rhActual) && !isNaN(row.tempActual) && !isNaN(row.windSpeedActual)) +export const validActualPredicate = (row: MoreCast2Row) => + !isNaN(row.precipActual) && !isNaN(row.rhActual) && !isNaN(row.tempActual) && !isNaN(row.windSpeedActual) + +export const isActualOrValidForecastPredicate = (row: MoreCast2Row) => + validForecastPredicate(row) || validActualPredicate(row) const ForecastSummaryDataGrid = ({ loading, @@ -62,6 +64,7 @@ const ForecastSummaryDataGrid = ({ const dispatch: AppDispatch = useDispatch() const handleCellEditStop = async (params: GridCellEditStopParams) => { const editedRow = params.row + dispatch(storeUserEditedRows([editedRow])) const mustBeFilled = [ editedRow.tempForecast?.value, @@ -75,9 +78,9 @@ const ForecastSummaryDataGrid = ({ } } const idBeforeEditedRow = getYesterdayRowID(editedRow) - const rowsForUpdate = rows.filter(row => row.id >= idBeforeEditedRow).filter(isActualOrValidForecastPredicate) - const data = await fetchCalculatedIndices(rowsForUpdate) - dispatch(updateWeatherIndeterminates(data)) + const rowsForSimulation = rows.filter(row => row.id >= idBeforeEditedRow).filter(isActualOrValidForecastPredicate) + const simulatedForecasts = await fetchCalculatedIndices(rowsForSimulation) + dispatch(updateWeatherIndeterminates(simulatedForecasts)) } return ( diff --git a/web/src/features/moreCast2/components/TabbedDataGrid.tsx b/web/src/features/moreCast2/components/TabbedDataGrid.tsx index 60534da6c..3d836f689 100644 --- a/web/src/features/moreCast2/components/TabbedDataGrid.tsx +++ b/web/src/features/moreCast2/components/TabbedDataGrid.tsx @@ -1,14 +1,19 @@ import { AlertColor, List, Stack } from '@mui/material' import { styled } from '@mui/material/styles' import { GridCellParams, GridColDef, GridColumnVisibilityModel, GridEventListener } from '@mui/x-data-grid' -import { ModelChoice, ModelType, submitMoreCastForecastRecords } from 'api/moreCast2API' +import { ModelChoice, ModelType, fetchCalculatedIndices, submitMoreCastForecastRecords } from 'api/moreCast2API' import { DataGridColumns, columnGroupingModel } from 'features/moreCast2/components/DataGridColumns' import ForecastDataGrid from 'features/moreCast2/components/ForecastDataGrid' -import ForecastSummaryDataGrid from 'features/moreCast2/components/ForecastSummaryDataGrid' +import ForecastSummaryDataGrid, { validActualPredicate } from 'features/moreCast2/components/ForecastSummaryDataGrid' import SelectableButton from 'features/moreCast2/components/SelectableButton' -import { selectWeatherIndeterminatesLoading } from 'features/moreCast2/slices/dataSlice' +import { + selectUserEditedRows, + selectWeatherIndeterminatesLoading, + storeUserEditedRows, + updateWeatherIndeterminates +} from 'features/moreCast2/slices/dataSlice' import React, { useEffect, useState } from 'react' -import { useSelector } from 'react-redux' +import { useDispatch, useSelector } from 'react-redux' import { MoreCast2ForecastRow, MoreCast2Row, PredictionItem } from 'features/moreCast2/interfaces' import { selectSelectedStations } from 'features/moreCast2/slices/selectedStationsSlice' import { groupBy, isEqual, isUndefined } from 'lodash' @@ -17,8 +22,15 @@ import { ROLES } from 'features/auth/roles' import { selectAuthentication, selectWf1Authentication } from 'app/rootReducer' import { DateRange } from 'components/dateRangePicker/types' import MoreCast2Snackbar from 'features/moreCast2/components/MoreCast2Snackbar' -import { isForecastRowPredicate, getRowsToSave, isForecastValid } from 'features/moreCast2/saveForecasts' +import { + isForecastRowPredicate, + getRowsToSave, + isForecastValid, + validForecastPredicate +} from 'features/moreCast2/saveForecasts' import MoreCast2DateRangePicker from 'features/moreCast2/components/MoreCast2DateRangePicker' +import { AppDispatch } from 'app/store' +import { deepClone } from '@mui/x-data-grid/utils/utils' export const Root = styled('div')({ display: 'flex', @@ -43,10 +55,12 @@ interface TabbedDataGridProps { } const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProps) => { + const dispatch: AppDispatch = useDispatch() const selectedStations = useSelector(selectSelectedStations) const loading = useSelector(selectWeatherIndeterminatesLoading) const { roles, isAuthenticated } = useSelector(selectAuthentication) const { wf1Token } = useSelector(selectWf1Authentication) + const userEditedRows = useSelector(selectUserEditedRows) // A copy of the sortedMoreCast2Rows as local state const [allRows, setAllRows] = useState(morecast2Rows) @@ -92,6 +106,11 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp setAllRows([...morecast2Rows]) }, [morecast2Rows]) + useEffect(() => { + const labelledRows = mapForecastChoiceLabels(morecast2Rows, deepClone(userEditedRows)) + setAllRows(labelledRows) + }, [userEditedRows, morecast2Rows]) + useEffect(() => { const newVisibleRows: MoreCast2Row[] = [] const stationCodes = selectedStations.map(station => station.station_code) @@ -179,6 +198,34 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp const [clickedColDef, setClickedColDef] = useState(null) + const filterRowsForSimulation = (rows: MoreCast2Row[]): MoreCast2Row[] => { + const forecasts = rows.filter(validForecastPredicate) + const actuals = rows.filter(validActualPredicate) + const mostRecentActual = actuals.pop() + + return mostRecentActual ? [mostRecentActual, ...forecasts] : rows + } + + const mapForecastChoiceLabels = (newRows: MoreCast2Row[], storedRows: MoreCast2Row[]): MoreCast2Row[] => { + const storedRowChoicesMap = new Map() + + for (const row of storedRows) { + storedRowChoicesMap.set(row.id, row) + } + + for (const row of newRows) { + const matchingRow = storedRowChoicesMap.get(row.id) + if (matchingRow) { + row.precipForecast = matchingRow.precipForecast + row.tempForecast = matchingRow.tempForecast + row.rhForecast = matchingRow.rhForecast + row.windDirectionForecast = matchingRow.windDirectionForecast + row.windSpeedForecast = matchingRow.windSpeedForecast + } + } + return newRows + } + // Updates forecast field for a given weather parameter (temp, rh, precip, etc...) based on the // model/source selected in the column header menu const updateColumnWithModel = (modelType: ModelType, colDef: GridColDef) => { @@ -196,7 +243,7 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp // Persistence forecasting. Get the most recent actual and persist it through the rest of the // days in this forecast period. - const updateColumnFromLastActual = (forecastField: keyof MoreCast2Row, actualField: keyof MoreCast2Row) => { + const updateColumnFromLastActual = async (forecastField: keyof MoreCast2Row, actualField: keyof MoreCast2Row) => { const newRows = [...visibleRows] // Group our visible rows by station code and work on each group sepearately const groupedByStationCode = groupBy(newRows, 'stationCode') @@ -219,10 +266,14 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp predictionItem.value = mostRecentValue as number }) } + const rowsForSimulation = filterRowsForSimulation(newRows) + const simulatedForecasts = await fetchCalculatedIndices(rowsForSimulation) + dispatch(storeUserEditedRows(newRows)) + dispatch(updateWeatherIndeterminates(simulatedForecasts)) setVisibleRows(newRows) } - const updateColumnFromModel = ( + const updateColumnFromModel = async ( modelType: ModelType, forecastField: keyof MoreCast2Row, actualField: keyof MoreCast2Row, @@ -240,6 +291,10 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp predictionItem.value = (row[sourceKey] as number) ?? NaN } } + const rowsForSimulation = filterRowsForSimulation(newRows) + const simulatedForecasts = await fetchCalculatedIndices(rowsForSimulation) + dispatch(storeUserEditedRows(newRows)) + dispatch(updateWeatherIndeterminates(simulatedForecasts)) setVisibleRows(newRows) } diff --git a/web/src/features/moreCast2/slices/dataSlice.ts b/web/src/features/moreCast2/slices/dataSlice.ts index c7264327f..1d05624c1 100644 --- a/web/src/features/moreCast2/slices/dataSlice.ts +++ b/web/src/features/moreCast2/slices/dataSlice.ts @@ -24,6 +24,7 @@ interface State { actuals: WeatherIndeterminate[] forecasts: WeatherIndeterminate[] predictions: WeatherIndeterminate[] + userEditedRows: MoreCast2Row[] } export const initialState: State = { @@ -31,7 +32,8 @@ export const initialState: State = { error: null, actuals: [], forecasts: [], - predictions: [] + predictions: [], + userEditedRows: [] } const dataSlice = createSlice({ @@ -43,6 +45,7 @@ const dataSlice = createSlice({ state.actuals = [] state.forecasts = [] state.predictions = [] + state.userEditedRows = [] state.loading = true }, getWeatherIndeterminatesFailed(state: State, action: PayloadAction) { @@ -61,8 +64,21 @@ const dataSlice = createSlice({ state.forecasts = state.forecasts.map(forecast => { const updatedForecast = updatedForecasts.find(item => item.id === forecast.id) - return updatedForecast ? updatedForecast : forecast + return updatedForecast || forecast }) + }, + storeUserEditedRows(state: State, action: PayloadAction) { + const storedRows = [...state.userEditedRows] + + for (const row of action.payload) { + const existingIndex = storedRows.findIndex(storedRow => storedRow.id === row.id) + if (existingIndex !== -1) { + storedRows[existingIndex] = row + } else { + storedRows.push(row) + } + } + state.userEditedRows = storedRows } } }) @@ -71,7 +87,8 @@ export const { getWeatherIndeterminatesStart, getWeatherIndeterminatesFailed, getWeatherIndeterminatesSuccess, - updateWeatherIndeterminates + updateWeatherIndeterminates, + storeUserEditedRows } = dataSlice.actions export default dataSlice.reducer @@ -472,6 +489,11 @@ export const selectAllMoreCast2Rows = createSelector([selectWeatherIndeterminate return sortedRows }) +export const selectUserEditedRows = createSelector([selectWeatherIndeterminates], weatherIndeterminates => { + const rows = weatherIndeterminates.userEditedRows + return rows +}) + export const selectForecastMoreCast2Rows = createSelector([selectAllMoreCast2Rows], allMorecast2Rows => allMorecast2Rows?.map(row => ({ id: row.id, From 19c7057ce7cc21fa4e225980063a0ff22f19a408 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Mon, 30 Oct 2023 12:22:23 -0700 Subject: [PATCH 11/36] moves api call to func, adds success/fail slice --- .../components/ForecastSummaryDataGrid.tsx | 7 ++-- .../moreCast2/components/TabbedDataGrid.tsx | 12 +++---- .../features/moreCast2/slices/dataSlice.ts | 32 ++++++++++++++++--- 3 files changed, 36 insertions(+), 15 deletions(-) diff --git a/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx b/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx index 3e304f129..294bed4d6 100644 --- a/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx +++ b/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx @@ -1,7 +1,7 @@ import React from 'react' import { styled } from '@mui/material/styles' import { DataGrid, GridColDef, GridEventListener, GridCellEditStopParams } from '@mui/x-data-grid' -import { ModelChoice, ModelType, fetchCalculatedIndices } from 'api/moreCast2API' +import { ModelChoice, ModelType } from 'api/moreCast2API' import { MoreCast2Row } from 'features/moreCast2/interfaces' import { LinearProgress } from '@mui/material' import ApplyToColumnMenu from 'features/moreCast2/components/ApplyToColumnMenu' @@ -9,7 +9,7 @@ import { DataGridColumns } from 'features/moreCast2/components/DataGridColumns' import { isNaN } from 'lodash' import { rowIDHasher } from 'features/moreCast2/util' import { validForecastPredicate } from 'features/moreCast2/saveForecasts' -import { updateWeatherIndeterminates, storeUserEditedRows } from 'features/moreCast2/slices/dataSlice' +import { storeUserEditedRows, getSimulatedIndices } from 'features/moreCast2/slices/dataSlice' import { AppDispatch } from 'app/store' import { useDispatch } from 'react-redux' @@ -79,8 +79,7 @@ const ForecastSummaryDataGrid = ({ } const idBeforeEditedRow = getYesterdayRowID(editedRow) const rowsForSimulation = rows.filter(row => row.id >= idBeforeEditedRow).filter(isActualOrValidForecastPredicate) - const simulatedForecasts = await fetchCalculatedIndices(rowsForSimulation) - dispatch(updateWeatherIndeterminates(simulatedForecasts)) + dispatch(getSimulatedIndices(rowsForSimulation)) } return ( diff --git a/web/src/features/moreCast2/components/TabbedDataGrid.tsx b/web/src/features/moreCast2/components/TabbedDataGrid.tsx index 3d836f689..a804aa316 100644 --- a/web/src/features/moreCast2/components/TabbedDataGrid.tsx +++ b/web/src/features/moreCast2/components/TabbedDataGrid.tsx @@ -1,16 +1,16 @@ import { AlertColor, List, Stack } from '@mui/material' import { styled } from '@mui/material/styles' import { GridCellParams, GridColDef, GridColumnVisibilityModel, GridEventListener } from '@mui/x-data-grid' -import { ModelChoice, ModelType, fetchCalculatedIndices, submitMoreCastForecastRecords } from 'api/moreCast2API' +import { ModelChoice, ModelType, submitMoreCastForecastRecords } from 'api/moreCast2API' import { DataGridColumns, columnGroupingModel } from 'features/moreCast2/components/DataGridColumns' import ForecastDataGrid from 'features/moreCast2/components/ForecastDataGrid' import ForecastSummaryDataGrid, { validActualPredicate } from 'features/moreCast2/components/ForecastSummaryDataGrid' import SelectableButton from 'features/moreCast2/components/SelectableButton' import { + getSimulatedIndices, selectUserEditedRows, selectWeatherIndeterminatesLoading, - storeUserEditedRows, - updateWeatherIndeterminates + storeUserEditedRows } from 'features/moreCast2/slices/dataSlice' import React, { useEffect, useState } from 'react' import { useDispatch, useSelector } from 'react-redux' @@ -267,9 +267,8 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp }) } const rowsForSimulation = filterRowsForSimulation(newRows) - const simulatedForecasts = await fetchCalculatedIndices(rowsForSimulation) dispatch(storeUserEditedRows(newRows)) - dispatch(updateWeatherIndeterminates(simulatedForecasts)) + dispatch(getSimulatedIndices(rowsForSimulation)) setVisibleRows(newRows) } @@ -292,9 +291,8 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp } } const rowsForSimulation = filterRowsForSimulation(newRows) - const simulatedForecasts = await fetchCalculatedIndices(rowsForSimulation) dispatch(storeUserEditedRows(newRows)) - dispatch(updateWeatherIndeterminates(simulatedForecasts)) + dispatch(getSimulatedIndices(rowsForSimulation)) setVisibleRows(newRows) } diff --git a/web/src/features/moreCast2/slices/dataSlice.ts b/web/src/features/moreCast2/slices/dataSlice.ts index 1d05624c1..51e3ad5d2 100644 --- a/web/src/features/moreCast2/slices/dataSlice.ts +++ b/web/src/features/moreCast2/slices/dataSlice.ts @@ -8,7 +8,8 @@ import { WeatherDeterminate, WeatherDeterminateChoices, WeatherDeterminateType, - UpdatedWeatherIndeterminateResponse + UpdatedWeatherIndeterminateResponse, + fetchCalculatedIndices } from 'api/moreCast2API' import { AppThunk } from 'app/store' import { createDateInterval, rowIDHasher } from 'features/moreCast2/util' @@ -59,7 +60,7 @@ const dataSlice = createSlice({ state.predictions = action.payload.predictions state.loading = false }, - updateWeatherIndeterminates(state: State, action: PayloadAction) { + simulateWeatherIndeterminatesSuccess(state: State, action: PayloadAction) { const updatedForecasts = addUniqueIds(action.payload.simulatedForecasts) state.forecasts = state.forecasts.map(forecast => { @@ -67,6 +68,9 @@ const dataSlice = createSlice({ return updatedForecast || forecast }) }, + simulateWeatherIndeterminatesFailed(state: State, action: PayloadAction) { + state.error = action.payload + }, storeUserEditedRows(state: State, action: PayloadAction) { const storedRows = [...state.userEditedRows] @@ -87,7 +91,8 @@ export const { getWeatherIndeterminatesStart, getWeatherIndeterminatesFailed, getWeatherIndeterminatesSuccess, - updateWeatherIndeterminates, + simulateWeatherIndeterminatesSuccess, + simulateWeatherIndeterminatesFailed, storeUserEditedRows } = dataSlice.actions @@ -99,7 +104,7 @@ export default dataSlice.reducer * @param stations The list of stations to retreive data for. * @param fromDate The start date from which to retrieve data from (inclusive). * @param toDate The end date from which to retrieve data from (inclusive). - * @returns An array or WeatherIndeterminates. + * @returns An array of WeatherIndeterminates. */ export const getWeatherIndeterminates = (stations: StationGroupMember[], fromDate: DateTime, toDate: DateTime): AppThunk => @@ -141,6 +146,25 @@ export const getWeatherIndeterminates = } } +/** + * Use the morecast2API to get simulated Fire Weather Index value from the backend. + * Results are stored the Redux store. + * @param rowsForSimulation List of MoreCast2Row's to simulate. The first row in the array must contain + * valid values for all Fire Weather Indices. + * @returns Array of MoreCast2Rows + */ +export const getSimulatedIndices = + (rowsForSimulation: MoreCast2Row[]): AppThunk => + async dispatch => { + try { + const simulatedForecasts = await fetchCalculatedIndices(rowsForSimulation) + dispatch(simulateWeatherIndeterminatesSuccess(simulatedForecasts)) + } catch (err) { + dispatch(simulateWeatherIndeterminatesFailed((err as Error).toString())) + logError(err) + } + } + export const createMoreCast2Rows = ( actuals: WeatherIndeterminate[], forecasts: WeatherIndeterminate[], From 867014418a418c796e81b8a31baed8ad6681e3ab Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Mon, 30 Oct 2023 14:55:12 -0700 Subject: [PATCH 12/36] backend - use single function for fwi calculations - calculate ALL fwi values where possible --- api/app/morecast_v2/forecasts.py | 47 +++++++++++++++++--------------- api/app/routers/morecast_v2.py | 45 ++++-------------------------- 2 files changed, 31 insertions(+), 61 deletions(-) diff --git a/api/app/morecast_v2/forecasts.py b/api/app/morecast_v2/forecasts.py index 553ed43b4..5b732b94c 100644 --- a/api/app/morecast_v2/forecasts.py +++ b/api/app/morecast_v2/forecasts.py @@ -7,11 +7,11 @@ from collections import defaultdict from app.utils.time import vancouver_tz -from typing import List, Optional +from typing import List, Optional, Tuple from sqlalchemy.orm import Session from app.db.crud.morecast_v2 import get_forecasts_in_range from app.db.models.morecast_v2 import MorecastForecastRecord -from app.schemas.morecast_v2 import MoreCastForecastOutput, StationDailyFromWF1, WF1ForecastRecordType, WF1PostForecast, WeatherIndeterminate +from app.schemas.morecast_v2 import MoreCastForecastOutput, StationDailyFromWF1, WF1ForecastRecordType, WF1PostForecast, WeatherIndeterminate, WeatherDeterminate from app.wildfire_one.schema_parsers import WFWXWeatherStation from app.wildfire_one.wfwx_api import get_auth_header, get_forecasts_for_stations_by_date_range, get_wfwx_stations_from_station_codes from app.fire_behaviour import cffdrs @@ -108,36 +108,39 @@ def filter_for_api_forecasts(forecasts: List[WeatherIndeterminate], actuals: Lis return filtered_forecasts -def get_forecasted_fwi_values(actuals: List[WeatherIndeterminate], forecasts: List[WeatherIndeterminate]) -> List[WeatherIndeterminate]: +def get_fwi_values(actuals: List[WeatherIndeterminate], forecasts: List[WeatherIndeterminate]) -> Tuple[List[WeatherIndeterminate], List[WeatherIndeterminate]]: """ - Fills forecasts with Fire Weather Index System values by calculating based off previous actuals and subsequent forecasts. + Calculates actuals and forecasts with Fire Weather Index System values by calculating based off previous actuals and subsequent forecasts. :param actuals: List of actual weather values :type actuals: List[WeatherIndeterminate] :param forecasts: List of existing forecasted values :type forecasts: List[WeatherIndeterminate] - :return: Updated and filled forecasts - :rtype: List[WeatherIndeterminate] + :return: Actuals and forecasts with calculated fire weather index values + :rtype: Tuple[List[WeatherIndeterminate], List[WeatherIndeterminate] """ - actuals_dict = defaultdict(dict) - for actual in actuals: - actuals_dict[actual.station_code][actual.utc_timestamp.date()] = actual - - previous_indeterminate = None - for idx, forecast in enumerate(forecasts): - if previous_indeterminate and previous_indeterminate.station_code == forecast.station_code: - updated_forecast = calculate_fwi_indices(previous_indeterminate, forecast) - forecasts[idx] = updated_forecast - else: - last_actual = actuals_dict[forecast.station_code][forecast.utc_timestamp.date() - timedelta(days=1)] - updated_forecast = calculate_fwi_indices(last_actual, forecast) - forecasts[idx] = updated_forecast - previous_indeterminate = forecast + all_indeterminates = actuals + forecasts + indeterminates_dict = defaultdict(dict) - return forecasts + # Shape indeterminates into nested dicts for quick and easy look ups by station code and date + for indeterminate in all_indeterminates: + indeterminates_dict[indeterminate.station_code][indeterminate.utc_timestamp.date()] = indeterminate + + for idx, indeterminate in enumerate(all_indeterminates): + last_indeterminate = indeterminates_dict[indeterminate.station_code].get( + indeterminate.utc_timestamp.date() - timedelta(days=1), None) + if last_indeterminate: + updated_forecast = calculate_fwi_values(last_indeterminate, indeterminate) + all_indeterminates[idx] = updated_forecast + + forecasts = [indeterminate for indeterminate in all_indeterminates if indeterminate.determinate == + WeatherDeterminate.FORECAST] + actuals = [indeterminate for indeterminate in all_indeterminates if indeterminate.determinate == WeatherDeterminate.ACTUAL] + + return actuals, forecasts -def calculate_fwi_indices(yesterday: WeatherIndeterminate, today: WeatherIndeterminate) -> WeatherIndeterminate: +def calculate_fwi_values(yesterday: WeatherIndeterminate, today: WeatherIndeterminate) -> WeatherIndeterminate: """ Uses CFFDRS library to calculate Fire Weather Index System values diff --git a/api/app/routers/morecast_v2.py b/api/app/routers/morecast_v2.py index a6acfb0eb..b6bac26df 100644 --- a/api/app/routers/morecast_v2.py +++ b/api/app/routers/morecast_v2.py @@ -13,7 +13,7 @@ from app.db.crud.morecast_v2 import get_forecasts_in_range, get_user_forecasts_for_date, save_all_forecasts from app.db.database import get_read_session_scope, get_write_session_scope from app.db.models.morecast_v2 import MorecastForecastRecord -from app.morecast_v2.forecasts import filter_for_api_forecasts, get_forecasts +from app.morecast_v2.forecasts import filter_for_api_forecasts, get_forecasts, calculate_fwi_values, get_fwi_values from app.schemas.morecast_v2 import (IndeterminateDailiesResponse, MoreCastForecastOutput, MoreCastForecastRequest, @@ -31,8 +31,6 @@ get_dailies_for_stations_and_date, get_daily_determinates_for_stations_and_date, get_wfwx_stations_from_station_codes) from app.wildfire_one.wfwx_post_api import post_forecasts -from app.morecast_v2.forecasts import get_forecasted_fwi_values -from app.fire_behaviour import cffdrs logger = logging.getLogger(__name__) @@ -196,7 +194,7 @@ async def get_determinates_for_date_range(start_date: date, end_date_of_interest, unique_station_codes) - wf1_forecasts = get_forecasted_fwi_values(wf1_actuals, wf1_forecasts) + wf1_actuals, wf1_forecasts = get_fwi_values(wf1_actuals, wf1_forecasts) # Find the min and max dates for actuals from wf1. These define the range of dates for which # we need to retrieve forecasts from our API database. Note that not all stations report actuals # at the same time, so every station won't necessarily have an actual for each date in the range. @@ -239,39 +237,8 @@ async def calculate_forecasted_indices(simulate_records: SimulateIndeterminateIn yesterday_record = indeterminates[0] indeterminates_to_simulate = indeterminates[1:] - for record in indeterminates_to_simulate: - # weather params for calculation date - month_to_calculate_for = int(record.utc_timestamp.strftime('%m')) - latitude = record.latitude - temp = record.temperature - rh = record.relative_humidity - precip = record.precipitation - wind_speed = record.wind_speed - record.fine_fuel_moisture_code = cffdrs.fine_fuel_moisture_code(ffmc=yesterday_record.fine_fuel_moisture_code, - temperature=temp, - relative_humidity=rh, - precipitation=precip, - wind_speed=wind_speed) - record.duff_moisture_code = cffdrs.duff_moisture_code(dmc=yesterday_record.duff_moisture_code, - temperature=temp, - relative_humidity=rh, - precipitation=precip, - latitude=latitude, - month=month_to_calculate_for, - latitude_adjust=True - ) - record.drought_code = cffdrs.drought_code(dc=yesterday_record.drought_code, - temperature=temp, - relative_humidity=rh, - precipitation=precip, - latitude=latitude, - month=month_to_calculate_for, - latitude_adjust=True - ) - record.initial_spread_index = cffdrs.initial_spread_index(ffmc=record.fine_fuel_moisture_code, - wind_speed=record.wind_speed) - record.build_up_index = cffdrs.bui_calc(dmc=record.duff_moisture_code, dc=record.drought_code) - record.fire_weather_index = cffdrs.fire_weather_index( - isi=record.initial_spread_index, bui=record.build_up_index) - yesterday_record = record + for idx, record in enumerate(indeterminates_to_simulate): + calculated_record = calculate_fwi_values(yesterday_record, record) + indeterminates_to_simulate[idx] = calculated_record + yesterday_record = calculated_record return (SimulatedWeatherIndeterminateResponse(simulatedForecasts=indeterminates_to_simulate)) From abae508e854ca0d73adbfaac20797add109a4e59 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Mon, 30 Oct 2023 15:04:27 -0700 Subject: [PATCH 13/36] Get extra day of determinates for fwi calc --- api/app/routers/morecast_v2.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/api/app/routers/morecast_v2.py b/api/app/routers/morecast_v2.py index b6bac26df..ef1018235 100644 --- a/api/app/routers/morecast_v2.py +++ b/api/app/routers/morecast_v2.py @@ -184,17 +184,23 @@ async def get_determinates_for_date_range(start_date: date, end_time = vancouver_tz.localize(datetime.combine(end_date, time.max)) start_date_of_interest = get_hour_20_from_date(start_date) end_date_of_interest = get_hour_20_from_date(end_date) + start_date_for_fwi_calc = start_date_of_interest - timedelta(days=1) async with ClientSession() as session: header = await get_auth_header(session) # get station information from the wfwx api wfwx_stations = await get_wfwx_stations_from_station_codes(session, header, unique_station_codes) wf1_actuals, wf1_forecasts = await get_daily_determinates_for_stations_and_date(session, header, - start_date_of_interest, + start_date_for_fwi_calc, end_date_of_interest, unique_station_codes) wf1_actuals, wf1_forecasts = get_fwi_values(wf1_actuals, wf1_forecasts) + + # drop the days before the date of interest that were needed to calculate fwi values + wf1_actuals = [actual for actual in wf1_actuals if actual.utc_timestamp >= start_date_of_interest] + wf1_forecasts = [forecast for forecast in wf1_forecasts if forecast.utc_timestamp >= start_date_of_interest] + # Find the min and max dates for actuals from wf1. These define the range of dates for which # we need to retrieve forecasts from our API database. Note that not all stations report actuals # at the same time, so every station won't necessarily have an actual for each date in the range. From 068468760425c27af76459abfdbec96239ea9f49 Mon Sep 17 00:00:00 2001 From: Conor Brady Date: Tue, 31 Oct 2023 10:49:45 -0700 Subject: [PATCH 14/36] Fix router test, handle empty dates --- api/app/morecast_v2/forecasts.py | 5 +- api/app/routers/morecast_v2.py | 4 +- api/app/tests/fixtures/wf1/lookup.json | 3 + ..._endingTimestamp_1679256000000_statio.json | 581 ++++++++++++++++++ 4 files changed, 590 insertions(+), 3 deletions(-) create mode 100644 api/app/tests/fixtures/wf1/wfwx/v1/dailies/search__findDailiesByStationIdIsInAndWeatherTimestampBetweenOrderByStationIdAscWeatherTimestampAsc__size_1000_page_0_startingTimestamp_1678824000000_endingTimestamp_1679256000000_statio.json diff --git a/api/app/morecast_v2/forecasts.py b/api/app/morecast_v2/forecasts.py index 5b732b94c..b8a2e18ab 100644 --- a/api/app/morecast_v2/forecasts.py +++ b/api/app/morecast_v2/forecasts.py @@ -17,7 +17,10 @@ from app.fire_behaviour import cffdrs -def get_forecasts(db_session: Session, start_time: datetime, end_time: datetime, station_codes: List[int]) -> List[MoreCastForecastOutput]: +def get_forecasts(db_session: Session, start_time: Optional[datetime], end_time: Optional[datetime], station_codes: List[int]) -> List[MoreCastForecastOutput]: + if start_time is None or end_time is None: + return [] + result = get_forecasts_in_range(db_session, start_time, end_time, station_codes) forecasts: List[WeatherIndeterminate] = [MoreCastForecastOutput(station_code=forecast.station_code, diff --git a/api/app/routers/morecast_v2.py b/api/app/routers/morecast_v2.py index ef1018235..08da8025d 100644 --- a/api/app/routers/morecast_v2.py +++ b/api/app/routers/morecast_v2.py @@ -205,8 +205,8 @@ async def get_determinates_for_date_range(start_date: date, # we need to retrieve forecasts from our API database. Note that not all stations report actuals # at the same time, so every station won't necessarily have an actual for each date in the range. wf1_actuals_dates = [actual.utc_timestamp for actual in wf1_actuals] - min_wf1_actuals_date = min(wf1_actuals_dates) - max_wf1_actuals_date = max(wf1_actuals_dates) + min_wf1_actuals_date = min(wf1_actuals_dates, default=None) + max_wf1_actuals_date = max(wf1_actuals_dates, default=None) with get_read_session_scope() as db_session: forecasts_from_db: List[MoreCastForecastOutput] = get_forecasts( diff --git a/api/app/tests/fixtures/wf1/lookup.json b/api/app/tests/fixtures/wf1/lookup.json index 5b074a399..6d6a265c9 100644 --- a/api/app/tests/fixtures/wf1/lookup.json +++ b/api/app/tests/fixtures/wf1/lookup.json @@ -124,6 +124,9 @@ }, "{'size': '1000', 'page': 0, 'startingTimestamp': 1678910400000, 'endingTimestamp': 1679256000000, 'stationIds': ['bfe0a6e2-e269-0210-e053-259e228e58c7', 'bfe0a6e2-e26b-0210-e053-259e228e58c7', 'bfe0a6e2-e3bc-0210-e053-259e228e58c7']}": { "None": "wfwx/v1/dailies/search__findDailiesByStationIdIsInAndWeatherTimestampBetweenOrderByStationIdAscWeatherTimestampAsc__size_1000_page_0_startingTimestamp_1678910400000_endingTimestamp_1679256000000_statio.json" + }, + "{'size': '1000', 'page': 0, 'startingTimestamp': 1678824000000, 'endingTimestamp': 1679256000000, 'stationIds': ['bfe0a6e2-e269-0210-e053-259e228e58c7', 'bfe0a6e2-e26b-0210-e053-259e228e58c7', 'bfe0a6e2-e3bc-0210-e053-259e228e58c7']}": { + "None": "wfwx/v1/dailies/search__findDailiesByStationIdIsInAndWeatherTimestampBetweenOrderByStationIdAscWeatherTimestampAsc__size_1000_page_0_startingTimestamp_1678824000000_endingTimestamp_1679256000000_statio.json" } } } diff --git a/api/app/tests/fixtures/wf1/wfwx/v1/dailies/search__findDailiesByStationIdIsInAndWeatherTimestampBetweenOrderByStationIdAscWeatherTimestampAsc__size_1000_page_0_startingTimestamp_1678824000000_endingTimestamp_1679256000000_statio.json b/api/app/tests/fixtures/wf1/wfwx/v1/dailies/search__findDailiesByStationIdIsInAndWeatherTimestampBetweenOrderByStationIdAscWeatherTimestampAsc__size_1000_page_0_startingTimestamp_1678824000000_endingTimestamp_1679256000000_statio.json new file mode 100644 index 000000000..26d4eebff --- /dev/null +++ b/api/app/tests/fixtures/wf1/wfwx/v1/dailies/search__findDailiesByStationIdIsInAndWeatherTimestampBetweenOrderByStationIdAscWeatherTimestampAsc__size_1000_page_0_startingTimestamp_1678824000000_endingTimestamp_1679256000000_statio.json @@ -0,0 +1,581 @@ +{ + "_embedded": { + "dailies": [ + { + "id": "9dccebf5-c0b0-49ad-9074-cfde7e6d71f9", + "createdBy": "GPEARCE", + "lastEntityUpdateTimestamp": 1665087010888, + "updateDate": "2022-10-06T20:10:10.000+0000", + "lastModifiedBy": "WFWX_WEATHER_API", + "archive": false, + "station": "https://bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/stations/bfe0a6e2-e269-0210-e053-259e228e58c7", + "stationId": "bfe0a6e2-e269-0210-e053-259e228e58c7", + "stationData": { + "id": "bfe0a6e2-e269-0210-e053-259e228e58c7", + "displayLabel": "ALEXIS CREEK", + "createdBy": "LEGACY_DATA_LOAD", + "lastModifiedBy": "WFWX_WEATHER_API", + "lastEntityUpdateTimestamp": 1677961862675, + "updateDate": "2023-03-04T20:31:02.000+0000", + "stationCode": 209, + "stationAcronym": "FAC", + "stationStatus": { + "id": "ACTIVE", + "displayLabel": "Active", + "displayOrder": 1, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "dataSource": { + "id": "AUTO_CALLR", + "displayLabel": "Auto-Caller", + "displayOrder": 1, + "effectiveDate": "1950-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "dataLoggerType": { + "id": "FWS12S", + "displayLabel": "FWS12S", + "displayOrder": 6, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "siteType": { + "id": "WXSTN_GOES", + "displayLabel": "Weather Station - GOES", + "displayOrder": 5, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "weatherZone": { + "id": 17, + "displayLabel": "Zone 17", + "dangerRegion": 2, + "displayOrder": 17 + }, + "stationAccessTypeCode": { + "id": "4WD", + "displayLabel": "4 Wheel Drive", + "displayOrder": 1, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "hourly": true, + "pluvioSnowGuage": false, + "latitude": 52.08377, + "longitude": -123.2732667, + "geometry": { + "crs": { + "type": "name", + "properties": { + "name": "EPSG:4326" + } + }, + "coordinates": [ + -123.2732667, + 52.08377 + ], + "type": "Point" + }, + "elevation": 791, + "windspeedAdjRoughness": 1.0, + "windspeedHeight": { + "id": "9.0_11.9", + "displayLabel": "9.0 - 11.9 (m)", + "displayOrder": 7, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "surfaceType": null, + "overWinterPrecipAdj": true, + "influencingSlope": 2, + "installationDate": 315558000000, + "decommissionDate": null, + "influencingAspect": null, + "owner": { + "id": "a9843bb7-0f44-4a80-8a62-79cca2a5c758", + "displayLabel": "BC Wildfire Service", + "displayOrder": 2, + "ownerName": "BCWS", + "ownerDisplayName": "BC Wildfire Service", + "ownerComment": null, + "createdBy": "DATA_LOAD", + "lastModifiedBy": "DATAFIX - WFWX-891", + "effectiveDate": "1950-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "fireCentre": { + "id": 2, + "displayLabel": "Cariboo Fire Centre", + "alias": 7, + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "zone": { + "id": 7, + "displayLabel": "Chilcotin Zone", + "alias": 5, + "fireCentre": "Cariboo Fire Centre", + "fireCentreAlias": 7, + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "comments": "1st Daily 19800501; coordinates updated 2007/10/03 (P2)", + "ecccFileBaseUrlTemplate": null, + "lastProcessedSourceTime": "2023-03-04T12:31:02", + "crdStationName": null, + "stationAccessDescription": "Alexis creek Wx station is located 150M N/W of the Chilcolin FLNRO field office. Head west on Highway 20 from Williams Lake for an hour and thirty minutes or 112 Kilometers turning right on to Stum Lake Road for 100m and turn right again." + }, + "weatherTimestamp": 1665086400000, + "temperature": 18.7, + "dewPoint": 5.8, + "temperatureMin": null, + "temperatureMax": null, + "relativeHumidity": 43.0, + "relativeHumidityMin": null, + "relativeHumidityMax": null, + "windSpeed": 6.2, + "adjustedWindSpeed": 6.2, + "precipitation": 0.0, + "dangerForest": 4, + "dangerGrassland": 1, + "dangerScrub": 5, + "recordType": { + "id": "ACTUAL", + "displayLabel": "Actual", + "displayOrder": 1, + "createdBy": "DATA_LOAD", + "lastModifiedBy": "DATA_LOAD" + }, + "windDirection": 163.0, + "fineFuelMoistureCode": 89.849, + "duffMoistureCode": 117.819, + "droughtCode": 874.588, + "initialSpreadIndex": 5.734, + "buildUpIndex": 176.272, + "fireWeatherIndex": 26.261, + "dailySeverityRating": null, + "grasslandCuring": null, + "observationValidInd": true, + "observationValidComment": null, + "missingHourlyData": false, + "previousState": null, + "businessKey": "1665086400000-bfe0a6e2-e269-0210-e053-259e228e58c7", + "_links": { + "self": { + "href": "https://bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/dailies/9dccebf5-c0b0-49ad-9074-cfde7e6d71f9" + }, + "daily": { + "href": "https://bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/dailies/9dccebf5-c0b0-49ad-9074-cfde7e6d71f9" + }, + "station": { + "href": "https://bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/stations/bfe0a6e2-e269-0210-e053-259e228e58c7" + } + } + }, + { + "id": "535ce76b-e485-41a8-bd42-7400bc98ae83", + "createdBy": "GPEARCE", + "lastEntityUpdateTimestamp": 1665087635657, + "updateDate": "2022-10-06T20:20:35.000+0000", + "lastModifiedBy": "WFWX_WEATHER_API", + "archive": false, + "station": "https://bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/stations/bfe0a6e2-e26b-0210-e053-259e228e58c7", + "stationId": "bfe0a6e2-e26b-0210-e053-259e228e58c7", + "stationData": { + "id": "bfe0a6e2-e26b-0210-e053-259e228e58c7", + "displayLabel": "NAZKO", + "createdBy": "LEGACY_DATA_LOAD", + "lastModifiedBy": "WFWX_WEATHER_API", + "lastEntityUpdateTimestamp": 1677961870471, + "updateDate": "2023-03-04T20:31:10.000+0000", + "stationCode": 211, + "stationAcronym": "FNZ", + "stationStatus": { + "id": "ACTIVE", + "displayLabel": "Active", + "displayOrder": 1, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "dataSource": { + "id": "AUTO_CALLR", + "displayLabel": "Auto-Caller", + "displayOrder": 1, + "effectiveDate": "1950-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "dataLoggerType": { + "id": "FWS12S", + "displayLabel": "FWS12S", + "displayOrder": 6, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "siteType": { + "id": "WXSTN_GOES", + "displayLabel": "Weather Station - GOES", + "displayOrder": 5, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "weatherZone": { + "id": 17, + "displayLabel": "Zone 17", + "dangerRegion": 2, + "displayOrder": 17 + }, + "stationAccessTypeCode": { + "id": "4WD", + "displayLabel": "4 Wheel Drive", + "displayOrder": 1, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "hourly": true, + "pluvioSnowGuage": false, + "latitude": 52.9575, + "longitude": -123.5958, + "geometry": { + "crs": { + "type": "name", + "properties": { + "name": "EPSG:4326" + } + }, + "coordinates": [ + -123.5958, + 52.9575 + ], + "type": "Point" + }, + "elevation": 910, + "windspeedAdjRoughness": 1.0, + "windspeedHeight": { + "id": "9.0_11.9", + "displayLabel": "9.0 - 11.9 (m)", + "displayOrder": 7, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "surfaceType": null, + "overWinterPrecipAdj": true, + "influencingSlope": 0, + "installationDate": 801122400000, + "decommissionDate": null, + "influencingAspect": null, + "owner": { + "id": "a9843bb7-0f44-4a80-8a62-79cca2a5c758", + "displayLabel": "BC Wildfire Service", + "displayOrder": 2, + "ownerName": "BCWS", + "ownerDisplayName": "BC Wildfire Service", + "ownerComment": null, + "createdBy": "DATA_LOAD", + "lastModifiedBy": "DATAFIX - WFWX-891", + "effectiveDate": "1950-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "fireCentre": { + "id": 2, + "displayLabel": "Cariboo Fire Centre", + "alias": 7, + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "zone": { + "id": 3, + "displayLabel": "Quesnel Zone", + "alias": 1, + "fireCentre": "Cariboo Fire Centre", + "fireCentreAlias": 7, + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "comments": "1st daily 19810501; CRMP- GOES NESID BCF594CA F6 LOGGER (1M SP)", + "ecccFileBaseUrlTemplate": null, + "lastProcessedSourceTime": "2023-03-04T12:31:10", + "crdStationName": null, + "stationAccessDescription": "Nazko highway out of Quesnel." + }, + "weatherTimestamp": 1665086400000, + "temperature": 15.6, + "dewPoint": 6.4, + "temperatureMin": null, + "temperatureMax": null, + "relativeHumidity": 54.0, + "relativeHumidityMin": null, + "relativeHumidityMax": null, + "windSpeed": 5.9, + "adjustedWindSpeed": 5.9, + "precipitation": 0.0, + "dangerForest": 3, + "dangerGrassland": 1, + "dangerScrub": 5, + "recordType": { + "id": "ACTUAL", + "displayLabel": "Actual", + "displayOrder": 1, + "createdBy": "DATA_LOAD", + "lastModifiedBy": "DATA_LOAD" + }, + "windDirection": 318.0, + "fineFuelMoistureCode": 87.996, + "duffMoistureCode": 60.072, + "droughtCode": 600.487, + "initialSpreadIndex": 4.329, + "buildUpIndex": 96.108, + "fireWeatherIndex": 17.102, + "dailySeverityRating": null, + "grasslandCuring": null, + "observationValidInd": true, + "observationValidComment": null, + "missingHourlyData": false, + "previousState": null, + "businessKey": "1665086400000-bfe0a6e2-e26b-0210-e053-259e228e58c7", + "_links": { + "self": { + "href": "https://bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/dailies/535ce76b-e485-41a8-bd42-7400bc98ae83" + }, + "daily": { + "href": "https://bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/dailies/535ce76b-e485-41a8-bd42-7400bc98ae83" + }, + "station": { + "href": "https://bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/stations/bfe0a6e2-e26b-0210-e053-259e228e58c7" + } + } + }, + { + "id": "5ae182a2-1ec6-4324-b9f0-9c4c6f7a1416", + "createdBy": "WFWX_WEATHER_API", + "lastEntityUpdateTimestamp": 1665086701632, + "updateDate": "2022-10-06T20:05:01.000+0000", + "lastModifiedBy": "WFWX_WEATHER_API", + "archive": true, + "station": "https://bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/stations/bfe0a6e2-e3bc-0210-e053-259e228e58c7", + "stationId": "bfe0a6e2-e3bc-0210-e053-259e228e58c7", + "stationData": { + "id": "bfe0a6e2-e3bc-0210-e053-259e228e58c7", + "displayLabel": "ASPEN GROVE", + "createdBy": "LEGACY_DATA_LOAD", + "lastModifiedBy": "WFWX_WEATHER_API", + "lastEntityUpdateTimestamp": 1677961854286, + "updateDate": "2023-03-04T20:30:54.000+0000", + "stationCode": 302, + "stationAcronym": "FSG", + "stationStatus": { + "id": "ACTIVE", + "displayLabel": "Active", + "displayOrder": 1, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "dataSource": { + "id": "AUTO_CALLR", + "displayLabel": "Auto-Caller", + "displayOrder": 1, + "effectiveDate": "1950-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "dataLoggerType": { + "id": "FWS12S", + "displayLabel": "FWS12S", + "displayOrder": 6, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "siteType": { + "id": "WXSTN_CELL", + "displayLabel": "Weather Station - Cell", + "displayOrder": 6, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "weatherZone": { + "id": 20, + "displayLabel": "Zone 20", + "dangerRegion": 3, + "displayOrder": 20 + }, + "stationAccessTypeCode": { + "id": "4WD", + "displayLabel": "4 Wheel Drive", + "displayOrder": 1, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "hourly": true, + "pluvioSnowGuage": false, + "latitude": 49.94811, + "longitude": -120.62107, + "geometry": { + "crs": { + "type": "name", + "properties": { + "name": "EPSG:4326" + } + }, + "coordinates": [ + -120.62107, + 49.94811 + ], + "type": "Point" + }, + "elevation": 1065, + "windspeedAdjRoughness": 1.0, + "windspeedHeight": { + "id": "9.0_11.9", + "displayLabel": "9.0 - 11.9 (m)", + "displayOrder": 7, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "surfaceType": null, + "overWinterPrecipAdj": false, + "influencingSlope": 0, + "installationDate": 912322800000, + "decommissionDate": null, + "influencingAspect": null, + "owner": { + "id": "a9843bb7-0f44-4a80-8a62-79cca2a5c758", + "displayLabel": "BC Wildfire Service", + "displayOrder": 2, + "ownerName": "BCWS", + "ownerDisplayName": "BC Wildfire Service", + "ownerComment": null, + "createdBy": "DATA_LOAD", + "lastModifiedBy": "DATAFIX - WFWX-891", + "effectiveDate": "1950-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "fireCentre": { + "id": 25, + "displayLabel": "Kamloops Fire Centre", + "alias": 5, + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "zone": { + "id": 31, + "displayLabel": "Merritt Zone", + "alias": 6, + "fireCentre": "Kamloops Fire Centre", + "fireCentreAlias": 5, + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "comments": "Installed Cell Modem June 2021.", + "ecccFileBaseUrlTemplate": null, + "lastProcessedSourceTime": "2023-03-04T12:30:54", + "crdStationName": null, + "stationAccessDescription": "Just off Hwy #5A by Aspen Grove near the junction with 97C, through the cattle gate over guard, 200m to site." + }, + "weatherTimestamp": 1665086400000, + "temperature": 19.5, + "dewPoint": 4.3, + "temperatureMin": -60.0, + "temperatureMax": 55.0, + "relativeHumidity": 37.0, + "relativeHumidityMin": 0.0, + "relativeHumidityMax": 105.0, + "windSpeed": 10.6, + "adjustedWindSpeed": 10.6, + "precipitation": 0.0, + "dangerForest": 4, + "dangerGrassland": 1, + "dangerScrub": 5, + "recordType": { + "id": "ACTUAL", + "displayLabel": "Actual", + "displayOrder": 1, + "createdBy": "DATA_LOAD", + "lastModifiedBy": "DATA_LOAD" + }, + "windDirection": 5.0, + "fineFuelMoistureCode": 91.439, + "duffMoistureCode": 104.704, + "droughtCode": 778.345, + "initialSpreadIndex": 8.981, + "buildUpIndex": 156.707, + "fireWeatherIndex": 34.633, + "dailySeverityRating": null, + "grasslandCuring": null, + "observationValidInd": true, + "observationValidComment": null, + "missingHourlyData": false, + "previousState": null, + "businessKey": "1665086400000-bfe0a6e2-e3bc-0210-e053-259e228e58c7", + "_links": { + "self": { + "href": "https://bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/dailies/5ae182a2-1ec6-4324-b9f0-9c4c6f7a1416" + }, + "daily": { + "href": "https://bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/dailies/5ae182a2-1ec6-4324-b9f0-9c4c6f7a1416" + }, + "station": { + "href": "https://bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/stations/bfe0a6e2-e3bc-0210-e053-259e228e58c7" + } + } + } + ] + }, + "_links": { + "self": { + "href": "https://bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/dailies/search/findDailiesByStationIdIsInAndWeatherTimestampBetweenOrderByStationIdAscWeatherTimestampAsc?stationIds=bfe0a6e2-e269-0210-e053-259e228e58c7,bfe0a6e2-e26b-0210-e053-259e228e58c7,bfe0a6e2-e3bc-0210-e053-259e228e58c7&startingTimestamp=1665086400000&endingTimestamp=1665086400000" + } + } +} \ No newline at end of file From 12892bc9841620dd5cb9280cc46eeb517514efe5 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Tue, 31 Oct 2023 14:50:59 -0700 Subject: [PATCH 15/36] add backend tests for get_fwi_values --- api/app/tests/morecast_v2/test_forecasts.py | 58 ++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/api/app/tests/morecast_v2/test_forecasts.py b/api/app/tests/morecast_v2/test_forecasts.py index 266646941..5d8ad4638 100644 --- a/api/app/tests/morecast_v2/test_forecasts.py +++ b/api/app/tests/morecast_v2/test_forecasts.py @@ -2,8 +2,10 @@ from typing import Optional from unittest.mock import Mock, patch import pytest +from math import isclose from app.db.models.morecast_v2 import MorecastForecastRecord -from app.morecast_v2.forecasts import actual_exists, construct_wf1_forecast, construct_wf1_forecasts, filter_for_api_forecasts, get_forecasts +from app.morecast_v2.forecasts import (actual_exists, construct_wf1_forecast, + construct_wf1_forecasts, filter_for_api_forecasts, get_forecasts, get_fwi_values) from app.schemas.morecast_v2 import (StationDailyFromWF1, WeatherDeterminate, WeatherIndeterminate, WF1ForecastRecordType, WF1PostForecast) from app.wildfire_one.schema_parsers import WFWXWeatherStation @@ -40,6 +42,44 @@ update_timestamp=end_time, update_user='test2') +weather_indeterminate_1 = WeatherIndeterminate(station_code=123, + station_name="TEST_STATION", + determinate=WeatherDeterminate.ACTUAL, + utc_timestamp=start_time, + latitude=51.507, + longitude=-121.162, + temperature=4.1, + relative_humidity=34.0, + precipitation=0.0, + wind_direction=184.0, + wind_speed=8.9, + fine_fuel_moisture_code=62, + duff_moisture_code=27, + drought_code=487, + initial_spread_index=4, + build_up_index=52, + fire_weather_index=14, + danger_rating=2) + +weather_indeterminate_2 = WeatherIndeterminate(station_code=123, + station_name="TEST_STATION", + determinate=WeatherDeterminate.FORECAST, + utc_timestamp=end_time, + latitude=51.507, + longitude=-121.162, + temperature=6.3, + relative_humidity=35.0, + precipitation=0.0, + wind_direction=176.0, + wind_speed=8.9, + fine_fuel_moisture_code=None, + duff_moisture_code=None, + drought_code=None, + initial_spread_index=None, + build_up_index=None, + fire_weather_index=None, + danger_rating=None) + wfwx_weather_stations = [ WFWXWeatherStation( wfwx_id='1', @@ -87,10 +127,26 @@ def assert_wf1_forecast(result: WF1PostForecast, assert result.recordType == WF1ForecastRecordType() +def test_get_fwi_values(): + actuals, forecasts = get_fwi_values([weather_indeterminate_1], [weather_indeterminate_2]) + assert len(forecasts) == 1 + assert len(actuals) == 1 + assert isclose(forecasts[0].fine_fuel_moisture_code, 76.59454201861331) + assert isclose(forecasts[0].duff_moisture_code, 27.5921591) + assert isclose(forecasts[0].drought_code, 487.838) + assert isclose(forecasts[0].initial_spread_index, 1.3234484847240926) + assert isclose(forecasts[0].build_up_index, 48.347912947622426) + assert isclose(forecasts[0].fire_weather_index, 3.841725745428403) + + @patch('app.morecast_v2.forecasts.get_forecasts_in_range', return_value=[]) def test_get_forecasts_empty(_): result = get_forecasts(Mock(), start_time, end_time, []) assert len(result) == 0 + result = get_forecasts(Mock(), None, end_time, []) + assert len(result) == 0 + result = get_forecasts(Mock(), start_time, None, []) + assert len(result) == 0 @patch('app.morecast_v2.forecasts.get_forecasts_in_range', return_value=[morecast_record_1, morecast_record_2]) From 649ae553b43c5536c26b94c5bd4c42b89c4e285d Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Tue, 31 Oct 2023 14:56:41 -0700 Subject: [PATCH 16/36] code smell --- web/src/features/moreCast2/slices/dataSlice.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/features/moreCast2/slices/dataSlice.ts b/web/src/features/moreCast2/slices/dataSlice.ts index 51e3ad5d2..301346de1 100644 --- a/web/src/features/moreCast2/slices/dataSlice.ts +++ b/web/src/features/moreCast2/slices/dataSlice.ts @@ -65,7 +65,7 @@ const dataSlice = createSlice({ state.forecasts = state.forecasts.map(forecast => { const updatedForecast = updatedForecasts.find(item => item.id === forecast.id) - return updatedForecast || forecast + return updatedForecast ?? forecast }) }, simulateWeatherIndeterminatesFailed(state: State, action: PayloadAction) { From 7657c36f84c3c68cee06912865bfbe12ec77d8e4 Mon Sep 17 00:00:00 2001 From: Conor Brady Date: Tue, 31 Oct 2023 16:20:12 -0700 Subject: [PATCH 17/36] Add dataslice tests --- .../moreCast2/slices/dataSlice.test.ts | 85 ++++++++++++++++++- .../features/moreCast2/slices/dataSlice.ts | 2 +- 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/web/src/features/moreCast2/slices/dataSlice.test.ts b/web/src/features/moreCast2/slices/dataSlice.test.ts index e285cdffb..ba8dcbf30 100644 --- a/web/src/features/moreCast2/slices/dataSlice.test.ts +++ b/web/src/features/moreCast2/slices/dataSlice.test.ts @@ -11,8 +11,12 @@ import dataSliceReducer, { initialState, getWeatherIndeterminatesFailed, getWeatherIndeterminatesStart, - getWeatherIndeterminatesSuccess + getWeatherIndeterminatesSuccess, + simulateWeatherIndeterminatesSuccess, + simulateWeatherIndeterminatesFailed, + storeUserEditedRows } from 'features/moreCast2/slices/dataSlice' +import { rowIDHasher } from 'features/moreCast2/util' import { DateTime } from 'luxon' const FROM_DATE_STRING = '2023-04-27T20:00:00+00:00' @@ -49,7 +53,7 @@ const weatherIndeterminateGenerator = ( precipValue?: number ) => { return { - id: `${station_code}${utc_timestamp}`, + id: rowIDHasher(station_code, DateTime.fromISO(utc_timestamp)), station_code, station_name, determinate, @@ -163,6 +167,83 @@ describe('dataSlice', () => { it('should set a value for error state when getWeatherIndeterminatesFailed is called', () => { expect(dataSliceReducer(initialState, getWeatherIndeterminatesFailed(dummyError)).error).not.toBeNull() }) + + it('should handle missing forcasts for calculated indices update', () => { + expect( + dataSliceReducer( + initialState, + simulateWeatherIndeterminatesSuccess({ + simulatedForecasts: [] + }) + ).forecasts + ).toEqual([]) + }) + + it('should only overwrite updated forecasts', () => { + const weatherIndeterminate1 = weatherIndeterminateGenerator( + 1, + 'test1', + WeatherDeterminate.FORECAST, + FROM_DATE_STRING + ) + + const weatherIndeterminate2 = weatherIndeterminateGenerator( + 2, + 'test', + WeatherDeterminate.FORECAST, + TO_DATE_STRING + ) + + const updatedWeatherIndeterminate2 = { + ...weatherIndeterminate2, + fine_fuel_moisture_code: 1, + duff_moisture_code: 1, + drought_code: 1, + initial_spread_index: 1, + build_up_index: 1, + fire_weather_index: 1, + danger_rating: 1 + } + + expect( + dataSliceReducer( + { ...initialState, forecasts: [weatherIndeterminate1, weatherIndeterminate2] }, + simulateWeatherIndeterminatesSuccess({ + simulatedForecasts: [updatedWeatherIndeterminate2] + }) + ).forecasts + ).toEqual([weatherIndeterminate1, updatedWeatherIndeterminate2]) + }) + it('should set a value for error state when simulateWeatherIndeterminatesFailed is called', () => { + expect(dataSliceReducer(initialState, simulateWeatherIndeterminatesFailed(dummyError)).error).not.toBeNull() + }) + it('should store the edited rows', () => { + const rows = createMoreCast2Rows( + [], + [weatherIndeterminateGenerator(1, 'test1', WeatherDeterminate.FORECAST, FROM_DATE_STRING)], + [] + ) + expect(dataSliceReducer(initialState, storeUserEditedRows(rows)).userEditedRows).toEqual(rows) + }) + + it('should updated the edited rows', () => { + const forecast = weatherIndeterminateGenerator(1, 'test1', WeatherDeterminate.FORECAST, FROM_DATE_STRING) + const rows = createMoreCast2Rows([], [forecast], []) + expect(dataSliceReducer(initialState, storeUserEditedRows(rows)).userEditedRows).toEqual(rows) + + const updatedForecast = { + ...forecast, + fine_fuel_moisture_code: 1, + duff_moisture_code: 1, + drought_code: 1, + initial_spread_index: 1, + build_up_index: 1, + fire_weather_index: 1, + danger_rating: 1 + } + const updatedRows = createMoreCast2Rows([], [updatedForecast], []) + expect(dataSliceReducer(initialState, storeUserEditedRows(updatedRows)).userEditedRows).toEqual(updatedRows) + }) }) describe('fillMissingWeatherIndeterminates', () => { const fromDate = DateTime.fromISO('2023-04-27T20:00:00+00:00') diff --git a/web/src/features/moreCast2/slices/dataSlice.ts b/web/src/features/moreCast2/slices/dataSlice.ts index 51e3ad5d2..301346de1 100644 --- a/web/src/features/moreCast2/slices/dataSlice.ts +++ b/web/src/features/moreCast2/slices/dataSlice.ts @@ -65,7 +65,7 @@ const dataSlice = createSlice({ state.forecasts = state.forecasts.map(forecast => { const updatedForecast = updatedForecasts.find(item => item.id === forecast.id) - return updatedForecast || forecast + return updatedForecast ?? forecast }) }, simulateWeatherIndeterminatesFailed(state: State, action: PayloadAction) { From f6006cca1e13dd46260769691a4d873d77468ddf Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Tue, 31 Oct 2023 16:30:53 -0700 Subject: [PATCH 18/36] endpoint tests --- .../morecast_v2/test_morecast_v2_endpoint.py | 59 ++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/api/app/tests/morecast_v2/test_morecast_v2_endpoint.py b/api/app/tests/morecast_v2/test_morecast_v2_endpoint.py index 4b605d6ed..edce756cf 100644 --- a/api/app/tests/morecast_v2/test_morecast_v2_endpoint.py +++ b/api/app/tests/morecast_v2/test_morecast_v2_endpoint.py @@ -6,9 +6,10 @@ from app.schemas.shared import StationsRequest from app.tests.common import default_mock_client_get from app.schemas.morecast_v2 import (MoreCastForecastInput, - MoreCastForecastRequest, StationDailyFromWF1) + MoreCastForecastRequest, StationDailyFromWF1, WeatherDeterminate) import app.routers.morecast_v2 from app.tests.utils.mock_jwt_decode_role import MockJWTDecodeWithRole +from app.tests.morecast_v2.test_forecasts import weather_indeterminate_1, weather_indeterminate_2 morecast_v2_post_url = '/api/morecast-v2/forecast' @@ -17,6 +18,7 @@ today = '2022-10-07' morecast_v2_post_yesterday_dailies_url = f'/api/morecast-v2/yesterday-dailies/{today}' morecast_v2_post_determinates_url = '/api/morecast-v2/determinates/2023-03-15/2023-03-19' +morecast_v2_post_simulate_url = 'api/morecast-v2/simulate-indices/' decode_fn = "jwt.decode" @@ -158,3 +160,58 @@ def mock_admin_role_function(*_, **__): response = await async_client.post(morecast_v2_post_determinates_url, json={"stations": [209, 211, 302]}) assert response.status_code == 200 + + +def test_simulate_indeterminates_unauthorized(client: TestClient): + response = client.post(morecast_v2_post_simulate_url, json=[]) + assert response.status_code == 401 + + +@pytest.mark.anyio +async def test_simulate_indeterminates_authorized(anyio_backend, async_client: AsyncClient, monkeypatch: pytest.MonkeyPatch): + def mock_admin_role_function(*_, **__): + return MockJWTDecodeWithRole('morecast2_write_forecast') + + monkeypatch.setattr(decode_fn, mock_admin_role_function) + + simulate_records = [ + {"station_code": 1203, + "station_name": "DARKWOODS", + "determinate": "Actual", + "utc_timestamp": "2023-10-30T20:00:00Z", + "latitude": 49.3576111, + "longitude": -116.95025, + "temperature": -0.8, + "relative_humidity": 65.0, + "precipitation": 0.8, + "wind_direction": 201.0, + "wind_speed": 6.7, + "fine_fuel_moisture_code": 72.26436115751054, + "duff_moisture_code": 4.5262768, + "drought_code": 293.47, + "initial_spread_index": 0.9472828989641714, + "build_up_index": 8.716462008301884, + "fire_weather_index": 0.5312701384624857, + "danger_rating": 1}, + {"station_code": 1203, + "station_name": "DARKWOODS", + "determinate": "Actual", + "utc_timestamp": "2023-10-31T20:00:00Z", + "latitude": 49.3576111, + "longitude": -116.95025, + "temperature": 3.6, + "relative_humidity": 47.0, + "precipitation": 0.0, + "wind_direction": 214.0, + "wind_speed": 8.2, + "fine_fuel_moisture_code": 78.95635139203418, + "duff_moisture_code": 4.90371312, + "drought_code": 294.822, + "initial_spread_index": 1.5488967924410735, + "build_up_index": 9.41589468613904, + "fire_weather_index": 0.9046887731834204, + "danger_rating": 1} + ] + + response = await async_client.post(morecast_v2_post_simulate_url, json={"simulate_records": simulate_records}) + assert response.status_code == 200 From f071f13e20d2e3854fa2bc67cdd976d2fcc6c6f7 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Tue, 31 Oct 2023 16:31:34 -0700 Subject: [PATCH 19/36] unused imports --- api/app/tests/morecast_v2/test_morecast_v2_endpoint.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/app/tests/morecast_v2/test_morecast_v2_endpoint.py b/api/app/tests/morecast_v2/test_morecast_v2_endpoint.py index edce756cf..98168007e 100644 --- a/api/app/tests/morecast_v2/test_morecast_v2_endpoint.py +++ b/api/app/tests/morecast_v2/test_morecast_v2_endpoint.py @@ -6,10 +6,9 @@ from app.schemas.shared import StationsRequest from app.tests.common import default_mock_client_get from app.schemas.morecast_v2 import (MoreCastForecastInput, - MoreCastForecastRequest, StationDailyFromWF1, WeatherDeterminate) + MoreCastForecastRequest, StationDailyFromWF1) import app.routers.morecast_v2 from app.tests.utils.mock_jwt_decode_role import MockJWTDecodeWithRole -from app.tests.morecast_v2.test_forecasts import weather_indeterminate_1, weather_indeterminate_2 morecast_v2_post_url = '/api/morecast-v2/forecast' From d20bef9524703194735a3fb9b6504cf4ca64244b Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Wed, 1 Nov 2023 15:39:09 -0700 Subject: [PATCH 20/36] processRowUpdate bug squasher? --- .../components/ForecastSummaryDataGrid.tsx | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx b/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx index 294bed4d6..c611bf33d 100644 --- a/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx +++ b/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx @@ -1,6 +1,6 @@ import React from 'react' import { styled } from '@mui/material/styles' -import { DataGrid, GridColDef, GridEventListener, GridCellEditStopParams } from '@mui/x-data-grid' +import { DataGrid, GridColDef, GridEventListener } from '@mui/x-data-grid' import { ModelChoice, ModelType } from 'api/moreCast2API' import { MoreCast2Row } from 'features/moreCast2/interfaces' import { LinearProgress } from '@mui/material' @@ -62,24 +62,25 @@ const ForecastSummaryDataGrid = ({ handleClose }: ForecastSummaryDataGridProps) => { const dispatch: AppDispatch = useDispatch() - const handleCellEditStop = async (params: GridCellEditStopParams) => { - const editedRow = params.row - dispatch(storeUserEditedRows([editedRow])) + const processRowUpdate = async (newRow: MoreCast2Row) => { + dispatch(storeUserEditedRows([newRow])) const mustBeFilled = [ - editedRow.tempForecast?.value, - editedRow.rhForecast?.value, - editedRow.windSpeedForecast?.value, - editedRow.precipForecast?.value + newRow.tempForecast?.value, + newRow.rhForecast?.value, + newRow.windSpeedForecast?.value, + newRow.precipForecast?.value ] for (const value of mustBeFilled) { if (isNaN(value)) { - return editedRow + return newRow } } - const idBeforeEditedRow = getYesterdayRowID(editedRow) + const idBeforeEditedRow = getYesterdayRowID(newRow) const rowsForSimulation = rows.filter(row => row.id >= idBeforeEditedRow).filter(isActualOrValidForecastPredicate) dispatch(getSimulatedIndices(rowsForSimulation)) + + return newRow } return ( @@ -98,7 +99,7 @@ const ForecastSummaryDataGrid = ({ columns={DataGridColumns.getSummaryColumns()} rows={rows} isCellEditable={params => params.row[params.field] !== ModelChoice.ACTUAL} - onCellEditStop={handleCellEditStop} + processRowUpdate={processRowUpdate} /> Date: Wed, 1 Nov 2023 15:54:54 -0700 Subject: [PATCH 21/36] smelly code --- .../components/ForecastSummaryDataGrid.tsx | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx b/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx index c611bf33d..b6bdd1505 100644 --- a/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx +++ b/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx @@ -71,14 +71,13 @@ const ForecastSummaryDataGrid = ({ newRow.windSpeedForecast?.value, newRow.precipForecast?.value ] - for (const value of mustBeFilled) { - if (isNaN(value)) { - return newRow - } + const isValidForecast = mustBeFilled.every(value => !isNaN(value)) + + if (isValidForecast) { + const idBeforeEditedRow = getYesterdayRowID(newRow) + const rowsForSimulation = rows.filter(row => row.id >= idBeforeEditedRow).filter(isActualOrValidForecastPredicate) + dispatch(getSimulatedIndices(rowsForSimulation)) } - const idBeforeEditedRow = getYesterdayRowID(newRow) - const rowsForSimulation = rows.filter(row => row.id >= idBeforeEditedRow).filter(isActualOrValidForecastPredicate) - dispatch(getSimulatedIndices(rowsForSimulation)) return newRow } From 1a046f1db02681fd8a7f822536b4f766cba27249 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Thu, 2 Nov 2023 16:05:44 -0700 Subject: [PATCH 22/36] always be able to handle multiple stations - frontend and backend updates --- api/app/routers/morecast_v2.py | 18 ++++---- .../components/ForecastSummaryDataGrid.tsx | 45 +++++++------------ .../moreCast2/components/TabbedDataGrid.tsx | 35 ++++++++++----- .../features/moreCast2/saveForecast.test.ts | 3 +- web/src/features/moreCast2/saveForecasts.ts | 13 +----- web/src/features/moreCast2/util.ts | 27 ++++++++++- 6 files changed, 76 insertions(+), 65 deletions(-) diff --git a/api/app/routers/morecast_v2.py b/api/app/routers/morecast_v2.py index 08da8025d..a6422324c 100644 --- a/api/app/routers/morecast_v2.py +++ b/api/app/routers/morecast_v2.py @@ -21,6 +21,7 @@ ObservedDailiesForStations, StationDailiesResponse, WeatherIndeterminate, + WeatherDeterminate, SimulateIndeterminateIndices, SimulatedWeatherIndeterminateResponse) from app.schemas.shared import StationsRequest @@ -239,12 +240,11 @@ async def calculate_forecasted_indices(simulate_records: SimulateIndeterminateIn """ indeterminates = simulate_records.simulate_records logger.info( - f'/simulate-indices/ - simulating forecast records for {indeterminates[0].station_name}') - - yesterday_record = indeterminates[0] - indeterminates_to_simulate = indeterminates[1:] - for idx, record in enumerate(indeterminates_to_simulate): - calculated_record = calculate_fwi_values(yesterday_record, record) - indeterminates_to_simulate[idx] = calculated_record - yesterday_record = calculated_record - return (SimulatedWeatherIndeterminateResponse(simulatedForecasts=indeterminates_to_simulate)) + f'/simulate-indices/ - simulating forecast records for stations: {set(indeterminate.station_name for indeterminate in indeterminates)}') + + forecasts = [indeterminate for indeterminate in indeterminates if indeterminate.determinate == + WeatherDeterminate.FORECAST] + actuals = [indeterminate for indeterminate in indeterminates if indeterminate.determinate == WeatherDeterminate.ACTUAL] + + _, forecasts = get_fwi_values(actuals, forecasts) + return (SimulatedWeatherIndeterminateResponse(simulatedForecasts=forecasts)) diff --git a/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx b/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx index b6bdd1505..ff8b73840 100644 --- a/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx +++ b/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx @@ -6,12 +6,10 @@ import { MoreCast2Row } from 'features/moreCast2/interfaces' import { LinearProgress } from '@mui/material' import ApplyToColumnMenu from 'features/moreCast2/components/ApplyToColumnMenu' import { DataGridColumns } from 'features/moreCast2/components/DataGridColumns' -import { isNaN } from 'lodash' -import { rowIDHasher } from 'features/moreCast2/util' -import { validForecastPredicate } from 'features/moreCast2/saveForecasts' import { storeUserEditedRows, getSimulatedIndices } from 'features/moreCast2/slices/dataSlice' import { AppDispatch } from 'app/store' import { useDispatch } from 'react-redux' +import { validActualOrForecastPredicate, validForecastPredicate } from 'features/moreCast2/util' const PREFIX = 'ForecastSummaryDataGrid' @@ -39,19 +37,6 @@ interface ForecastSummaryDataGridProps { handleClose: () => void } -const getYesterdayRowID = (todayRow: MoreCast2Row): string => { - const yesterdayDate = todayRow.forDate.minus({ days: 1 }) - const yesterdayID = rowIDHasher(todayRow.stationCode, yesterdayDate) - - return yesterdayID -} - -export const validActualPredicate = (row: MoreCast2Row) => - !isNaN(row.precipActual) && !isNaN(row.rhActual) && !isNaN(row.tempActual) && !isNaN(row.windSpeedActual) - -export const isActualOrValidForecastPredicate = (row: MoreCast2Row) => - validForecastPredicate(row) || validActualPredicate(row) - const ForecastSummaryDataGrid = ({ loading, rows, @@ -62,24 +47,24 @@ const ForecastSummaryDataGrid = ({ handleClose }: ForecastSummaryDataGridProps) => { const dispatch: AppDispatch = useDispatch() - const processRowUpdate = async (newRow: MoreCast2Row) => { - dispatch(storeUserEditedRows([newRow])) + const processRowUpdate = async (editedRow: MoreCast2Row) => { + dispatch(storeUserEditedRows([editedRow])) + + if (validForecastPredicate(editedRow)) { + const validRowsForStation = rows.filter( + row => row.stationCode === editedRow.stationCode && validActualOrForecastPredicate(row) + ) - const mustBeFilled = [ - newRow.tempForecast?.value, - newRow.rhForecast?.value, - newRow.windSpeedForecast?.value, - newRow.precipForecast?.value - ] - const isValidForecast = mustBeFilled.every(value => !isNaN(value)) + const yesterday = editedRow.forDate.minus({ days: 1 }) + const yesterdayRow = validRowsForStation.find(row => row.forDate.toISODate() === yesterday.toISODate()) - if (isValidForecast) { - const idBeforeEditedRow = getYesterdayRowID(newRow) - const rowsForSimulation = rows.filter(row => row.id >= idBeforeEditedRow).filter(isActualOrValidForecastPredicate) - dispatch(getSimulatedIndices(rowsForSimulation)) + if (yesterdayRow) { + const rowsForSimulation = validRowsForStation.filter(row => row.forDate >= yesterday) + dispatch(getSimulatedIndices(rowsForSimulation)) + } } - return newRow + return editedRow } return ( diff --git a/web/src/features/moreCast2/components/TabbedDataGrid.tsx b/web/src/features/moreCast2/components/TabbedDataGrid.tsx index a804aa316..1e3deeffe 100644 --- a/web/src/features/moreCast2/components/TabbedDataGrid.tsx +++ b/web/src/features/moreCast2/components/TabbedDataGrid.tsx @@ -4,7 +4,7 @@ import { GridCellParams, GridColDef, GridColumnVisibilityModel, GridEventListene import { ModelChoice, ModelType, submitMoreCastForecastRecords } from 'api/moreCast2API' import { DataGridColumns, columnGroupingModel } from 'features/moreCast2/components/DataGridColumns' import ForecastDataGrid from 'features/moreCast2/components/ForecastDataGrid' -import ForecastSummaryDataGrid, { validActualPredicate } from 'features/moreCast2/components/ForecastSummaryDataGrid' +import ForecastSummaryDataGrid from 'features/moreCast2/components/ForecastSummaryDataGrid' import SelectableButton from 'features/moreCast2/components/SelectableButton' import { getSimulatedIndices, @@ -22,15 +22,11 @@ import { ROLES } from 'features/auth/roles' import { selectAuthentication, selectWf1Authentication } from 'app/rootReducer' import { DateRange } from 'components/dateRangePicker/types' import MoreCast2Snackbar from 'features/moreCast2/components/MoreCast2Snackbar' -import { - isForecastRowPredicate, - getRowsToSave, - isForecastValid, - validForecastPredicate -} from 'features/moreCast2/saveForecasts' +import { isForecastRowPredicate, getRowsToSave, isForecastValid } from 'features/moreCast2/saveForecasts' import MoreCast2DateRangePicker from 'features/moreCast2/components/MoreCast2DateRangePicker' import { AppDispatch } from 'app/store' import { deepClone } from '@mui/x-data-grid/utils/utils' +import { validActualPredicate, validForecastPredicate } from 'features/moreCast2/util' export const Root = styled('div')({ display: 'flex', @@ -198,12 +194,21 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp const [clickedColDef, setClickedColDef] = useState(null) - const filterRowsForSimulation = (rows: MoreCast2Row[]): MoreCast2Row[] => { + const filterRowsForSimulation = (rows: MoreCast2Row[]): MoreCast2Row[] | undefined => { const forecasts = rows.filter(validForecastPredicate) const actuals = rows.filter(validActualPredicate) - const mostRecentActual = actuals.pop() + const mostRecentActualMap = new Map() + + for (const row of actuals) { + const recentActual = mostRecentActualMap.get(row.stationCode) + if (!recentActual || recentActual.forDate < row.forDate) { + mostRecentActualMap.set(row.stationCode, row) + } + } + const mostRecentActuals = Array.from(mostRecentActualMap.values()) + const rowsForSimulation = [...mostRecentActuals, ...forecasts] - return mostRecentActual ? [mostRecentActual, ...forecasts] : rows + return forecasts.length > 0 ? rowsForSimulation : undefined } const mapForecastChoiceLabels = (newRows: MoreCast2Row[], storedRows: MoreCast2Row[]): MoreCast2Row[] => { @@ -267,8 +272,11 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp }) } const rowsForSimulation = filterRowsForSimulation(newRows) + if (rowsForSimulation) { + dispatch(getSimulatedIndices(rowsForSimulation)) + } + dispatch(storeUserEditedRows(newRows)) - dispatch(getSimulatedIndices(rowsForSimulation)) setVisibleRows(newRows) } @@ -291,8 +299,11 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp } } const rowsForSimulation = filterRowsForSimulation(newRows) + if (rowsForSimulation) { + dispatch(getSimulatedIndices(rowsForSimulation)) + } + dispatch(storeUserEditedRows(newRows)) - dispatch(getSimulatedIndices(rowsForSimulation)) setVisibleRows(newRows) } diff --git a/web/src/features/moreCast2/saveForecast.test.ts b/web/src/features/moreCast2/saveForecast.test.ts index c008e4e52..c11b66b1b 100644 --- a/web/src/features/moreCast2/saveForecast.test.ts +++ b/web/src/features/moreCast2/saveForecast.test.ts @@ -1,6 +1,7 @@ import { ModelChoice } from 'api/moreCast2API' import { MoreCast2Row } from 'features/moreCast2/interfaces' -import { getRowsToSave, isForecastValid, validForecastPredicate } from 'features/moreCast2/saveForecasts' +import { getRowsToSave, isForecastValid } from 'features/moreCast2/saveForecasts' +import { validForecastPredicate } from 'features/moreCast2/util' import { DateTime } from 'luxon' const baseRow = { diff --git a/web/src/features/moreCast2/saveForecasts.ts b/web/src/features/moreCast2/saveForecasts.ts index d13fe7430..475822c9a 100644 --- a/web/src/features/moreCast2/saveForecasts.ts +++ b/web/src/features/moreCast2/saveForecasts.ts @@ -1,6 +1,6 @@ import { ModelChoice } from 'api/moreCast2API' import { MoreCast2ForecastRow, MoreCast2Row } from 'features/moreCast2/interfaces' -import { isUndefined } from 'lodash' +import { validForecastPredicate } from 'features/moreCast2/util' // Forecast rows contain all NaN values in their 'actual' fields export const isForecastRowPredicate = (row: MoreCast2Row) => @@ -10,17 +10,6 @@ export const isForecastRowPredicate = (row: MoreCast2Row) => isNaN(row.windDirectionActual) && isNaN(row.windSpeedActual) -// A valid forecast row has values for precipForecast, rhForecast, tempForecast and windSpeedForecast -export const validForecastPredicate = (row: MoreCast2Row) => - !isUndefined(row.precipForecast) && - !isNaN(row.precipForecast.value) && - !isUndefined(row.rhForecast) && - !isNaN(row.rhForecast.value) && - !isUndefined(row.tempForecast) && - !isNaN(row.tempForecast.value) && - !isUndefined(row.windSpeedForecast) && - !isNaN(row.windSpeedForecast.value) - export const getForecastRows = (rows: MoreCast2Row[]): MoreCast2Row[] => { return rows ? rows.filter(isForecastRowPredicate) : [] } diff --git a/web/src/features/moreCast2/util.ts b/web/src/features/moreCast2/util.ts index 77e63882a..c82aa999b 100644 --- a/web/src/features/moreCast2/util.ts +++ b/web/src/features/moreCast2/util.ts @@ -1,7 +1,8 @@ import { DateTime, Interval } from 'luxon' import { ModelChoice, MoreCast2ForecastRecord } from 'api/moreCast2API' -import { MoreCast2ForecastRow } from 'features/moreCast2/interfaces' +import { MoreCast2ForecastRow, MoreCast2Row } from 'features/moreCast2/interfaces' import { StationGroupMember } from 'api/stationAPI' +import { isUndefined } from 'lodash' export const parseForecastsHelper = ( forecasts: MoreCast2ForecastRecord[], @@ -79,3 +80,27 @@ export const createLabel = (isActual: boolean, label: string) => { return createWeatherModelLabel(label) } + +export const getYesterdayRowID = (todayRow: MoreCast2Row): string => { + const yesterdayDate = todayRow.forDate.minus({ days: 1 }) + const yesterdayID = rowIDHasher(todayRow.stationCode, yesterdayDate) + + return yesterdayID +} + +export const validActualOrForecastPredicate = (row: MoreCast2Row) => + validForecastPredicate(row) || validActualPredicate(row) + +export const validActualPredicate = (row: MoreCast2Row) => + !isNaN(row.precipActual) && !isNaN(row.rhActual) && !isNaN(row.tempActual) && !isNaN(row.windSpeedActual) + +// A valid forecast row has values for precipForecast, rhForecast, tempForecast and windSpeedForecast +export const validForecastPredicate = (row: MoreCast2Row) => + !isUndefined(row.precipForecast) && + !isNaN(row.precipForecast.value) && + !isUndefined(row.rhForecast) && + !isNaN(row.rhForecast.value) && + !isUndefined(row.tempForecast) && + !isNaN(row.tempForecast.value) && + !isUndefined(row.windSpeedForecast) && + !isNaN(row.windSpeedForecast.value) From cb18c6b807caa7f18be1e980333153750d496684 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Thu, 2 Nov 2023 16:09:36 -0700 Subject: [PATCH 23/36] clean up imports --- api/app/routers/morecast_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/app/routers/morecast_v2.py b/api/app/routers/morecast_v2.py index a6422324c..22b216c2a 100644 --- a/api/app/routers/morecast_v2.py +++ b/api/app/routers/morecast_v2.py @@ -13,7 +13,7 @@ from app.db.crud.morecast_v2 import get_forecasts_in_range, get_user_forecasts_for_date, save_all_forecasts from app.db.database import get_read_session_scope, get_write_session_scope from app.db.models.morecast_v2 import MorecastForecastRecord -from app.morecast_v2.forecasts import filter_for_api_forecasts, get_forecasts, calculate_fwi_values, get_fwi_values +from app.morecast_v2.forecasts import filter_for_api_forecasts, get_forecasts, get_fwi_values from app.schemas.morecast_v2 import (IndeterminateDailiesResponse, MoreCastForecastOutput, MoreCastForecastRequest, From 0159c1df27f2652ef3ebaf09fbbae303b83cd624 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Mon, 6 Nov 2023 11:00:00 -0800 Subject: [PATCH 24/36] update test_forecasts test --- api/app/morecast_v2/forecasts.py | 9 +- api/app/tests/morecast_v2/test_forecasts.py | 127 ++++++++++++++------ 2 files changed, 92 insertions(+), 44 deletions(-) diff --git a/api/app/morecast_v2/forecasts.py b/api/app/morecast_v2/forecasts.py index b8a2e18ab..e1988fd17 100644 --- a/api/app/morecast_v2/forecasts.py +++ b/api/app/morecast_v2/forecasts.py @@ -136,11 +136,12 @@ def get_fwi_values(actuals: List[WeatherIndeterminate], forecasts: List[WeatherI updated_forecast = calculate_fwi_values(last_indeterminate, indeterminate) all_indeterminates[idx] = updated_forecast - forecasts = [indeterminate for indeterminate in all_indeterminates if indeterminate.determinate == - WeatherDeterminate.FORECAST] - actuals = [indeterminate for indeterminate in all_indeterminates if indeterminate.determinate == WeatherDeterminate.ACTUAL] + updated_forecasts = [indeterminate for indeterminate in all_indeterminates if indeterminate.determinate == + WeatherDeterminate.FORECAST] + updated_actuals = [ + indeterminate for indeterminate in all_indeterminates if indeterminate.determinate == WeatherDeterminate.ACTUAL] - return actuals, forecasts + return updated_actuals, updated_forecasts def calculate_fwi_values(yesterday: WeatherIndeterminate, today: WeatherIndeterminate) -> WeatherIndeterminate: diff --git a/api/app/tests/morecast_v2/test_forecasts.py b/api/app/tests/morecast_v2/test_forecasts.py index 5d8ad4638..548546cea 100644 --- a/api/app/tests/morecast_v2/test_forecasts.py +++ b/api/app/tests/morecast_v2/test_forecasts.py @@ -42,43 +42,81 @@ update_timestamp=end_time, update_user='test2') -weather_indeterminate_1 = WeatherIndeterminate(station_code=123, - station_name="TEST_STATION", - determinate=WeatherDeterminate.ACTUAL, - utc_timestamp=start_time, - latitude=51.507, - longitude=-121.162, - temperature=4.1, - relative_humidity=34.0, - precipitation=0.0, - wind_direction=184.0, - wind_speed=8.9, - fine_fuel_moisture_code=62, - duff_moisture_code=27, - drought_code=487, - initial_spread_index=4, - build_up_index=52, - fire_weather_index=14, - danger_rating=2) - -weather_indeterminate_2 = WeatherIndeterminate(station_code=123, - station_name="TEST_STATION", - determinate=WeatherDeterminate.FORECAST, - utc_timestamp=end_time, - latitude=51.507, - longitude=-121.162, - temperature=6.3, - relative_humidity=35.0, - precipitation=0.0, - wind_direction=176.0, - wind_speed=8.9, - fine_fuel_moisture_code=None, - duff_moisture_code=None, - drought_code=None, - initial_spread_index=None, - build_up_index=None, - fire_weather_index=None, - danger_rating=None) +actual_indeterminate_1 = WeatherIndeterminate(station_code=123, + station_name="TEST_STATION", + determinate=WeatherDeterminate.ACTUAL, + utc_timestamp=start_time, + latitude=51.507, + longitude=-121.162, + temperature=4.1, + relative_humidity=34.0, + precipitation=0.0, + wind_direction=184.0, + wind_speed=8.9, + fine_fuel_moisture_code=62, + duff_moisture_code=27, + drought_code=487, + initial_spread_index=4, + build_up_index=52, + fire_weather_index=14, + danger_rating=2) + +forecast_indeterminate_1 = WeatherIndeterminate(station_code=123, + station_name="TEST_STATION", + determinate=WeatherDeterminate.FORECAST, + utc_timestamp=end_time, + latitude=51.507, + longitude=-121.162, + temperature=6.3, + relative_humidity=35.0, + precipitation=0.0, + wind_direction=176.0, + wind_speed=8.9, + fine_fuel_moisture_code=None, + duff_moisture_code=None, + drought_code=None, + initial_spread_index=None, + build_up_index=None, + fire_weather_index=None, + danger_rating=None) + +actual_indeterminate_2 = WeatherIndeterminate(station_code=321, + station_name="TEST_STATION2", + determinate=WeatherDeterminate.ACTUAL, + utc_timestamp=start_time, + latitude=49.4358, + longitude=-116.7464, + temperature=28.3, + relative_humidity=34.0, + precipitation=0.0, + wind_direction=180.0, + wind_speed=5.6, + fine_fuel_moisture_code=91, + duff_moisture_code=91, + drought_code=560, + initial_spread_index=7, + build_up_index=130, + fire_weather_index=28, + danger_rating=3) + +forecast_indeterminate_2 = WeatherIndeterminate(station_code=321, + station_name="TEST_STATION2", + determinate=WeatherDeterminate.FORECAST, + utc_timestamp=end_time, + latitude=49.4358, + longitude=-116.7464, + temperature=27.0, + relative_humidity=50.0, + precipitation=1.0, + wind_direction=176.0, + wind_speed=12, + fine_fuel_moisture_code=None, + duff_moisture_code=None, + drought_code=None, + initial_spread_index=None, + build_up_index=None, + fire_weather_index=None, + danger_rating=None) wfwx_weather_stations = [ WFWXWeatherStation( @@ -128,9 +166,11 @@ def assert_wf1_forecast(result: WF1PostForecast, def test_get_fwi_values(): - actuals, forecasts = get_fwi_values([weather_indeterminate_1], [weather_indeterminate_2]) - assert len(forecasts) == 1 - assert len(actuals) == 1 + actuals, forecasts = get_fwi_values([actual_indeterminate_1, actual_indeterminate_2], [ + forecast_indeterminate_1, forecast_indeterminate_2]) + assert len(forecasts) == 2 + assert len(actuals) == 2 + # The below values were calculated using the CFFDRS library and the values from the test indeterminates as input assert isclose(forecasts[0].fine_fuel_moisture_code, 76.59454201861331) assert isclose(forecasts[0].duff_moisture_code, 27.5921591) assert isclose(forecasts[0].drought_code, 487.838) @@ -138,6 +178,13 @@ def test_get_fwi_values(): assert isclose(forecasts[0].build_up_index, 48.347912947622426) assert isclose(forecasts[0].fire_weather_index, 3.841725745428403) + assert isclose(forecasts[1].fine_fuel_moisture_code, 87.00116939852603) + assert isclose(forecasts[1].duff_moisture_code, 92.7296955) + assert isclose(forecasts[1].drought_code, 564.564) + assert isclose(forecasts[1].initial_spread_index, 5.1025731818740345) + assert isclose(forecasts[1].build_up_index, 131.47318170452328) + assert isclose(forecasts[1].fire_weather_index, 22.263212983628037) + @patch('app.morecast_v2.forecasts.get_forecasts_in_range', return_value=[]) def test_get_forecasts_empty(_): From f7354e11e7b9829b6cbc8c61bac39c423aa2beb7 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Mon, 6 Nov 2023 12:27:09 -0800 Subject: [PATCH 25/36] deal with falsy 0's in backend --- api/app/morecast_v2/forecasts.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/api/app/morecast_v2/forecasts.py b/api/app/morecast_v2/forecasts.py index e1988fd17..7194b1fa0 100644 --- a/api/app/morecast_v2/forecasts.py +++ b/api/app/morecast_v2/forecasts.py @@ -164,13 +164,13 @@ def calculate_fwi_values(yesterday: WeatherIndeterminate, today: WeatherIndeterm precip = today.precipitation wind_spd = today.wind_speed - if yesterday.fine_fuel_moisture_code: + if yesterday.fine_fuel_moisture_code is not None: today.fine_fuel_moisture_code = cffdrs.fine_fuel_moisture_code(ffmc=yesterday.fine_fuel_moisture_code, temperature=temp, relative_humidity=rh, precipitation=precip, wind_speed=wind_spd) - if yesterday.duff_moisture_code: + if yesterday.duff_moisture_code is not None: today.duff_moisture_code = cffdrs.duff_moisture_code(dmc=yesterday.duff_moisture_code, temperature=temp, relative_humidity=rh, @@ -179,7 +179,7 @@ def calculate_fwi_values(yesterday: WeatherIndeterminate, today: WeatherIndeterm month=month_to_calculate_for, latitude_adjust=True ) - if yesterday.drought_code: + if yesterday.drought_code is not None: today.drought_code = cffdrs.drought_code(dc=yesterday.drought_code, temperature=temp, relative_humidity=rh, @@ -188,12 +188,12 @@ def calculate_fwi_values(yesterday: WeatherIndeterminate, today: WeatherIndeterm month=month_to_calculate_for, latitude_adjust=True ) - if today.fine_fuel_moisture_code: + if today.fine_fuel_moisture_code is not None: today.initial_spread_index = cffdrs.initial_spread_index(ffmc=today.fine_fuel_moisture_code, wind_speed=today.wind_speed) - if today.duff_moisture_code and today.drought_code: + if today.duff_moisture_code is not None and today.drought_code is not None: today.build_up_index = cffdrs.bui_calc(dmc=today.duff_moisture_code, dc=today.drought_code) - if today.initial_spread_index and today.build_up_index: + if today.initial_spread_index is not None and today.build_up_index is not None: today.fire_weather_index = cffdrs.fire_weather_index(isi=today.initial_spread_index, bui=today.build_up_index) return today From 01dc387e695a1acbc8fbce22353b8c47d3a5df97 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Mon, 6 Nov 2023 14:30:05 -0800 Subject: [PATCH 26/36] adds util tests --- .../features/moreCast2/slices/dataSlice.ts | 2 +- web/src/features/moreCast2/util.test.ts | 42 ++++++++++++++++++- web/src/features/moreCast2/util.ts | 7 ---- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/web/src/features/moreCast2/slices/dataSlice.ts b/web/src/features/moreCast2/slices/dataSlice.ts index 301346de1..e33560e9d 100644 --- a/web/src/features/moreCast2/slices/dataSlice.ts +++ b/web/src/features/moreCast2/slices/dataSlice.ts @@ -583,7 +583,7 @@ const getNumberOrNaN = (value: number | null) => { * @param forDate The date the row is for. * @returns */ -const createEmptyMoreCast2Row = ( +export const createEmptyMoreCast2Row = ( id: string, stationCode: number, stationName: string, diff --git a/web/src/features/moreCast2/util.test.ts b/web/src/features/moreCast2/util.test.ts index 72cc74ce2..b805e0842 100644 --- a/web/src/features/moreCast2/util.test.ts +++ b/web/src/features/moreCast2/util.test.ts @@ -1,6 +1,14 @@ import { DateTime } from 'luxon' import { ModelChoice } from 'api/moreCast2API' -import { createDateInterval, createWeatherModelLabel, parseForecastsHelper, rowIDHasher } from 'features/moreCast2/util' +import { createEmptyMoreCast2Row } from 'features/moreCast2/slices/dataSlice' +import { + createDateInterval, + createWeatherModelLabel, + parseForecastsHelper, + rowIDHasher, + validActualPredicate, + validForecastPredicate +} from 'features/moreCast2/util' const TEST_DATE = '2023-02-16T20:00:00+00:00' const TEST_DATE2 = '2023-02-17T20:00:00+00:00' @@ -152,3 +160,35 @@ describe('createWeatherModelLabel', () => { expect(result).toBe('GDPS bias') }) }) +describe('validActualPredicate', () => { + const row = createEmptyMoreCast2Row('id', 123, 'testStation', DateTime.fromISO('2023-05-25T09:08:34.123'), 56, -123) + it('should return true if a row contains valid Actual values', () => { + row.precipActual = 1 + row.tempActual = 1 + row.rhActual = 1 + row.windSpeedActual = 1 + const result = validActualPredicate(row) + expect(result).toBe(true) + }) + it('should return false if a row does not contain valid Actual values', () => { + row.precipActual = NaN + const result = validActualPredicate(row) + expect(result).toBe(false) + }) +}) +describe('validForecastPredicate', () => { + const row = createEmptyMoreCast2Row('id', 123, 'testStation', DateTime.fromISO('2023-05-25T09:08:34.123'), 56, -123) + it('should return true if a row contains valid Forecast values', () => { + row.precipForecast = { choice: 'FORECAST', value: 2 } + row.tempForecast = { choice: 'FORECAST', value: 2 } + row.rhForecast = { choice: 'FORECAST', value: 2 } + row.windSpeedForecast = { choice: 'FORECAST', value: 2 } + const result = validForecastPredicate(row) + expect(result).toBe(true) + }) + it('should return false if a row does not contain valid Forecast values', () => { + row.precipForecast = undefined + const result = validForecastPredicate(row) + expect(result).toBe(false) + }) +}) diff --git a/web/src/features/moreCast2/util.ts b/web/src/features/moreCast2/util.ts index c82aa999b..cb0828c14 100644 --- a/web/src/features/moreCast2/util.ts +++ b/web/src/features/moreCast2/util.ts @@ -81,13 +81,6 @@ export const createLabel = (isActual: boolean, label: string) => { return createWeatherModelLabel(label) } -export const getYesterdayRowID = (todayRow: MoreCast2Row): string => { - const yesterdayDate = todayRow.forDate.minus({ days: 1 }) - const yesterdayID = rowIDHasher(todayRow.stationCode, yesterdayDate) - - return yesterdayID -} - export const validActualOrForecastPredicate = (row: MoreCast2Row) => validForecastPredicate(row) || validActualPredicate(row) From 400adaa9a845d79436a018665ea05415a7f35546 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Mon, 6 Nov 2023 15:55:32 -0800 Subject: [PATCH 27/36] adds row filtering tests --- .../components/ForecastSummaryDataGrid.tsx | 34 +++++++----- .../forecastSummaryDataGrid.test.tsx | 53 +++++++++++++++++++ 2 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 web/src/features/moreCast2/components/forecastSummaryDataGrid.test.tsx diff --git a/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx b/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx index ff8b73840..72804fa52 100644 --- a/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx +++ b/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx @@ -37,6 +37,25 @@ interface ForecastSummaryDataGridProps { handleClose: () => void } +export const filterRowsForSimulationFromEdited = ( + editedRow: MoreCast2Row, + allRows: MoreCast2Row[] +): MoreCast2Row[] | undefined => { + if (validForecastPredicate(editedRow)) { + const validRowsForStation = allRows.filter( + row => row.stationCode === editedRow.stationCode && validActualOrForecastPredicate(row) + ) + + const yesterday = editedRow.forDate.minus({ days: 1 }) + const yesterdayRow = validRowsForStation.find(row => row.forDate.toISODate() === yesterday.toISODate()) + + if (yesterdayRow) { + const rowsForSimulation = validRowsForStation.filter(row => row.forDate >= yesterday) + return rowsForSimulation + } else return undefined + } +} + const ForecastSummaryDataGrid = ({ loading, rows, @@ -50,18 +69,9 @@ const ForecastSummaryDataGrid = ({ const processRowUpdate = async (editedRow: MoreCast2Row) => { dispatch(storeUserEditedRows([editedRow])) - if (validForecastPredicate(editedRow)) { - const validRowsForStation = rows.filter( - row => row.stationCode === editedRow.stationCode && validActualOrForecastPredicate(row) - ) - - const yesterday = editedRow.forDate.minus({ days: 1 }) - const yesterdayRow = validRowsForStation.find(row => row.forDate.toISODate() === yesterday.toISODate()) - - if (yesterdayRow) { - const rowsForSimulation = validRowsForStation.filter(row => row.forDate >= yesterday) - dispatch(getSimulatedIndices(rowsForSimulation)) - } + const rowsForSimulation = filterRowsForSimulationFromEdited(editedRow, rows) + if (rowsForSimulation) { + dispatch(getSimulatedIndices(rowsForSimulation)) } return editedRow diff --git a/web/src/features/moreCast2/components/forecastSummaryDataGrid.test.tsx b/web/src/features/moreCast2/components/forecastSummaryDataGrid.test.tsx new file mode 100644 index 000000000..30942be93 --- /dev/null +++ b/web/src/features/moreCast2/components/forecastSummaryDataGrid.test.tsx @@ -0,0 +1,53 @@ +import { filterRowsForSimulationFromEdited } from 'features/moreCast2/components/ForecastSummaryDataGrid' +import { DateTime } from 'luxon' +import { createEmptyMoreCast2Row } from 'features/moreCast2/slices/dataSlice' +import { MoreCast2Row } from 'features/moreCast2/interfaces' + +const TEST_DATE = DateTime.fromISO('2023-02-16T20:00:00+00:00') + +const buildValidForecastRow = (stationCode: number, forDate: DateTime): MoreCast2Row => { + const forecastRow = createEmptyMoreCast2Row('id', stationCode, 'stationName', forDate, 1, 2) + forecastRow.precipForecast = { choice: 'FORECAST', value: 2 } + forecastRow.tempForecast = { choice: 'FORECAST', value: 2 } + forecastRow.rhForecast = { choice: 'FORECAST', value: 2 } + forecastRow.windSpeedForecast = { choice: 'FORECAST', value: 2 } + + return forecastRow +} + +const buildValidActualRow = (stationCode: number, forDate: DateTime): MoreCast2Row => { + const actualRow = createEmptyMoreCast2Row('id', stationCode, 'stationName', forDate, 1, 2) + actualRow.precipActual = 1 + actualRow.tempActual = 1 + actualRow.rhActual = 1 + actualRow.windSpeedActual = 1 + + return actualRow +} + +const buildInvalidForecastRow = (stationCode: number, forDate: DateTime): MoreCast2Row => { + const forecastRow = createEmptyMoreCast2Row('id', stationCode, 'stationName', forDate, 1, 2) + + return forecastRow +} + +// rows to filter in +const actual1A = buildValidActualRow(1, TEST_DATE) +const forecast1A = buildValidForecastRow(1, TEST_DATE.plus({ days: 1 })) // edited row +const forecast1B = buildValidForecastRow(1, TEST_DATE.plus({ days: 2 })) + +// rows to filter out +const actual1B = buildValidActualRow(1, TEST_DATE.minus({ days: 1 })) +const forecast1C = buildInvalidForecastRow(1, TEST_DATE.plus({ days: 3 })) +const actual2A = buildValidActualRow(2, TEST_DATE.minus({ days: 1 })) +const forecast2A = buildValidForecastRow(2, TEST_DATE.plus({ days: 1 })) + +const rows = [actual1A, actual1B, forecast1A, forecast1B, forecast1C, actual2A, forecast2A] + +describe('filterRowsForSimulationFromEdited', () => { + it('should filter for valid rows before and after the edited row', () => { + const filteredRows = filterRowsForSimulationFromEdited(forecast1A, rows) + expect(filteredRows).toEqual(expect.arrayContaining([actual1A, forecast1A, forecast1B])) + expect(filteredRows).not.toEqual(expect.arrayContaining([actual1B, forecast1C, actual2A, forecast2A])) + }) +}) From e9ec48dd652f6722b674aac3b4f3a87f83956c04 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Tue, 7 Nov 2023 09:13:27 -0800 Subject: [PATCH 28/36] row filter tests and rowFilters.ts --- .../components/ForecastSummaryDataGrid.tsx | 21 +------ .../moreCast2/components/TabbedDataGrid.tsx | 23 +------- ...ryDataGrid.test.tsx => rowFilters.test.ts} | 57 +++++++++++++++---- web/src/features/moreCast2/rowFilters.ts | 41 +++++++++++++ 4 files changed, 90 insertions(+), 52 deletions(-) rename web/src/features/moreCast2/{components/forecastSummaryDataGrid.test.tsx => rowFilters.test.ts} (50%) create mode 100644 web/src/features/moreCast2/rowFilters.ts diff --git a/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx b/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx index 72804fa52..787b97557 100644 --- a/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx +++ b/web/src/features/moreCast2/components/ForecastSummaryDataGrid.tsx @@ -9,7 +9,7 @@ import { DataGridColumns } from 'features/moreCast2/components/DataGridColumns' import { storeUserEditedRows, getSimulatedIndices } from 'features/moreCast2/slices/dataSlice' import { AppDispatch } from 'app/store' import { useDispatch } from 'react-redux' -import { validActualOrForecastPredicate, validForecastPredicate } from 'features/moreCast2/util' +import { filterRowsForSimulationFromEdited } from 'features/moreCast2/rowFilters' const PREFIX = 'ForecastSummaryDataGrid' @@ -37,25 +37,6 @@ interface ForecastSummaryDataGridProps { handleClose: () => void } -export const filterRowsForSimulationFromEdited = ( - editedRow: MoreCast2Row, - allRows: MoreCast2Row[] -): MoreCast2Row[] | undefined => { - if (validForecastPredicate(editedRow)) { - const validRowsForStation = allRows.filter( - row => row.stationCode === editedRow.stationCode && validActualOrForecastPredicate(row) - ) - - const yesterday = editedRow.forDate.minus({ days: 1 }) - const yesterdayRow = validRowsForStation.find(row => row.forDate.toISODate() === yesterday.toISODate()) - - if (yesterdayRow) { - const rowsForSimulation = validRowsForStation.filter(row => row.forDate >= yesterday) - return rowsForSimulation - } else return undefined - } -} - const ForecastSummaryDataGrid = ({ loading, rows, diff --git a/web/src/features/moreCast2/components/TabbedDataGrid.tsx b/web/src/features/moreCast2/components/TabbedDataGrid.tsx index 1e3deeffe..c5a9827e2 100644 --- a/web/src/features/moreCast2/components/TabbedDataGrid.tsx +++ b/web/src/features/moreCast2/components/TabbedDataGrid.tsx @@ -26,7 +26,7 @@ import { isForecastRowPredicate, getRowsToSave, isForecastValid } from 'features import MoreCast2DateRangePicker from 'features/moreCast2/components/MoreCast2DateRangePicker' import { AppDispatch } from 'app/store' import { deepClone } from '@mui/x-data-grid/utils/utils' -import { validActualPredicate, validForecastPredicate } from 'features/moreCast2/util' +import { filterAllVisibleRowsForSimulation } from 'features/moreCast2/rowFilters' export const Root = styled('div')({ display: 'flex', @@ -194,23 +194,6 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp const [clickedColDef, setClickedColDef] = useState(null) - const filterRowsForSimulation = (rows: MoreCast2Row[]): MoreCast2Row[] | undefined => { - const forecasts = rows.filter(validForecastPredicate) - const actuals = rows.filter(validActualPredicate) - const mostRecentActualMap = new Map() - - for (const row of actuals) { - const recentActual = mostRecentActualMap.get(row.stationCode) - if (!recentActual || recentActual.forDate < row.forDate) { - mostRecentActualMap.set(row.stationCode, row) - } - } - const mostRecentActuals = Array.from(mostRecentActualMap.values()) - const rowsForSimulation = [...mostRecentActuals, ...forecasts] - - return forecasts.length > 0 ? rowsForSimulation : undefined - } - const mapForecastChoiceLabels = (newRows: MoreCast2Row[], storedRows: MoreCast2Row[]): MoreCast2Row[] => { const storedRowChoicesMap = new Map() @@ -271,7 +254,7 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp predictionItem.value = mostRecentValue as number }) } - const rowsForSimulation = filterRowsForSimulation(newRows) + const rowsForSimulation = filterAllVisibleRowsForSimulation(newRows) if (rowsForSimulation) { dispatch(getSimulatedIndices(rowsForSimulation)) } @@ -298,7 +281,7 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp predictionItem.value = (row[sourceKey] as number) ?? NaN } } - const rowsForSimulation = filterRowsForSimulation(newRows) + const rowsForSimulation = filterAllVisibleRowsForSimulation(newRows) if (rowsForSimulation) { dispatch(getSimulatedIndices(rowsForSimulation)) } diff --git a/web/src/features/moreCast2/components/forecastSummaryDataGrid.test.tsx b/web/src/features/moreCast2/rowFilters.test.ts similarity index 50% rename from web/src/features/moreCast2/components/forecastSummaryDataGrid.test.tsx rename to web/src/features/moreCast2/rowFilters.test.ts index 30942be93..43a6a4f51 100644 --- a/web/src/features/moreCast2/components/forecastSummaryDataGrid.test.tsx +++ b/web/src/features/moreCast2/rowFilters.test.ts @@ -1,7 +1,7 @@ -import { filterRowsForSimulationFromEdited } from 'features/moreCast2/components/ForecastSummaryDataGrid' import { DateTime } from 'luxon' import { createEmptyMoreCast2Row } from 'features/moreCast2/slices/dataSlice' import { MoreCast2Row } from 'features/moreCast2/interfaces' +import { filterRowsForSimulationFromEdited, filterAllVisibleRowsForSimulation } from 'features/moreCast2/rowFilters' const TEST_DATE = DateTime.fromISO('2023-02-16T20:00:00+00:00') @@ -31,23 +31,56 @@ const buildInvalidForecastRow = (stationCode: number, forDate: DateTime): MoreCa return forecastRow } -// rows to filter in -const actual1A = buildValidActualRow(1, TEST_DATE) +const actual1B = buildValidActualRow(1, TEST_DATE.minus({ days: 1 })) // exclude +const actual1A = buildValidActualRow(1, TEST_DATE) // include const forecast1A = buildValidForecastRow(1, TEST_DATE.plus({ days: 1 })) // edited row -const forecast1B = buildValidForecastRow(1, TEST_DATE.plus({ days: 2 })) +const forecast1B = buildValidForecastRow(1, TEST_DATE.plus({ days: 2 })) // include +const forecast1C = buildInvalidForecastRow(1, TEST_DATE.plus({ days: 3 })) // exclude -// rows to filter out -const actual1B = buildValidActualRow(1, TEST_DATE.minus({ days: 1 })) -const forecast1C = buildInvalidForecastRow(1, TEST_DATE.plus({ days: 3 })) -const actual2A = buildValidActualRow(2, TEST_DATE.minus({ days: 1 })) -const forecast2A = buildValidForecastRow(2, TEST_DATE.plus({ days: 1 })) +const actual2B = buildValidActualRow(2, TEST_DATE.minus({ days: 1 })) // exclude +const actual2A = buildValidActualRow(2, TEST_DATE) // include +const forecast2A = buildValidForecastRow(2, TEST_DATE.plus({ days: 1 })) // include +const forecast2B = buildValidForecastRow(2, TEST_DATE.plus({ days: 2 })) // include +const forecast2C = buildInvalidForecastRow(2, TEST_DATE.plus({ days: 3 })) // exclude +const forecast2D = buildInvalidForecastRow(2, TEST_DATE.plus({ days: 4 })) // exclude -const rows = [actual1A, actual1B, forecast1A, forecast1B, forecast1C, actual2A, forecast2A] +const rows = [ + actual1A, + actual1B, + forecast1A, + forecast1B, + forecast1C, + actual2A, + forecast2A, + actual2B, + forecast2B, + forecast2C, + forecast2D +] describe('filterRowsForSimulationFromEdited', () => { + const filteredRows = filterRowsForSimulationFromEdited(forecast1A, rows) it('should filter for valid rows before and after the edited row', () => { - const filteredRows = filterRowsForSimulationFromEdited(forecast1A, rows) expect(filteredRows).toEqual(expect.arrayContaining([actual1A, forecast1A, forecast1B])) - expect(filteredRows).not.toEqual(expect.arrayContaining([actual1B, forecast1C, actual2A, forecast2A])) + }) + it('should not include invalid rows, rows from other stations, or unnecessary rows for calculations', () => { + expect(filteredRows).not.toEqual( + expect.arrayContaining([actual1B, forecast1C, actual2A, forecast2A, actual2B, forecast2B, forecast2C]) + ) + }) +}) +describe('filterAllVisibleRowsForSimulation', () => { + const filteredRows = filterAllVisibleRowsForSimulation(rows) + it('should only include valid forecasts and most recent actuals for each station', () => { + expect(filteredRows).toEqual( + expect.arrayContaining([actual1A, forecast1A, forecast1B, actual2A, forecast2A, forecast2B]) + ) + }) + it('should not contain invalid forecasts', () => { + expect(filteredRows).not.toContain(forecast2D) + }) + it('should not contain unnecessary actuals', () => { + expect(filteredRows).not.toContain(actual2B) + expect(filteredRows).not.toContain(actual1B) }) }) diff --git a/web/src/features/moreCast2/rowFilters.ts b/web/src/features/moreCast2/rowFilters.ts new file mode 100644 index 000000000..cd82f09de --- /dev/null +++ b/web/src/features/moreCast2/rowFilters.ts @@ -0,0 +1,41 @@ +import { MoreCast2Row } from 'features/moreCast2/interfaces' +import { validForecastPredicate, validActualPredicate, validActualOrForecastPredicate } from 'features/moreCast2/util' + +export const filterAllVisibleRowsForSimulation = (rows: MoreCast2Row[]): MoreCast2Row[] | undefined => { + const forecasts = rows.filter(validForecastPredicate) + const actuals = rows.filter(validActualPredicate) + const mostRecentActualMap = new Map() + let rowsForSimulation = undefined + + if (forecasts.length > 0) { + for (const row of actuals) { + const recentActual = mostRecentActualMap.get(row.stationCode) + if (!recentActual || recentActual.forDate < row.forDate) { + mostRecentActualMap.set(row.stationCode, row) + } + } + const mostRecentActuals = Array.from(mostRecentActualMap.values()) + rowsForSimulation = [...mostRecentActuals, ...forecasts] + } + + return rowsForSimulation +} + +export const filterRowsForSimulationFromEdited = ( + editedRow: MoreCast2Row, + allRows: MoreCast2Row[] +): MoreCast2Row[] | undefined => { + if (validForecastPredicate(editedRow)) { + const validRowsForStation = allRows.filter( + row => row.stationCode === editedRow.stationCode && validActualOrForecastPredicate(row) + ) + + const yesterday = editedRow.forDate.minus({ days: 1 }) + const yesterdayRow = validRowsForStation.find(row => row.forDate.toISODate() === yesterday.toISODate()) + + if (yesterdayRow) { + const rowsForSimulation = validRowsForStation.filter(row => row.forDate >= yesterday) + return rowsForSimulation + } else return undefined + } +} From 1b57b961d8c6b83cbedcb89cbc57eca096164b7c Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Tue, 7 Nov 2023 09:25:38 -0800 Subject: [PATCH 29/36] update rowFilters test --- web/src/features/moreCast2/rowFilters.test.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/web/src/features/moreCast2/rowFilters.test.ts b/web/src/features/moreCast2/rowFilters.test.ts index 43a6a4f51..5245a2332 100644 --- a/web/src/features/moreCast2/rowFilters.test.ts +++ b/web/src/features/moreCast2/rowFilters.test.ts @@ -64,8 +64,8 @@ describe('filterRowsForSimulationFromEdited', () => { expect(filteredRows).toEqual(expect.arrayContaining([actual1A, forecast1A, forecast1B])) }) it('should not include invalid rows, rows from other stations, or unnecessary rows for calculations', () => { - expect(filteredRows).not.toEqual( - expect.arrayContaining([actual1B, forecast1C, actual2A, forecast2A, actual2B, forecast2B, forecast2C]) + expect(filteredRows).toEqual( + expect.not.arrayContaining([actual1B, forecast1C, actual2A, forecast2A, actual2B, forecast2B, forecast2C]) ) }) }) @@ -77,7 +77,18 @@ describe('filterAllVisibleRowsForSimulation', () => { ) }) it('should not contain invalid forecasts', () => { - expect(filteredRows).not.toContain(forecast2D) + expect(filteredRows).toEqual( + expect.not.arrayContaining([ + actual1B, + forecast1C, + actual2A, + forecast2A, + actual2B, + forecast2B, + forecast2C, + forecast2D + ]) + ) }) it('should not contain unnecessary actuals', () => { expect(filteredRows).not.toContain(actual2B) From 776db259c8f2a779d224673a2171b3cb8d1e76d5 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Tue, 7 Nov 2023 09:32:16 -0800 Subject: [PATCH 30/36] rowFilters test refinement --- web/src/features/moreCast2/rowFilters.test.ts | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/web/src/features/moreCast2/rowFilters.test.ts b/web/src/features/moreCast2/rowFilters.test.ts index 5245a2332..0e3b0c52e 100644 --- a/web/src/features/moreCast2/rowFilters.test.ts +++ b/web/src/features/moreCast2/rowFilters.test.ts @@ -63,10 +63,16 @@ describe('filterRowsForSimulationFromEdited', () => { it('should filter for valid rows before and after the edited row', () => { expect(filteredRows).toEqual(expect.arrayContaining([actual1A, forecast1A, forecast1B])) }) - it('should not include invalid rows, rows from other stations, or unnecessary rows for calculations', () => { - expect(filteredRows).toEqual( - expect.not.arrayContaining([actual1B, forecast1C, actual2A, forecast2A, actual2B, forecast2B, forecast2C]) - ) + it('should not contain invalid forecasts', () => { + expect(filteredRows).not.toContain(forecast1C) + }) + it('should not contain unecessary actuals', () => { + expect(filteredRows).not.toContain(actual1B) + }) + it('should not contain rows from otehr stations', () => { + expect(filteredRows).not.toContain(forecast2A) + expect(filteredRows).not.toContain(forecast2B) + expect(filteredRows).not.toContain(actual2A) }) }) describe('filterAllVisibleRowsForSimulation', () => { @@ -77,18 +83,9 @@ describe('filterAllVisibleRowsForSimulation', () => { ) }) it('should not contain invalid forecasts', () => { - expect(filteredRows).toEqual( - expect.not.arrayContaining([ - actual1B, - forecast1C, - actual2A, - forecast2A, - actual2B, - forecast2B, - forecast2C, - forecast2D - ]) - ) + expect(filteredRows).not.toContain(forecast1C) + expect(filteredRows).not.toContain(forecast2C) + expect(filteredRows).not.toContain(forecast2D) }) it('should not contain unnecessary actuals', () => { expect(filteredRows).not.toContain(actual2B) From 708aecd0f9c5ff4ca36a892b81242c6ce615db25 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Tue, 7 Nov 2023 09:52:23 -0800 Subject: [PATCH 31/36] typos --- web/src/features/moreCast2/rowFilters.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/src/features/moreCast2/rowFilters.test.ts b/web/src/features/moreCast2/rowFilters.test.ts index 0e3b0c52e..c14047715 100644 --- a/web/src/features/moreCast2/rowFilters.test.ts +++ b/web/src/features/moreCast2/rowFilters.test.ts @@ -66,10 +66,10 @@ describe('filterRowsForSimulationFromEdited', () => { it('should not contain invalid forecasts', () => { expect(filteredRows).not.toContain(forecast1C) }) - it('should not contain unecessary actuals', () => { + it('should not contain unnecessary actuals', () => { expect(filteredRows).not.toContain(actual1B) }) - it('should not contain rows from otehr stations', () => { + it('should not contain rows from other stations', () => { expect(filteredRows).not.toContain(forecast2A) expect(filteredRows).not.toContain(forecast2B) expect(filteredRows).not.toContain(actual2A) From a4fa72100117be509852df631c78b8dd1b1e3871 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Tue, 7 Nov 2023 16:19:26 -0800 Subject: [PATCH 32/36] camel to snake --- api/app/routers/morecast_v2.py | 2 +- api/app/schemas/morecast_v2.py | 2 +- web/src/api/moreCast2API.ts | 2 +- web/src/features/moreCast2/slices/dataSlice.test.ts | 4 ++-- web/src/features/moreCast2/slices/dataSlice.ts | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/app/routers/morecast_v2.py b/api/app/routers/morecast_v2.py index 22b216c2a..5a0398803 100644 --- a/api/app/routers/morecast_v2.py +++ b/api/app/routers/morecast_v2.py @@ -247,4 +247,4 @@ async def calculate_forecasted_indices(simulate_records: SimulateIndeterminateIn actuals = [indeterminate for indeterminate in indeterminates if indeterminate.determinate == WeatherDeterminate.ACTUAL] _, forecasts = get_fwi_values(actuals, forecasts) - return (SimulatedWeatherIndeterminateResponse(simulatedForecasts=forecasts)) + return (SimulatedWeatherIndeterminateResponse(simulated_forecasts=forecasts)) diff --git a/api/app/schemas/morecast_v2.py b/api/app/schemas/morecast_v2.py index 99d8d899f..14efeb54c 100644 --- a/api/app/schemas/morecast_v2.py +++ b/api/app/schemas/morecast_v2.py @@ -148,7 +148,7 @@ class SimulateIndeterminateIndices(BaseModel): class SimulatedWeatherIndeterminateResponse(BaseModel): - simulatedForecasts: List[WeatherIndeterminate] + simulated_forecasts: List[WeatherIndeterminate] class WF1ForecastRecordType(BaseModel): diff --git a/web/src/api/moreCast2API.ts b/web/src/api/moreCast2API.ts index ec291fcc2..f8278993f 100644 --- a/web/src/api/moreCast2API.ts +++ b/web/src/api/moreCast2API.ts @@ -153,7 +153,7 @@ export interface WeatherIndeterminateResponse { } export interface UpdatedWeatherIndeterminateResponse { - simulatedForecasts: WeatherIndeterminate[] + simulated_forecasts: WeatherIndeterminate[] } export const ModelOptions: ModelType[] = ModelChoices.filter(choice => !isEqual(choice, ModelChoice.MANUAL)) diff --git a/web/src/features/moreCast2/slices/dataSlice.test.ts b/web/src/features/moreCast2/slices/dataSlice.test.ts index ba8dcbf30..b4f79a875 100644 --- a/web/src/features/moreCast2/slices/dataSlice.test.ts +++ b/web/src/features/moreCast2/slices/dataSlice.test.ts @@ -173,7 +173,7 @@ describe('dataSlice', () => { dataSliceReducer( initialState, simulateWeatherIndeterminatesSuccess({ - simulatedForecasts: [] + simulated_forecasts: [] }) ).forecasts ).toEqual([]) @@ -209,7 +209,7 @@ describe('dataSlice', () => { dataSliceReducer( { ...initialState, forecasts: [weatherIndeterminate1, weatherIndeterminate2] }, simulateWeatherIndeterminatesSuccess({ - simulatedForecasts: [updatedWeatherIndeterminate2] + simulated_forecasts: [updatedWeatherIndeterminate2] }) ).forecasts ).toEqual([weatherIndeterminate1, updatedWeatherIndeterminate2]) diff --git a/web/src/features/moreCast2/slices/dataSlice.ts b/web/src/features/moreCast2/slices/dataSlice.ts index e33560e9d..b3101c66a 100644 --- a/web/src/features/moreCast2/slices/dataSlice.ts +++ b/web/src/features/moreCast2/slices/dataSlice.ts @@ -61,7 +61,7 @@ const dataSlice = createSlice({ state.loading = false }, simulateWeatherIndeterminatesSuccess(state: State, action: PayloadAction) { - const updatedForecasts = addUniqueIds(action.payload.simulatedForecasts) + const updatedForecasts = addUniqueIds(action.payload.simulated_forecasts) state.forecasts = state.forecasts.map(forecast => { const updatedForecast = updatedForecasts.find(item => item.id === forecast.id) From 983e4bf5f3c63bc8a62518998552480a294a42a8 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Wed, 8 Nov 2023 10:09:08 -0800 Subject: [PATCH 33/36] add tests for mapForecastChoiceLabels --- .../moreCast2/components/TabbedDataGrid.tsx | 21 +----------------- web/src/features/moreCast2/rowFilters.test.ts | 20 ++++++++++++----- web/src/features/moreCast2/util.test.ts | 22 +++++++++++++++++++ web/src/features/moreCast2/util.ts | 20 +++++++++++++++++ 4 files changed, 57 insertions(+), 26 deletions(-) diff --git a/web/src/features/moreCast2/components/TabbedDataGrid.tsx b/web/src/features/moreCast2/components/TabbedDataGrid.tsx index c5a9827e2..c501992a1 100644 --- a/web/src/features/moreCast2/components/TabbedDataGrid.tsx +++ b/web/src/features/moreCast2/components/TabbedDataGrid.tsx @@ -27,6 +27,7 @@ import MoreCast2DateRangePicker from 'features/moreCast2/components/MoreCast2Dat import { AppDispatch } from 'app/store' import { deepClone } from '@mui/x-data-grid/utils/utils' import { filterAllVisibleRowsForSimulation } from 'features/moreCast2/rowFilters' +import { mapForecastChoiceLabels } from 'features/moreCast2/util' export const Root = styled('div')({ display: 'flex', @@ -194,26 +195,6 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp const [clickedColDef, setClickedColDef] = useState(null) - const mapForecastChoiceLabels = (newRows: MoreCast2Row[], storedRows: MoreCast2Row[]): MoreCast2Row[] => { - const storedRowChoicesMap = new Map() - - for (const row of storedRows) { - storedRowChoicesMap.set(row.id, row) - } - - for (const row of newRows) { - const matchingRow = storedRowChoicesMap.get(row.id) - if (matchingRow) { - row.precipForecast = matchingRow.precipForecast - row.tempForecast = matchingRow.tempForecast - row.rhForecast = matchingRow.rhForecast - row.windDirectionForecast = matchingRow.windDirectionForecast - row.windSpeedForecast = matchingRow.windSpeedForecast - } - } - return newRows - } - // Updates forecast field for a given weather parameter (temp, rh, precip, etc...) based on the // model/source selected in the column header menu const updateColumnWithModel = (modelType: ModelType, colDef: GridColDef) => { diff --git a/web/src/features/moreCast2/rowFilters.test.ts b/web/src/features/moreCast2/rowFilters.test.ts index c14047715..ebc0e5d9e 100644 --- a/web/src/features/moreCast2/rowFilters.test.ts +++ b/web/src/features/moreCast2/rowFilters.test.ts @@ -2,15 +2,23 @@ import { DateTime } from 'luxon' import { createEmptyMoreCast2Row } from 'features/moreCast2/slices/dataSlice' import { MoreCast2Row } from 'features/moreCast2/interfaces' import { filterRowsForSimulationFromEdited, filterAllVisibleRowsForSimulation } from 'features/moreCast2/rowFilters' +import { ModelType } from 'api/moreCast2API' +import { rowIDHasher } from 'features/moreCast2/util' const TEST_DATE = DateTime.fromISO('2023-02-16T20:00:00+00:00') -const buildValidForecastRow = (stationCode: number, forDate: DateTime): MoreCast2Row => { - const forecastRow = createEmptyMoreCast2Row('id', stationCode, 'stationName', forDate, 1, 2) - forecastRow.precipForecast = { choice: 'FORECAST', value: 2 } - forecastRow.tempForecast = { choice: 'FORECAST', value: 2 } - forecastRow.rhForecast = { choice: 'FORECAST', value: 2 } - forecastRow.windSpeedForecast = { choice: 'FORECAST', value: 2 } +export const buildValidForecastRow = ( + stationCode: number, + forDate: DateTime, + choice: ModelType = 'FORECAST' +): MoreCast2Row => { + const id = rowIDHasher(stationCode, forDate) + const forecastRow = createEmptyMoreCast2Row(id, stationCode, 'stationName', forDate, 1, 2) + forecastRow.precipForecast = { choice: choice, value: 2 } + forecastRow.tempForecast = { choice: choice, value: 2 } + forecastRow.rhForecast = { choice: choice, value: 2 } + forecastRow.windSpeedForecast = { choice: choice, value: 2 } + forecastRow.id = id return forecastRow } diff --git a/web/src/features/moreCast2/util.test.ts b/web/src/features/moreCast2/util.test.ts index b805e0842..738d1425f 100644 --- a/web/src/features/moreCast2/util.test.ts +++ b/web/src/features/moreCast2/util.test.ts @@ -4,15 +4,18 @@ import { createEmptyMoreCast2Row } from 'features/moreCast2/slices/dataSlice' import { createDateInterval, createWeatherModelLabel, + mapForecastChoiceLabels, parseForecastsHelper, rowIDHasher, validActualPredicate, validForecastPredicate } from 'features/moreCast2/util' +import { buildValidForecastRow } from 'features/moreCast2/rowFilters.test' const TEST_DATE = '2023-02-16T20:00:00+00:00' const TEST_DATE2 = '2023-02-17T20:00:00+00:00' const TEST_CODE = 209 +const TEST_DATETIME = DateTime.fromISO(TEST_DATE) describe('createDateInterval', () => { it('should return array with single date when fromDate and toDate are the same', () => { @@ -192,3 +195,22 @@ describe('validForecastPredicate', () => { expect(result).toBe(false) }) }) +describe('mapForecastChoiceLabels', () => { + const forecast1A = buildValidForecastRow(123, TEST_DATETIME, 'FORECAST') + const forecast1B = buildValidForecastRow(123, TEST_DATETIME.plus({ days: 1 }), 'FORECAST') + const newRows = [forecast1A, forecast1B] + + const forecast2A = buildValidForecastRow(123, TEST_DATETIME, 'GDPS') + const forecast2B = buildValidForecastRow(123, TEST_DATETIME.plus({ days: 1 }), 'MANUAL') + forecast2A.tempForecast!.choice = 'HRDPS' + forecast2B.precipForecast!.choice = 'GFS' + const storedRows = [forecast2A, forecast2B] + + it('should map the correct label to the correct row', () => { + const labelledRows = mapForecastChoiceLabels(newRows, storedRows) + expect(labelledRows[0].tempForecast!.choice).toBe('HRDPS') + expect(labelledRows[0].precipForecast!.choice).toBe('GDPS') + expect(labelledRows[1].precipForecast!.choice).toBe('GFS') + expect(labelledRows[1].rhForecast!.choice).toBe('MANUAL') + }) +}) diff --git a/web/src/features/moreCast2/util.ts b/web/src/features/moreCast2/util.ts index cb0828c14..8b9109e65 100644 --- a/web/src/features/moreCast2/util.ts +++ b/web/src/features/moreCast2/util.ts @@ -97,3 +97,23 @@ export const validForecastPredicate = (row: MoreCast2Row) => !isNaN(row.tempForecast.value) && !isUndefined(row.windSpeedForecast) && !isNaN(row.windSpeedForecast.value) + +export const mapForecastChoiceLabels = (newRows: MoreCast2Row[], storedRows: MoreCast2Row[]): MoreCast2Row[] => { + const storedRowChoicesMap = new Map() + + for (const row of storedRows) { + storedRowChoicesMap.set(row.id, row) + } + + for (const row of newRows) { + const matchingRow = storedRowChoicesMap.get(row.id) + if (matchingRow) { + row.precipForecast = matchingRow.precipForecast + row.tempForecast = matchingRow.tempForecast + row.rhForecast = matchingRow.rhForecast + row.windDirectionForecast = matchingRow.windDirectionForecast + row.windSpeedForecast = matchingRow.windSpeedForecast + } + } + return newRows +} From 3c937f54e5f28258183a7ffbcd75acaafd66f129 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Wed, 8 Nov 2023 10:17:45 -0800 Subject: [PATCH 34/36] address some comments --- api/app/morecast_v2/forecasts.py | 2 +- web/src/features/moreCast2/components/TabbedDataGrid.tsx | 4 ++-- web/src/features/moreCast2/rowFilters.ts | 3 ++- web/src/features/moreCast2/slices/dataSlice.ts | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/app/morecast_v2/forecasts.py b/api/app/morecast_v2/forecasts.py index 7194b1fa0..3be5acb61 100644 --- a/api/app/morecast_v2/forecasts.py +++ b/api/app/morecast_v2/forecasts.py @@ -132,7 +132,7 @@ def get_fwi_values(actuals: List[WeatherIndeterminate], forecasts: List[WeatherI for idx, indeterminate in enumerate(all_indeterminates): last_indeterminate = indeterminates_dict[indeterminate.station_code].get( indeterminate.utc_timestamp.date() - timedelta(days=1), None) - if last_indeterminate: + if last_indeterminate is not None: updated_forecast = calculate_fwi_values(last_indeterminate, indeterminate) all_indeterminates[idx] = updated_forecast diff --git a/web/src/features/moreCast2/components/TabbedDataGrid.tsx b/web/src/features/moreCast2/components/TabbedDataGrid.tsx index c501992a1..36b884bf5 100644 --- a/web/src/features/moreCast2/components/TabbedDataGrid.tsx +++ b/web/src/features/moreCast2/components/TabbedDataGrid.tsx @@ -212,7 +212,7 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp // Persistence forecasting. Get the most recent actual and persist it through the rest of the // days in this forecast period. - const updateColumnFromLastActual = async (forecastField: keyof MoreCast2Row, actualField: keyof MoreCast2Row) => { + const updateColumnFromLastActual = (forecastField: keyof MoreCast2Row, actualField: keyof MoreCast2Row) => { const newRows = [...visibleRows] // Group our visible rows by station code and work on each group sepearately const groupedByStationCode = groupBy(newRows, 'stationCode') @@ -244,7 +244,7 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp setVisibleRows(newRows) } - const updateColumnFromModel = async ( + const updateColumnFromModel = ( modelType: ModelType, forecastField: keyof MoreCast2Row, actualField: keyof MoreCast2Row, diff --git a/web/src/features/moreCast2/rowFilters.ts b/web/src/features/moreCast2/rowFilters.ts index cd82f09de..2b0052f6d 100644 --- a/web/src/features/moreCast2/rowFilters.ts +++ b/web/src/features/moreCast2/rowFilters.ts @@ -36,6 +36,7 @@ export const filterRowsForSimulationFromEdited = ( if (yesterdayRow) { const rowsForSimulation = validRowsForStation.filter(row => row.forDate >= yesterday) return rowsForSimulation - } else return undefined + } } + return undefined } diff --git a/web/src/features/moreCast2/slices/dataSlice.ts b/web/src/features/moreCast2/slices/dataSlice.ts index b3101c66a..68284fbb5 100644 --- a/web/src/features/moreCast2/slices/dataSlice.ts +++ b/web/src/features/moreCast2/slices/dataSlice.ts @@ -148,7 +148,7 @@ export const getWeatherIndeterminates = /** * Use the morecast2API to get simulated Fire Weather Index value from the backend. - * Results are stored the Redux store. + * Results are stored in the Redux store. * @param rowsForSimulation List of MoreCast2Row's to simulate. The first row in the array must contain * valid values for all Fire Weather Indices. * @returns Array of MoreCast2Rows From 59dc6f442cf12c39b14278f58ce55ee3fe93b961 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Wed, 8 Nov 2023 12:13:40 -0800 Subject: [PATCH 35/36] address more comments --- web/src/features/moreCast2/components/TabbedDataGrid.tsx | 4 ---- web/src/features/moreCast2/slices/dataSlice.ts | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/web/src/features/moreCast2/components/TabbedDataGrid.tsx b/web/src/features/moreCast2/components/TabbedDataGrid.tsx index 36b884bf5..c8341d162 100644 --- a/web/src/features/moreCast2/components/TabbedDataGrid.tsx +++ b/web/src/features/moreCast2/components/TabbedDataGrid.tsx @@ -99,10 +99,6 @@ const TabbedDataGrid = ({ morecast2Rows, fromTo, setFromTo }: TabbedDataGridProp setContextMenu(null) } - useEffect(() => { - setAllRows([...morecast2Rows]) - }, [morecast2Rows]) - useEffect(() => { const labelledRows = mapForecastChoiceLabels(morecast2Rows, deepClone(userEditedRows)) setAllRows(labelledRows) diff --git a/web/src/features/moreCast2/slices/dataSlice.ts b/web/src/features/moreCast2/slices/dataSlice.ts index 68284fbb5..079eef5b6 100644 --- a/web/src/features/moreCast2/slices/dataSlice.ts +++ b/web/src/features/moreCast2/slices/dataSlice.ts @@ -392,8 +392,8 @@ export const fillMissingWeatherIndeterminates = ( for (const [key, values] of Object.entries(groupedByStationCode)) { const stationCode = parseInt(key) const stationName = stationMap.get(stationCode) ?? '' - const latitude = values[0]?.latitude ?? 0 - const longitude = values[0]?.longitude ?? 0 + const latitude = values[0]?.latitude + const longitude = values[0]?.longitude // We expect one actual per date in our date interval if (values.length < dateInterval.length) { for (const date of dateInterval) { From 862f2763ade2d5751e64e1bf671655b567285852 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Wed, 8 Nov 2023 12:32:43 -0800 Subject: [PATCH 36/36] add tests for rowFilters returning undefined --- web/src/features/moreCast2/rowFilters.test.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/web/src/features/moreCast2/rowFilters.test.ts b/web/src/features/moreCast2/rowFilters.test.ts index ebc0e5d9e..b2af1ae38 100644 --- a/web/src/features/moreCast2/rowFilters.test.ts +++ b/web/src/features/moreCast2/rowFilters.test.ts @@ -68,7 +68,7 @@ const rows = [ describe('filterRowsForSimulationFromEdited', () => { const filteredRows = filterRowsForSimulationFromEdited(forecast1A, rows) - it('should filter for valid rows before and after the edited row', () => { + it('should filter for valid rows before and after the edited row ', () => { expect(filteredRows).toEqual(expect.arrayContaining([actual1A, forecast1A, forecast1B])) }) it('should not contain invalid forecasts', () => { @@ -82,6 +82,11 @@ describe('filterRowsForSimulationFromEdited', () => { expect(filteredRows).not.toContain(forecast2B) expect(filteredRows).not.toContain(actual2A) }) + it('should return undefined if yesterday does not contain a valid row', () => { + actual1A.precipActual = NaN + const filteredRows = filterRowsForSimulationFromEdited(forecast1A, rows) + expect(filteredRows).toBe(undefined) + }) }) describe('filterAllVisibleRowsForSimulation', () => { const filteredRows = filterAllVisibleRowsForSimulation(rows) @@ -99,4 +104,9 @@ describe('filterAllVisibleRowsForSimulation', () => { expect(filteredRows).not.toContain(actual2B) expect(filteredRows).not.toContain(actual1B) }) + it('should return undefined if there are no valid forecasts', () => { + const rows = [actual1A, forecast1C, forecast2C, forecast2D] + const filteredRows = filterAllVisibleRowsForSimulation(rows) + expect(filteredRows).toBe(undefined) + }) })