diff --git a/src/pudl/package_data/eia861/file_map.csv b/src/pudl/package_data/eia861/file_map.csv index b2657f48fb..c970de642e 100644 --- a/src/pudl/package_data/eia861/file_map.csv +++ b/src/pudl/package_data/eia861/file_map.csv @@ -16,7 +16,7 @@ non_net_metering_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,Non_N operational_data_eia861,File1.xls,FILE1.xls,2001/file1.xls,2002/file1.xls,2003/file1.xls,2004/file1.xls,2005/file1.xls,file1.xls,2007/file1_2007.xls,2008/file1_2008.xls,2009/file1_2009.xls,file1_2010.xls,file1_2011.xls,operational_data_2012.xls,Operational_Data_2013.xlsx,Operational_Data_2014.xlsx,Operational_Data_2015.xlsx,Operational_Data_2016.xlsx,Operational_Data_2017.xlsx,Operational_Data_2018.xlsx,Operational_Data_2019.xlsx,Operational_Data_2020.xlsx,Operational_Data_2021.xlsx,Operational_Data_2022.xlsx,Operational_Data_2023.xlsx reliability_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,Reliability_2013.xlsx,Reliability_2014.xlsx,Reliability_2015.xlsx,Reliability_2016.xlsx,Reliability_2017.xlsx,Reliability_2018.xlsx,Reliability_2019.xlsx,Reliability_2020.xlsx,Reliability_2021.xlsx,Reliability_2022.xlsx,Reliability_2023.xlsx sales_eia861,FILE2.xls,FILE2.xls,2001/file2.xls,2002/file2.xls,2003/file2.xls,2004/file2.xls,2005/file2.xls,file2.xls,2007/file2_2007.xls,2008/file2_2008.xls,2009/file2_2009.xls,file2_2010.xls,file2_2011.xls,Sales_Ult_Cust_2012.xlsx,Sales_Ult_Cust_2013.xls,Sales_Ult_Cust_2014.xls,Sales_Ult_Cust_2015.xlsx,Sales_Ult_Cust_2016.xlsx,Sales_Ult_Cust_2017.xlsx,Sales_Ult_Cust_2018.xlsx,Sales_Ult_Cust_2019.xlsx,Sales_Ult_Cust_2020.xlsx,Sales_Ult_Cust_2021.xlsx,Sales_Ult_Cust_2022.xlsx,Sales_Ult_Cust_2023.xlsx -sales_customer_sited_eia861,,,,,,,,,,,,,,Sales_Ult_Cust_CS_2012.xlsx,Sales_Ult_Cust_CS_2013.xls,Sales_Ult_Cust_CS_2014.xls,Sales_Ult_Cust_CS_2015.xlsx,Sales_Ult_Cust_CS_2016.xlsx,Sales_Ult_Cust_CS_2017.xlsx,Sales_Ult_Cust_2018_CS.xlsx,Sales_Ult_Cust_2019_CS.xlsx,Sales_Ult_Cust_2020_CS.xlsx,Sales_Ult_Cust_CS_2021.xlsx,Sales_Ult_Cust_CS_2022.xlsx,Sales_Ult_Cust_CS_2023.xlsx +sales_customer_sited_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,Sales_Ult_Cust_CS_2012.xlsx,Sales_Ult_Cust_CS_2013.xls,Sales_Ult_Cust_CS_2014.xls,Sales_Ult_Cust_CS_2015.xlsx,Sales_Ult_Cust_CS_2016.xlsx,Sales_Ult_Cust_CS_2017.xlsx,Sales_Ult_Cust_2018_CS.xlsx,Sales_Ult_Cust_2019_CS.xlsx,Sales_Ult_Cust_2020_CS.xlsx,Sales_Ult_Cust_CS_2021.xlsx,Sales_Ult_Cust_CS_2022.xlsx,Sales_Ult_Cust_CS_2023.xlsx service_territory_eia861,FILE5.xls,FILE5.xls,2001/file4.xls,2002/file4.xls,2003/file4.xls,2004/file4.xls,2005/file4.xls,file4.xls,2007/file4_2007.xls,2008/file4_2008.xls,2009/file4_2009.xls,file4_2010.xls,file4_2011.xls,service_territory_2012.xls,Service_Territory_2013.xls,Service_Territory_2014.xls,Service_Territory_2015.xlsx,Service_Territory_2016.xlsx,Service_Territory_2017.xlsx,Service_Territory_2018.xlsx,Service_Territory_2019.xlsx,Service_Territory_2020.xlsx,Service_Territory_2021.xlsx,Service_Territory_2022.xlsx,Service_Territory_2023.xlsx short_form_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,short_form_2012.xls,Short_Form_2013.xls,Short_Form_2014.xls,Short_Form_2015.xlsx,Short_Form_2016.xlsx,Short_Form_2017.xlsx,Short_Form_2018.xlsx,-1,Short_Form_2020.xlsx,Short_Form_2021.xlsx,Short_Form_2022.xlsx,Short_Form_2023.xlsx utility_data_eia861,-1,-1,-1,-1,-1,-1,-1,-1,2007/file1a_2007.xls,2008/file1_a_2008.xls,2009/file1_a_2009.xls,file1a_2010.xls,file1a_2011.xls,utility_data_2012.xls,Utility_Data_2013.xls,Utility_Data_2014.xls,Utility_Data_2015.xlsx,Utility_Data_2016.xlsx,Utility_Data_2017.xlsx,Utility_Data_2018.xlsx,Utility_Data_2019.xlsx,Utility_Data_2020.xlsx,Utility_Data_2021.xlsx,Utility_Data_2022.xlsx,Utility_Data_2023.xlsx diff --git a/src/pudl/package_data/eia861/skipfooter.csv b/src/pudl/package_data/eia861/skipfooter.csv index 80c30f3be3..061e1ecc07 100644 --- a/src/pudl/package_data/eia861/skipfooter.csv +++ b/src/pudl/package_data/eia861/skipfooter.csv @@ -1,21 +1,21 @@ year_index,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022,2023 -advanced_metering_infrastructure_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1 -balancing_authority_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -delivery_companies_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1 -demand_response_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -demand_side_management_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-1 -distributed_generation_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-1 -distribution_systems_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -dynamic_pricing_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -energy_efficiency_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1 -frame_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -green_pricing_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-1 -mergers_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -net_metering_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1 -non_net_metering_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1 +advanced_metering_infrastructure_eia861,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1 +balancing_authority_eia861,-1,-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +delivery_companies_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1 +demand_response_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0 +demand_side_management_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 +distributed_generation_eia861,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1,-1,-1,-1 +distribution_systems_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0 +dynamic_pricing_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0 +energy_efficiency_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1 +frame_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0 +green_pricing_eia861,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 +mergers_eia861,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +net_metering_eia861,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1 +non_net_metering_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1 operational_data_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -reliability_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1 +reliability_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1 sales_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1 service_territory_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -short_form_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,1,1,1,1 -utility_data_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +short_form_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,1,1,1,1,1,-1,1,1,1,1 +utility_data_eia861,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 diff --git a/test/validate/service_territory_test.py b/test/validate/service_territory_test.py index a0f0d53446..f3b9ae514a 100644 --- a/test/validate/service_territory_test.py +++ b/test/validate/service_territory_test.py @@ -16,9 +16,9 @@ "df_name,expected_rows", [ ("summarized_demand_ferc714", 3_924), - ("fipsified_respondents_ferc714", 156_392), - ("compiled_geometry_balancing_authority_eia861", 113_142), - ("compiled_geometry_utility_eia861", 256_949), + ("fipsified_respondents_ferc714", 165_131), + ("compiled_geometry_balancing_authority_eia861", 117_577), + ("compiled_geometry_utility_eia861", 268_665), ], ) def test_minmax_rows( diff --git a/test/validate/state_demand_test.py b/test/validate/state_demand_test.py index d48cf4709d..c4ac4e1b4a 100644 --- a/test/validate/state_demand_test.py +++ b/test/validate/state_demand_test.py @@ -12,7 +12,7 @@ @pytest.mark.parametrize( "resource_id,expected_rows", - [("out_ferc714__hourly_estimated_state_demand", 7_599_842)], + [("out_ferc714__hourly_estimated_state_demand", 8_046_602)], ) def test_minmax_rows(live_dbs: bool, resource_id: str, expected_rows: int): """Verify that output DataFrames don't have too many or too few rows.