Add -1s to 861 skipfooter map, update validation tests

src/pudl/package_data/eia861/file_map.csv

@@ -16,7 +16,7 @@ non_net_metering_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,Non_N
 operational_data_eia861,File1.xls,FILE1.xls,2001/file1.xls,2002/file1.xls,2003/file1.xls,2004/file1.xls,2005/file1.xls,file1.xls,2007/file1_2007.xls,2008/file1_2008.xls,2009/file1_2009.xls,file1_2010.xls,file1_2011.xls,operational_data_2012.xls,Operational_Data_2013.xlsx,Operational_Data_2014.xlsx,Operational_Data_2015.xlsx,Operational_Data_2016.xlsx,Operational_Data_2017.xlsx,Operational_Data_2018.xlsx,Operational_Data_2019.xlsx,Operational_Data_2020.xlsx,Operational_Data_2021.xlsx,Operational_Data_2022.xlsx,Operational_Data_2023.xlsx
 reliability_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,Reliability_2013.xlsx,Reliability_2014.xlsx,Reliability_2015.xlsx,Reliability_2016.xlsx,Reliability_2017.xlsx,Reliability_2018.xlsx,Reliability_2019.xlsx,Reliability_2020.xlsx,Reliability_2021.xlsx,Reliability_2022.xlsx,Reliability_2023.xlsx
 sales_eia861,FILE2.xls,FILE2.xls,2001/file2.xls,2002/file2.xls,2003/file2.xls,2004/file2.xls,2005/file2.xls,file2.xls,2007/file2_2007.xls,2008/file2_2008.xls,2009/file2_2009.xls,file2_2010.xls,file2_2011.xls,Sales_Ult_Cust_2012.xlsx,Sales_Ult_Cust_2013.xls,Sales_Ult_Cust_2014.xls,Sales_Ult_Cust_2015.xlsx,Sales_Ult_Cust_2016.xlsx,Sales_Ult_Cust_2017.xlsx,Sales_Ult_Cust_2018.xlsx,Sales_Ult_Cust_2019.xlsx,Sales_Ult_Cust_2020.xlsx,Sales_Ult_Cust_2021.xlsx,Sales_Ult_Cust_2022.xlsx,Sales_Ult_Cust_2023.xlsx
-sales_customer_sited_eia861,,,,,,,,,,,,,,Sales_Ult_Cust_CS_2012.xlsx,Sales_Ult_Cust_CS_2013.xls,Sales_Ult_Cust_CS_2014.xls,Sales_Ult_Cust_CS_2015.xlsx,Sales_Ult_Cust_CS_2016.xlsx,Sales_Ult_Cust_CS_2017.xlsx,Sales_Ult_Cust_2018_CS.xlsx,Sales_Ult_Cust_2019_CS.xlsx,Sales_Ult_Cust_2020_CS.xlsx,Sales_Ult_Cust_CS_2021.xlsx,Sales_Ult_Cust_CS_2022.xlsx,Sales_Ult_Cust_CS_2023.xlsx
+sales_customer_sited_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,Sales_Ult_Cust_CS_2012.xlsx,Sales_Ult_Cust_CS_2013.xls,Sales_Ult_Cust_CS_2014.xls,Sales_Ult_Cust_CS_2015.xlsx,Sales_Ult_Cust_CS_2016.xlsx,Sales_Ult_Cust_CS_2017.xlsx,Sales_Ult_Cust_2018_CS.xlsx,Sales_Ult_Cust_2019_CS.xlsx,Sales_Ult_Cust_2020_CS.xlsx,Sales_Ult_Cust_CS_2021.xlsx,Sales_Ult_Cust_CS_2022.xlsx,Sales_Ult_Cust_CS_2023.xlsx
 service_territory_eia861,FILE5.xls,FILE5.xls,2001/file4.xls,2002/file4.xls,2003/file4.xls,2004/file4.xls,2005/file4.xls,file4.xls,2007/file4_2007.xls,2008/file4_2008.xls,2009/file4_2009.xls,file4_2010.xls,file4_2011.xls,service_territory_2012.xls,Service_Territory_2013.xls,Service_Territory_2014.xls,Service_Territory_2015.xlsx,Service_Territory_2016.xlsx,Service_Territory_2017.xlsx,Service_Territory_2018.xlsx,Service_Territory_2019.xlsx,Service_Territory_2020.xlsx,Service_Territory_2021.xlsx,Service_Territory_2022.xlsx,Service_Territory_2023.xlsx
 short_form_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,short_form_2012.xls,Short_Form_2013.xls,Short_Form_2014.xls,Short_Form_2015.xlsx,Short_Form_2016.xlsx,Short_Form_2017.xlsx,Short_Form_2018.xlsx,-1,Short_Form_2020.xlsx,Short_Form_2021.xlsx,Short_Form_2022.xlsx,Short_Form_2023.xlsx
 utility_data_eia861,-1,-1,-1,-1,-1,-1,-1,-1,2007/file1a_2007.xls,2008/file1_a_2008.xls,2009/file1_a_2009.xls,file1a_2010.xls,file1a_2011.xls,utility_data_2012.xls,Utility_Data_2013.xls,Utility_Data_2014.xls,Utility_Data_2015.xlsx,Utility_Data_2016.xlsx,Utility_Data_2017.xlsx,Utility_Data_2018.xlsx,Utility_Data_2019.xlsx,Utility_Data_2020.xlsx,Utility_Data_2021.xlsx,Utility_Data_2022.xlsx,Utility_Data_2023.xlsx

src/pudl/package_data/eia861/skipfooter.csv

@@ -1,21 +1,21 @@
 year_index,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022,2023
-advanced_metering_infrastructure_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1
-balancing_authority_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-delivery_companies_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1
-demand_response_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-demand_side_management_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-1
-distributed_generation_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-1
-distribution_systems_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-dynamic_pricing_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-energy_efficiency_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1
-frame_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-green_pricing_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-1
-mergers_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-net_metering_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1
-non_net_metering_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1
+advanced_metering_infrastructure_eia861,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1
+balancing_authority_eia861,-1,-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+delivery_companies_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1
+demand_response_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0
+demand_side_management_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+distributed_generation_eia861,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1,-1,-1,-1
+distribution_systems_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0
+dynamic_pricing_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0
+energy_efficiency_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1
+frame_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0
+green_pricing_eia861,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+mergers_eia861,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+net_metering_eia861,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1
+non_net_metering_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1
 operational_data_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-reliability_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1
+reliability_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1
 sales_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1
 service_territory_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-short_form_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,1,1,1,1
-utility_data_eia861,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+short_form_eia861,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,1,1,1,1,1,-1,1,1,1,1
+utility_data_eia861,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0

test/validate/service_territory_test.py

@@ -16,9 +16,9 @@
     "df_name,expected_rows",
     [
         ("summarized_demand_ferc714", 3_924),
-        ("fipsified_respondents_ferc714", 156_392),
-        ("compiled_geometry_balancing_authority_eia861", 113_142),
-        ("compiled_geometry_utility_eia861", 256_949),
+        ("fipsified_respondents_ferc714", 165_131),
+        ("compiled_geometry_balancing_authority_eia861", 117_577),
+        ("compiled_geometry_utility_eia861", 268_665),
     ],
 )
 def test_minmax_rows(

test/validate/state_demand_test.py

@@ -12,7 +12,7 @@
 
 @pytest.mark.parametrize(
     "resource_id,expected_rows",
-    [("out_ferc714__hourly_estimated_state_demand", 7_599_842)],
+    [("out_ferc714__hourly_estimated_state_demand", 8_046_602)],
 )
 def test_minmax_rows(live_dbs: bool, resource_id: str, expected_rows: int):
     """Verify that output DataFrames don't have too many or too few rows.