Merge pull request #175 from PSLmodels/scale-cps-weights

Scale down weights of CPS records

tests/expected_tax_expenditures

@@ -1,18 +1,14 @@
-YR,KIND,EST= 2023 paytax 1523.8
-YR,KIND,EST= 2023 iitax 2210.2
-YR,KIND,EST= 2023 ctc 129.2
-YR,KIND,EST= 2023 eitc 75.8
-YR,KIND,EST= 2023 social_security_partial_taxability 53.3
-YR,KIND,EST= 2023 niit -57.0
-YR,KIND,EST= 2023 cgqd_tax_preference 234.5
-YR,KIND,EST= 2023 qbid 57.0
-YR,KIND,EST= 2023 salt 20.7
-YR,KIND,EST= 2026 paytax 1767.4
-YR,KIND,EST= 2026 iitax 2785.9
-YR,KIND,EST= 2026 ctc 46.8
-YR,KIND,EST= 2026 eitc 84.8
-YR,KIND,EST= 2026 social_security_partial_taxability 72.2
-YR,KIND,EST= 2026 niit -55.1
-YR,KIND,EST= 2026 cgqd_tax_preference 232.4
+YR,KIND,EST= 2023 ctc 128.1
+YR,KIND,EST= 2023 eitc 75.4
+YR,KIND,EST= 2023 social_security_partial_taxability 49.5
+YR,KIND,EST= 2023 niit -56.9
+YR,KIND,EST= 2023 cgqd_tax_preference 234.1
+YR,KIND,EST= 2023 qbid 56.7
+YR,KIND,EST= 2023 salt 20.5
+YR,KIND,EST= 2026 ctc 46.4
+YR,KIND,EST= 2026 eitc 84.4
+YR,KIND,EST= 2026 social_security_partial_taxability 66.3
+YR,KIND,EST= 2026 niit -55.0
+YR,KIND,EST= 2026 cgqd_tax_preference 231.9
 YR,KIND,EST= 2026 qbid -0.0
-YR,KIND,EST= 2026 salt 162.3
+YR,KIND,EST= 2026 salt 161.3

tests/test_misc.py

@@ -20,5 +20,5 @@ def test_population(tmd_variables):
     people = tmd_variables.XTOT
     population = (weight * people).sum()
     assert (
-        abs(population / 1e6 / 334 - 1) < 0.1
-    ), "Population not within 10% of 334 million"
+        abs(population / 1e6 / 334.18 - 1) < 0.01
+    ), "Population not within 1% of 334.18 million"

tests/test_tax_expenditures.py

@@ -18,7 +18,7 @@ def test_tax_exp_diffs(
     tmd_weights_path,
     tmd_growfactors_path,
 ):
-    reltol = 0.003  # relative np.allclose tolerance
+    reltol = 0.01  # relative np.allclose tolerance
     _ = get_tax_expenditure_results(
         tmd_variables,
         2021,  # input variables data year

tests/test_tax_revenue.py

@@ -10,7 +10,7 @@
 LAST_CYR = 2026
 
 RELTOL_ITAX = 0.30
-RELTOL_PTAX = 0.06
+RELTOL_PTAX = 0.07
 
 DUMP = False  # True implies test always fails with complete output
 

tmd/datasets/tmd.py

@@ -1,12 +1,13 @@
+import numpy as np
 import pandas as pd
+from policyengine_us import Microsimulation
 from tmd.datasets.puf import PUF_2021, create_pe_puf_2021
 from tmd.datasets.cps import CPS_2021, create_cps_2021
 from tmd.datasets.taxcalc_dataset import create_tc_dataset
 from tmd.utils.trace import trace1
 from tmd.utils.taxcalc_utils import add_taxcalc_outputs
 from tmd.utils.reweight import reweight
 from tmd.storage import STORAGE_FOLDER
-from policyengine_us import Microsimulation
 
 
 def create_tmd_2021():
@@ -31,11 +32,18 @@ def create_tmd_2021():
 
     print("Adding Tax-Calculator outputs for 2021...")
     combined = add_taxcalc_outputs(combined, 2021, 2021)
-    combined["s006_original"] = combined.s006.values
+    # ... drop CPS records with positive 2021 income tax amount
+    idx = combined[((combined.data_source == 0) & (combined.iitax > 0))].index
+    combined.drop(idx, inplace=True)
+    # ... scale CPS records weight in order to get correct population count
+    cps_scale = 0.6248
+    scale = np.where(combined.data_source == 0, cps_scale, 1.0)
+    combined["s006"] *= scale
 
     trace1("B", combined)
 
     print("Reweighting...")
+    combined["s006_original"] = combined["s006"].values
     combined = reweight(combined, 2021)
 
     trace1("C", combined)

tmd/utils/taxcalc_utils.py

@@ -2,13 +2,12 @@
 This module provides utilities for working with Tax-Calculator.
 """
 
-import os
 import pathlib
 import yaml
 import numpy as np
 import pandas as pd
-from tmd.storage import STORAGE_FOLDER
 import taxcalc as tc
+from tmd.storage import STORAGE_FOLDER
 
 
 with open(STORAGE_FOLDER / "input" / "tc_variable_metadata.yaml") as f:
@@ -27,8 +26,7 @@ def get_tc_variable_description(variable: str) -> str:
     """
     if variable in taxcalc_variable_metadata.get("read", {}):
         return taxcalc_variable_metadata["read"][variable]["desc"]
-    elif variable in taxcalc_variable_metadata.get("calc", {}):
-        return taxcalc_variable_metadata["calc"][variable]["desc"]
+    return taxcalc_variable_metadata["calc"][variable]["desc"]
 
 
 def get_tc_is_input(variable: str) -> bool:
@@ -43,8 +41,7 @@ def get_tc_is_input(variable: str) -> bool:
     """
     if variable in taxcalc_variable_metadata.get("read", {}):
         return True
-    elif variable in taxcalc_variable_metadata.get("calc", {}):
-        return False
+    return False
 
 
 def add_taxcalc_outputs(
@@ -123,7 +120,6 @@ def get_tax_expenditure_results(
         growfactors=growfactors_file_path,
     )
     itax_baseline = (baseline.iitax * baseline.s006).sum() / 1e9
-    ptax_baseline = (baseline.payrolltax * baseline.s006).sum() / 1e9
 
     te_results = {}
     for reform_name, reform in te_reforms.items():
@@ -148,10 +144,6 @@ def get_tax_expenditure_results(
         open_mode = "a"
     year = simulation_year
     with open(taxexp_path, open_mode) as tefile:
-        res = f"YR,KIND,EST= {year} paytax {ptax_baseline:.1f}\n"
-        tefile.write(res)
-        res = f"YR,KIND,EST= {year} iitax {itax_baseline:.1f}\n"
-        tefile.write(res)
         for reform, estimate in te_results.items():
             res = f"YR,KIND,EST= {year} {reform} {estimate}\n"
             tefile.write(res)