From ae83561c37100e15baf1c4add560186654daf491 Mon Sep 17 00:00:00 2001 From: Jesse Mortenson Date: Thu, 28 Mar 2024 12:48:05 -0500 Subject: [PATCH] Deployed 6c69fef with MkDocs version: 1.2.3 --- 404.html | 32 +- api-v2/changelog/index.html | 32 +- api-v2/examples/index.html | 32 +- api-v2/index.html | 32 +- api-v2/other/index.html | 32 +- api-v2/root-nodes/index.html | 32 +- api-v2/types/index.html | 32 +- api-v3/changelog/index.html | 32 +- api-v3/index.html | 32 +- code-of-conduct/index.html | 32 +- contributing/documentation/index.html | 32 +- contributing/index.html | 32 +- contributing/local-database/index.html | 32 +- contributing/openstates-org/index.html | 32 +- contributing/people/index.html | 32 +- contributing/scrapers/index.html | 32 +- contributing/state-specific/index.html | 32 +- contributing/testing-scrapers/index.html | 32 +- contributing/text-extraction/index.html | 32 +- .../writing-a-committee-scraper/index.html | 32 +- data/categorization/index.html | 32 +- data/index.html | 32 +- data/session-naming/index.html | 47 +- .../001-purpose-and-process/index.html | 1288 --------------- .../002-legal-citations/index.html | 1406 ----------------- .../003-manual-people-data-tools/index.html | 1299 --------------- .../004-committee-data/index.html | 1275 --------------- .../005-dedupe-key/index.html | 1164 -------------- .../006-new-people-offices/index.html | 1187 -------------- enhancement-proposals/007-events/index.html | 1250 --------------- .../008-active-sessions/index.html | 1153 -------------- enhancement-proposals/index.html | 938 ----------- enhancement-proposals/template/index.html | 1042 ------------ index.html | 32 +- openstates.org/scheduled-tasks/index.html | 32 +- search/search_index.json | 2 +- sitemap.xml | 98 +- sitemap.xml.gz | Bin 529 -> 409 bytes 38 files changed, 425 insertions(+), 12492 deletions(-) delete mode 100644 enhancement-proposals/001-purpose-and-process/index.html delete mode 100644 enhancement-proposals/002-legal-citations/index.html delete mode 100644 enhancement-proposals/003-manual-people-data-tools/index.html delete mode 100644 enhancement-proposals/004-committee-data/index.html delete mode 100644 enhancement-proposals/005-dedupe-key/index.html delete mode 100644 enhancement-proposals/006-new-people-offices/index.html delete mode 100644 enhancement-proposals/007-events/index.html delete mode 100644 enhancement-proposals/008-active-sessions/index.html delete mode 100644 enhancement-proposals/index.html delete mode 100644 enhancement-proposals/template/index.html diff --git a/404.html b/404.html index 6c699cd..dc1b18c 100644 --- a/404.html +++ b/404.html @@ -709,8 +709,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -723,8 +723,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -737,8 +737,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -751,8 +751,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -765,8 +765,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -779,8 +779,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -793,8 +793,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -807,8 +807,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/api-v2/changelog/index.html b/api-v2/changelog/index.html index ac94e0a..76d7c24 100644 --- a/api-v2/changelog/index.html +++ b/api-v2/changelog/index.html @@ -822,8 +822,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -836,8 +836,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -850,8 +850,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -864,8 +864,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -878,8 +878,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -892,8 +892,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -906,8 +906,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -920,8 +920,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/api-v2/examples/index.html b/api-v2/examples/index.html index 5bf28e1..35b2d6d 100644 --- a/api-v2/examples/index.html +++ b/api-v2/examples/index.html @@ -801,8 +801,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -815,8 +815,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -829,8 +829,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -843,8 +843,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -857,8 +857,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -871,8 +871,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -885,8 +885,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -899,8 +899,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/api-v2/index.html b/api-v2/index.html index 293dc02..5a0c791 100644 --- a/api-v2/index.html +++ b/api-v2/index.html @@ -766,8 +766,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -780,8 +780,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -794,8 +794,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -808,8 +808,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -822,8 +822,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -836,8 +836,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -850,8 +850,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -864,8 +864,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/api-v2/other/index.html b/api-v2/other/index.html index e4c0775..5d29b6e 100644 --- a/api-v2/other/index.html +++ b/api-v2/other/index.html @@ -814,8 +814,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -828,8 +828,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -842,8 +842,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -856,8 +856,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -870,8 +870,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -884,8 +884,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -898,8 +898,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -912,8 +912,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/api-v2/root-nodes/index.html b/api-v2/root-nodes/index.html index 00b11f0..d43b3c1 100644 --- a/api-v2/root-nodes/index.html +++ b/api-v2/root-nodes/index.html @@ -879,8 +879,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -893,8 +893,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -907,8 +907,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -921,8 +921,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -935,8 +935,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -949,8 +949,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -963,8 +963,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -977,8 +977,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/api-v2/types/index.html b/api-v2/types/index.html index 88dd273..5e1ee75 100644 --- a/api-v2/types/index.html +++ b/api-v2/types/index.html @@ -972,8 +972,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -986,8 +986,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -1000,8 +1000,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -1014,8 +1014,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -1028,8 +1028,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -1042,8 +1042,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -1056,8 +1056,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -1070,8 +1070,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/api-v3/changelog/index.html b/api-v3/changelog/index.html index fb2ce99..3d27fb9 100644 --- a/api-v3/changelog/index.html +++ b/api-v3/changelog/index.html @@ -878,8 +878,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -892,8 +892,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -906,8 +906,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -920,8 +920,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -934,8 +934,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -948,8 +948,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -962,8 +962,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -976,8 +976,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/api-v3/index.html b/api-v3/index.html index 43ea073..66a4e15 100644 --- a/api-v3/index.html +++ b/api-v3/index.html @@ -773,8 +773,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -787,8 +787,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -801,8 +801,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -815,8 +815,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -829,8 +829,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -843,8 +843,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -857,8 +857,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -871,8 +871,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/code-of-conduct/index.html b/code-of-conduct/index.html index ecf8d51..f7ad27e 100644 --- a/code-of-conduct/index.html +++ b/code-of-conduct/index.html @@ -833,8 +833,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -847,8 +847,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -861,8 +861,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -875,8 +875,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -889,8 +889,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -903,8 +903,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -917,8 +917,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -931,8 +931,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/contributing/documentation/index.html b/contributing/documentation/index.html index 17cde37..4a949c2 100644 --- a/contributing/documentation/index.html +++ b/contributing/documentation/index.html @@ -766,8 +766,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -780,8 +780,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -794,8 +794,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -808,8 +808,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -822,8 +822,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -836,8 +836,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -850,8 +850,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -864,8 +864,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/contributing/index.html b/contributing/index.html index 057697a..2d6bef2 100644 --- a/contributing/index.html +++ b/contributing/index.html @@ -800,8 +800,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -814,8 +814,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -828,8 +828,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -842,8 +842,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -856,8 +856,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -870,8 +870,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -884,8 +884,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -898,8 +898,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/contributing/local-database/index.html b/contributing/local-database/index.html index bdb9004..f43bac4 100644 --- a/contributing/local-database/index.html +++ b/contributing/local-database/index.html @@ -807,8 +807,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -821,8 +821,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -835,8 +835,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -849,8 +849,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -863,8 +863,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -877,8 +877,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -891,8 +891,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -905,8 +905,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/contributing/openstates-org/index.html b/contributing/openstates-org/index.html index 52c780c..31a95f3 100644 --- a/contributing/openstates-org/index.html +++ b/contributing/openstates-org/index.html @@ -794,8 +794,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -808,8 +808,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -822,8 +822,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -836,8 +836,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -850,8 +850,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -864,8 +864,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -878,8 +878,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -892,8 +892,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/contributing/people/index.html b/contributing/people/index.html index ab10024..9bdcc45 100644 --- a/contributing/people/index.html +++ b/contributing/people/index.html @@ -800,8 +800,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -814,8 +814,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -828,8 +828,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -842,8 +842,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -856,8 +856,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -870,8 +870,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -884,8 +884,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -898,8 +898,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/contributing/scrapers/index.html b/contributing/scrapers/index.html index 453ec3b..0ba4a89 100644 --- a/contributing/scrapers/index.html +++ b/contributing/scrapers/index.html @@ -787,8 +787,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -801,8 +801,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -815,8 +815,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -829,8 +829,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -843,8 +843,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -857,8 +857,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -871,8 +871,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -885,8 +885,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/contributing/state-specific/index.html b/contributing/state-specific/index.html index 8ece39f..6926bc1 100644 --- a/contributing/state-specific/index.html +++ b/contributing/state-specific/index.html @@ -800,8 +800,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -814,8 +814,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -828,8 +828,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -842,8 +842,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -856,8 +856,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -870,8 +870,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -884,8 +884,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -898,8 +898,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/contributing/testing-scrapers/index.html b/contributing/testing-scrapers/index.html index 7dbfc7b..3c64c09 100644 --- a/contributing/testing-scrapers/index.html +++ b/contributing/testing-scrapers/index.html @@ -800,8 +800,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -814,8 +814,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -828,8 +828,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -842,8 +842,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -856,8 +856,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -870,8 +870,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -884,8 +884,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -898,8 +898,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/contributing/text-extraction/index.html b/contributing/text-extraction/index.html index e78bd5f..0126d5b 100644 --- a/contributing/text-extraction/index.html +++ b/contributing/text-extraction/index.html @@ -716,8 +716,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -730,8 +730,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -744,8 +744,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -758,8 +758,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -772,8 +772,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -786,8 +786,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -800,8 +800,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -814,8 +814,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/contributing/writing-a-committee-scraper/index.html b/contributing/writing-a-committee-scraper/index.html index 8fac459..74227f1 100644 --- a/contributing/writing-a-committee-scraper/index.html +++ b/contributing/writing-a-committee-scraper/index.html @@ -800,8 +800,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -814,8 +814,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -828,8 +828,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -842,8 +842,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -856,8 +856,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -870,8 +870,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -884,8 +884,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -898,8 +898,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/data/categorization/index.html b/data/categorization/index.html index 06717f5..5251468 100644 --- a/data/categorization/index.html +++ b/data/categorization/index.html @@ -773,8 +773,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -787,8 +787,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -801,8 +801,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -815,8 +815,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -829,8 +829,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -843,8 +843,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -857,8 +857,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -871,8 +871,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/data/index.html b/data/index.html index 4885494..0c10cae 100644 --- a/data/index.html +++ b/data/index.html @@ -759,8 +759,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -773,8 +773,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -787,8 +787,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -801,8 +801,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -815,8 +815,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -829,8 +829,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -843,8 +843,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -857,8 +857,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/data/session-naming/index.html b/data/session-naming/index.html index 032a1ef..900d3b9 100644 --- a/data/session-naming/index.html +++ b/data/session-naming/index.html @@ -793,8 +793,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -807,8 +807,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -821,8 +821,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -835,8 +835,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -849,8 +849,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -863,8 +863,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -877,8 +877,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -891,8 +891,8 @@
  • - - OSEP #8: Active Sessions + + None
    • @@ -1094,21 +1094,6 @@

    Years& - - -
    diff --git a/enhancement-proposals/001-purpose-and-process/index.html b/enhancement-proposals/001-purpose-and-process/index.html deleted file mode 100644 index 5d424ca..0000000 --- a/enhancement-proposals/001-purpose-and-process/index.html +++ /dev/null @@ -1,1288 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - OSEP #1: Purpose & Process - Open States - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - - - - Skip to content - - -
    -
    - -
    - - - - -
    - - -
    - -
    - - - - - - -
    -
    - - - -
    -
    -
    - - - - -
    -
    -
    - - - -
    -
    -
    - - - -
    -
    -
    - - -
    -
    - - - - - - - -

    OSEP #1: Purpose & Process

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Author(s)@jamesturk
    Implementer(s)@jamesturk
    StatusFinal
    Draft PR(s)n/a
    Approval PR(s)https://github.com/openstates/enhancement-proposals/pull/1
    Created2021-02-11
    Updated2021-08-17
    -
    -

    Abstract

    -

    Provide a formal mechanism by which major changes to Open States can be agreed upon & made.

    -

    Specification

    -

    Format

    -

    A proposal should have this structure:

    -
      -
    1. Short title (e.g. "Adding Effective Dates"), which is also in the filename (e.g. 007-adding-effective-dates.md).
      2. -
    2. Metadata: listed at the top of the file, to include:
        -
      • Author(s), Implementer(s) - these can be multiple project members, implementer can be blank initially.
        • -
      • Created, Updated: Updated should be updated when substantial changes are adopted.
        • -
      • Status: Draft, Accepted, Rejected, Withdrawn, Final, Superseded
        • -
      • Issue: To better track the current status of various proposals, all proposals must have a linked issue with the appropriate status tag.
        • -
      • Draft PR: Link to Draft PR discussion.
        • -
      • Approval PR: Link to Approval PR discussion.
        • -
      -
      3. -
    3. Abstract - Concise summary of the proposal.
      4. -
    4. Specification - Detailed description of how the proposal should work.
      5. -
    5. Rationale - Explanation of why this would be a good idea. The rationale should discuss considered alternatives and why they were rejected.
      6. -
    6. Drawbacks - A discussion of backwards compatibility, maintenance cost, etc. (Can be omitted or very simple in drafts.)
      7. -
    7. Implementation Plan - An estimate of what it would take to implement this. (Can be omitted or very simple in drafts.)
      8. -
    8. Copyright - Each proposal must end with a CC0 dedication.
      9. -
    -

    Issue Status

    -

    Process

    -

    Drafting A Proposal

    -

    Submitting a proposal will take a little bit more planning than just opening a GitHub issue, -but you can start there by opening a "pre-draft" issue if desired.

    -

    Issues on this repository can be used as a form of proto-proposal in the early planning phase. -No specific format is required, but these can be useful to gauge interest in a formal proposal if desired.

    -

    Small non-breaking changes and minor site & API enhancements will continue to be filed as typical issues in -openstates/issues but larger feature requests and similar will be moved to this repository with a note explaining this process.

    -

    When ready, draft proposals begin as PRs against this repository. -Once a proposal is deemed on-topic and in the above format, it will be accepted as a draft proposal.

    -

    Approval Phase

    -

    Once accepted as a draft, a full discussion will ensue. -This repository should serve as the system of record for this discussion. -(i.e. If a conversation requires a Slack conversation or similar, the results of that conversation should be attached to the relevant issue/PR)

    -

    If fields like implementer, drawbacks, and implementation plan were not specified in the draft, they must be determined before a proposal can be accepted.

    -

    After any requisite back & forth, the active core team will submit their decision. -All active core developers can review, with a goal of consensus, but in cases of indecision the project lead will make the final decision. -The status will then be adjusted to Accepted, Rejected, or Withdrawn.

    -

    Implementation Phase

    -

    Once accepted, work can begin as needed. -The proposal can & will likely be edited during this time. -Once the work is reviewed and accepted, the proposal's status will be updated to Final.

    -

    Rationale

    -

    Both over the years, and right now in GitHub issues & discussions there are a lot of proposals to improve the project in various ways.

    -

    These are typically handled via a combination of GitHub issues, Slack conversations, and individual whims. By introducing a formal but lightweight Open States Enhancement Proposal (OSEP) process, I am hoping to achieve three outcomes:

    -
      -
    1. Structure the conversation around improvements so that new contributors and core members alike can understand the scope of a proposed change.
      2. -
    2. Provide a means by which consensus can be gathered publicly, in a timely manner to reduce stalled decisions so that fewer issues are left in the "yeah.. we really should figure out how to X" state. Available development time will still be an obstacle at times, but at least when time is available it will be clear what can be worked on.
      3. -
    3. Keep a record of failed proposals so we retain institutional memory why some things aren't done. This can reduce the same conversation happening when new contributors arrive, and also let us revisit old decisions to see if the context around old decisions still holds or if there's need to re-evaluate.
      4. -
    -

    History

    -

    There used to be a process surrounding OCDEPs by which the core schema/etc. could be modified, -but that process was cumbersome and slow and wasn't reflective of the needs of Open States as the spec derived there was aiming at a common denominator between various team's needs.

    -

    Most other improvements have come after either lengthy conversation or on a whim, typically James'. -(Both of these come with their share of issues, hence the proposal.)

    -

    Alternatives could be to continue with the status quo or just using GitHub issues ad hoc. -Both feel inferior to having a documented process around this.

    -

    In August 2021, this proposal was updated to spell out the use of GitHub issues to track the current status of proposals.

    -

    Drawbacks

    -

    n/a

    -

    Implementation Plan

    -
      -
    • If you're reading this, the repository is already created.
      • -
    • Documentation and openstates/issues repo will need to be updated to point people to this process.
      • -
    - -

    This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.

    - - -
    -
    -
    - - - - Back to top - - -
    - - - -
    -
    -
    -
    - - - - - - - - \ No newline at end of file diff --git a/enhancement-proposals/002-legal-citations/index.html b/enhancement-proposals/002-legal-citations/index.html deleted file mode 100644 index e9526ed..0000000 --- a/enhancement-proposals/002-legal-citations/index.html +++ /dev/null @@ -1,1406 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - OSEP #2: Legal Citations - Open States - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - - - - Skip to content - - -
    -
    - -
    - - - - -
    - - -
    - -
    - - - - - - -
    -
    - - - -
    -
    -
    - - - - -
    -
    -
    - - - -
    -
    -
    - - - -
    -
    -
    - - -
    -
    - - - - - - - -

    OSEP #2: Legal Citations

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Author(s)Tim Showers
    Implementer(s)Tim Showers
    StatusAccepted
    Issuehttps://github.com/openstates/enhancement-proposals/issues/33
    Draft PR(s)https://github.com/openstates/enhancement-proposals/pull/9/
    Approval PR(s)https://github.com/openstates/enhancement-proposals/pull/TBD
    Created2020-02-17
    UpdatedTODO
    -
    -

    Abstract

    -

    A number of jurisdictions offer metadata to link bills to the sections of the legal code that they will alter, or to the chaptered laws. We should modify the bills model to allow scraping this data in a structured way.

    -

    Note: There are many different legal structures that can be altered by bills. In this document, "legal code" will be used as shorthand for "collection of laws or rules modified by a bill" such as statutes, codes, slip laws, public laws, register entries, constitutions, etc.

    -

    Specification

    -

    Add an underlying data structure for one or more legal citations to the bill model.

    -

    These will be accessed via a new method on the bill object:

    -
      -
    1. add_citation()
      2. -
    - -

    Structure:

    -

    A list of 0+ Citation dicts

    -
      -
    • publication - string - The affected publication. e.g. "Minnesota Statutes", "California Public Utilities Code", "DC Register", "Constitution of Nevada". Note that these cover a wide variety of different types of law and rule making.
      • -
    • citation - string - The reference to the (sub)section of the publication. Formats and abbreviations vary widely.
      • -
    • type - enum [proposed, chapter, final, other] - whether the citation is a part of a pending bill, a chapter law, or a final affected section of some publication after the bill as been made into law. The list of proposed citations may not match the final list due to changes between bill versions. See the Chapter Citations section of rationale below for further discussion of chapter laws. other would cover corner cases such as a constitutional amendment that was passed by the legislature, but required a referendum as well.
      • -
    • effective - optional datetimeoptional - effective date
      • -
    • expiration - optional datetimeoptional - expiration date
      • -
    • url - optional string - Link to the URL of the affected code
      • -
    -

    Examples

    -

    2019 WY HB 4 -Chaptered to CH0024 of 2019, Effective 7/1/2019

    -
    bill.add_citation(
-    "Wyoming Chapter Laws of 2019",
-    "CH0024",
-    type="chapter",
-    effective=datetime(2019,07,01)
-)
-
    -

    2020 MN HF 4285 -Chaptered to "Chapter 89", modified a number of Statutes, effective 08/01/20

    -
    bill.add_citation(
-    "Minnesota Session Laws, 2020",
-    "Chapter 89",
-    type="chapter",
-    effective=datetime(2020, 08, 01),
-    url="https://www.revisor.mn.gov/laws/2020/0/Session+Law/Chapter/89/",
-)
-
-bill.add_citation(
-    "Minnesota Statutes 2018",
-    "Chapter 27, Section 27",
-    type="final",
-    effective=datetime(2020, 08, 01)
-    url="https://www.revisor.mn.gov/laws/2020/0/Session+Law/Chapter/89/"
-)
-
-bill.add_citation(
-    "Minnesota Statutes 2018",
-    "Chapter 27, Section 13",
-    type="final",
-    effective=datetime(2020, 08, 01)
-    url="https://www.revisor.mn.gov/laws/2020/0/Session+Law/Chapter/89/"
-)
-
-bill.add_citation(
-    "Minnesota Statutes 2018",
-    "Chapter 27, Section 17",
-    type="final",
-    effective=datetime(2020, 08, 01)
-    url="https://www.revisor.mn.gov/laws/2020/0/Session+Law/Chapter/89/"
-)
-
    -

    23rd Council DC 997 which results in a DC register entry in vol 67.

    -
    bill.add_citation(
-    "DC Register",
-    "Vol 67 and Page 14429",
-    type="final",
-    expires=datetime(2021,03,06)
-)
-
    -

    MO 2020 HJR 104 a (failed) constitutional amendment.

    -
    bill.add_citation(
-    "Constitution of Missouri",
-    "Article X Section 6",
-    type="proposed",
-)
-
    -

    Rationale

    -

    We want to be able to catch this both proactively -- "Tell me if a bill is introduced that would alter the California corporations code", and historically -- "Who sponsored the bill that made (X) a misdemeanor instead of a felony?".

    -

    This also allows us to provide interesting session overview data -- "What laws were changed as a result of the 2019 session?", "What legislator introduced the most changes?".

    -

    Chapter Citations:

    -

    Many jurisdictions keep a running list of all the bills that have become law in a session, -often with effective dates and redlines to aid legal researchers. These are generally known as 'chaptered laws', and may or may not also link to the state codes. Sometimes these are a holding area for laws to folded into state code(s) at a later date for something like an annual printing, but not always.

    -

    Some states (MN) chapter things like budgets that aren't statutory, so there's a chapter law without a corresponding change to a legal code.

    -

    Note that the "Chaptered Laws" of a given session, and a legal code that might have structural elements called "chapters" aren't necessarily related. So 2019/CH24 may not modify chapter 24 of some title of a legal code.

    -

    Example chapter list from MN

    -

    Granularity of data

    -

    A given citation may be provided at multiple levels of granularity; for example:

    -
    "Minnesota Statutes 2018, section 31"
-
    -

    vs

    -
    "Minnesota Statutes 2018, section 31A.10",
-"Minnesota Statutes 2018, section 31A.14",
-"Minnesota Statutes 2018, section 31B.10-12"
-
    -

    There's often a comparatively simple first-pass approach that will yield the first result, while the second result requires a full-featured legal citation parser. The second approach also generally requires downloading and processing bill texts, which we prefer to keep seperate from the main bill scrapers for operational reasons.

    -

    Here we leave the granularity up the individual scraper and contributor, allowing for maximum flexibility at the cost of some uniformity. We want to collect the 'low hanging fruit' of references that are on pages we already routinely scrape or could easily collect without exploding the volume of scraper requests.

    -

    If a future contributor wants to add a more fully featured legal parser to one or more scrapers, we would need to balance code complexity and scrape time against the value of the collected data. This could potentially be integrated via a new module, or a processing pipeline step that lived seperately from the existing scrapers.

    -

    Further discussion of this issue can be found the initial draft's pull request.

    -

    Formatting as open-text

    -

    In line with the differing formats provided by jurisdictions, consumers have differing opinions on the most granular citations that are appropriate, and how they should be formatted.

    -

    A structured approach was considered, for example:

    -
    {'chapter': '12', 'section': '23A', 'subsection': '3', 'paragraph': '1'}
-
    -

    One consumer maybe interested in the specific paragraph for redline purposes, while another may want to point to the subsection or even section for providing context around the change.

    -

    The large number of possible formats, and inconsistent naming ('title' lives at different levels across jursidictions, for example) makes writing parsers and validators for this impractical, particularly if end consumers are likely to re-parse this data anyway.

    -

    Requiring 'parsed' citations also significantly raises the barrier to entry to adding this data to a scraper, and likely increases scraper failures as new citation formats are encountered.

    -

    Given these drawbacks, an unstructured string is a more effective format here.

    -

    See the Formatting section in drawbacks for further details.

    -

    Drawbacks

    -
      -
    1. Data Availability -- The availability of this data is limited. See Note[1] below for a non-exhaustive survey.
      2. -
    2. Formatting - State legal citation formats are extremely varied. See Note[2] for some examples. Rather than try to standardize them, I propose we just take the states citations as is and leave followup steps to the consumer.
      3. -
    3. Codes/Laws are confusing. States handle rulemaking in a variety of formats -- Constitutions, Statutes, Codes, Administrative Registers, Administrative Rulemaking, etc. -Many states also split their lawmaking into multiple publications, e.g. "criminal code", "corporations code", "environmental code", etc. Rather than try to handle that extreme complexity for a few jurisdictions, plaintext citations have us punt it to the end consumer.
      4. -
    4. Proposed legal citations don't seem to be available as structured data in any states right now so this would require regexing bill text which has historically not been a thing we've done in scrapers. I think it's still better to have the ability in the model.
      5. -
    -

    Backwards Compatibility Issues: None, except for one-time a mass 'updated at' change in the affected states as the scrapers are updated.

    -

    Implementation Plan

    -

    Tim adds the model to openstates-core, with some help from James on questions.

    -

    James updates the poetry requirements and rolls new images, and does database migrations.

    -

    Scraper writers add the functionality to states as desired. Tim would take the first shot at the known states.

    -

    Notes

    -
      -
    1. States that offer chapters post hoc -- DC, GA, US, WY, MN
      2. -
    2. Projects that try to parse legal citations: law-identifier, citation-regexes
      3. -
    - -

    This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.

    - - -
    -
    -
    - - - - Back to top - - -
    - - - -
    -
    -
    -
    - - - - - - - - \ No newline at end of file diff --git a/enhancement-proposals/003-manual-people-data-tools/index.html b/enhancement-proposals/003-manual-people-data-tools/index.html deleted file mode 100644 index d13cd4c..0000000 --- a/enhancement-proposals/003-manual-people-data-tools/index.html +++ /dev/null @@ -1,1299 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - OSEP #3: Manual People Data Tools - Open States - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - - - - Skip to content - - -
    -
    - -
    - - - - -
    - - -
    - -
    - - - - - - -
    -
    - - - -
    -
    -
    - - - - -
    -
    -
    - - - -
    -
    -
    - - - -
    -
    -
    - - -
    -
    - - - - - - - -

    OSEP #3: Manual People Data Tools

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Author(s)James Turk
    Implementer(s)James Turk
    StatusAccepted
    Issuehttps://github.com/openstates/enhancement-proposals/issues/32
    Draft PR(s)https://github.com/openstates/enhancement-proposals/pull/14
    Approval PR(s)https://github.com/openstates/enhancement-proposals/pull/TBD
    Created2021-02-23
    Updated2021-03-01
    -
    -

    Abstract

    -

    Introduce a public means by which we can improve and enhance legislator data, one that is available to less technical volunteers.

    -

    Specification

    -

    I am proposing we build a public tool to handle common corrections to legislator data. This tool would be used primarily by less technical volunteers that are not comfortable with the clone-edit YAML-PR workflow that currently exists.

    -

    The tool is aimed at four types of issues in particular:

    -
      -
    • adding details such as phone numbers, social media accounts, etc. to an individual or in bulk
      • -
    • retiring legislators that have left office
      • -
    • adding new one-off legislators that won special elections or appointments
      • -
    • adding sponsor/vote aliases to legislators
      • -
    -

    It's important to note that not all issues will be considered. For example: -- an election has occurred and we need a full update -- a systemic issue has resulted in us grabbing incorrect addresses, swapping voice/fax, or similar

    -

    These will continue to be best handled by re-scraping, and will be considered out of scope for this proposal.

    -

    What I am proposing is a publicly available web app that provides the following:

    -

    Use Case 1: Bulk Editing & Review

    -

    This will be the primary view into the new app. It will present the user with a table like:

    - - - - - - - - - - - - - - - - - -
    namedistrictchamber
    [Retire] [Edit]
    -

    There will be form controls to add editable columns including District & Capitol Voice, Fax, Address, Twitter, etc.

    -

    Ideally the tool could provide a summary table for a sense of coverage for important fields as well, e.g. (23% of legislators are missing phone number).

    -

    If a user wishes to perform a bulk edit, they will add the appropriate fields using the controls, and make the appropriate edits like they would in a spreadsheet.

    -

    At the bottom of this page, there will be a form that will ask them for a short description of their changes, including a link to sources they used.

    -

    Pressing this will save a ChangeSet to the local database.

    -

    Each row also has a retire & edit link.

    -

    Use Case 2: Retirement

    -

    Another option available, is to retire a given individual.

    -

    Clicking the retire link will prompt for the necessary details (retirement date, source, death?) and clicking save will create a simple ChangeSet indicating the retirement.

    -

    Use Case 3: New Legislators (and editing a full legislator)

    -

    In addition to the legislator table, there will also be individual edit pages. If a [New Legislator] button is pressed, the user will be prompted for the initial non-editable details such as Name, District, Chamber.

    -

    Pressing [Save] will create a ChangeSet with just this legislator's information.

    -

    (As noted above, this is only intended to be used for 1-2 legislators that have been appointed, a full session turnover should be processed by scraping.)

    -

    Use Case 4: Adding Sponsor/Vote Aliases

    -

    A separate view will be made available for the explicit purpose of showing unmatched sponsor & vote data. -This view will allow contributors to select the matching legislator (or designate appropriate unmatched statuses as necessary).

    -

    Pressing [Save] will create a ChangeSet that adds the sponsor/vote aliases to the appropriate legislators.

    -

    Saving ChangeSets

    -

    Each of the above actions saves data representing its delta to the existing legislator data. These will be stored in a local database, and converted to a PR against the openstates/people repository.

    -

    As the tool's UI allows, it may be possible to combine multiple ChangeSets into a single PR.

    -

    The PR will: -- provide ChangeSet represented in minimally-edited YAML -- to avoid dealing with GitHub credentials, the PR will likely be by openstates-bot or similar. -- the Open States user's username (planning to use existing OpenStates.org authentication most likely for simplicity) -- might want a feature to @ mention their GitHub account

    -

    Rationale

    -

    After a full audit of Open States' legislator data, and reviewing the recently filed legislator issues, it seems like there are 3 types of issues that are common. I think with a bit of work we could come up with a tool that allows less technical volunteers to handle all of these. I think it will be worthwhile as we commit to keeping people data as up-to-date as possible.

    -

    Other options considered included: -- a locally-runnable app that worked directly on the YAML. - - (This likely wouldn't really reduce the barrier to entry and would still require users to have basic GitHub skills.) -- an AirTable app, as started in the openstates/people#383 PR. - - (This still requires someone running the script locally, then handing the airtable over to PRs, also there are things that are likely easier with a simple CRUD app instead of trying to bend AirTable to the schema here.) -- a web YAML editor - - (This doesn't really allow retirements, bulk editing, etc. so doesn't feel like as much of a win.)

    -

    Drawbacks

    -

    There's of course added complexity in building and supporting a new tool. I am hoping that the scope detailed here will be possible to implement a working prototype in ~2 weeks and make continued revisions as the first batch of users work with it.

    -

    This tool will not be able to do everything, so YAML will still be required for certain tasks. While this could be seen as a drawback, this limitation is by design, since the YAML exists there is no need for the tool to make it possible to cover all the strange edge cases that exist in needing to change the data. One example is that for now the tool will not provide a way to change a legislator's district or party as those changes are rare and a bit more complex in the YAML than others. (It'd be possible to add those features later though if they prove necessary, but the features described above aim at 80% of the edits we want to do.)

    -

    Implementation Plan

    -

    I'd plan to work on this myself, possibly with the assistance of one other engineer if resources were available. If prioritized, I think a working version could be produced in the next couple of weeks, leaving room for UI and other functionality improvements down the line.

    - -

    This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.

    - - -
    -
    -
    - - - - Back to top - - -
    - - - -
    -
    -
    -
    - - - - - - - - \ No newline at end of file diff --git a/enhancement-proposals/004-committee-data/index.html b/enhancement-proposals/004-committee-data/index.html deleted file mode 100644 index ba8b919..0000000 --- a/enhancement-proposals/004-committee-data/index.html +++ /dev/null @@ -1,1275 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - OSEP #4: Committee Data - Open States - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - - - - Skip to content - - -
    -
    - -
    - - - - -
    - - -
    - -
    - - - - - - -
    -
    - - - -
    -
    -
    - - - - -
    -
    -
    - - - -
    -
    -
    - - - -
    -
    -
    - - -
    -
    - - - - - - - -

    OSEP #4: Committee Data

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Author(s)@jamesturk
    Implementer(s)@jamesturk
    StatusFinal
    Draft PR(s)https://github.com/openstates/enhancement-proposals/pull/18
    Approval PR(s)https://github.com/openstates/enhancement-proposals/pull/TBD
    Created2021-03-31
    Updated2021-08-03
    -
    -

    Abstract

    -

    Open States provided committee membership data from 2011-2018, but ceased to when it was no longer viable given the project's funding and staffing situations. This proposal would restore committee membership data via a hybrid scraped/manual process with a focus on maintaining up-to-date current membership of committees.

    -

    Specification

    -

    Data Model Changes

    -

    The data model Open States inherited from OCD/Popolo is still a part of the core OS models and will be renewed to support this work.

    -

    As part of this proposal, several small changes will be made to make working with this data slightly easier:

    -
      -
    • OrganizationLink and OrganizationSource will be migrated to inline JSON fields on the Organization model.
      • -
    • Organization will also gain an other_names JSON field.
      • -
    -

    These changes can be implemented in a way that does not impact any of Open States' public APIs.

    -

    Scraping

    -

    Right now we have 45 committee scrapers in openstates-scrapers that have not been run in years. A quick check of 10 of them showed that the vast majority do not run as-is.

    -

    That fact combined with the limitations in the old pupa import path for Organizations points towards us adopting a different method that allows easier augmentation of scraped data with manual. To this end, we will port/rewrite committee scrapers to a new format similar to the direction that new people scrapers are heading.

    -

    The proposed path looks like this:

    -
      -
    • The committee portions of openstates-people will be restored:
        -
      • each jurisdiction will again have a committees directory
        • -
      • lint_yaml.py will be renamed lint_people.py
        • -
      • a new lint_committees.py will be created
        • -
      • the committee portion of to_database.py will be restored
        • -
      • the committee portion of to_csv.py will be restored
        • -
      • documentation and CI processes will be updated to include committee data
        • -
      -
      • -
    • Committee scrapers will be added to openstates-people and be written in a way that yields compatible YAML.
        -
      • Existing committee scrapers will be kept in openstates-scrapers until that state has been ported to openstates-people.
        • -
      • new committee scrapers will be run regularly with changes to the YAML submitted as PRs (manually at first, but likely the subject of future automation)
        • -
      -
      • -
    -

    Manual Data

    -

    The existing organization schema will be updated to require either ID or name, if ID is present it will be used for linking, whereas name will serve as a placeholder for unlinked IDs.

    -

    Initially we'll be focused on manual data that is not impacted by the scrapers, such as alternate names and links. As such the tool that merges scraper output with the YAML will be focused on replacing member names with those found by the most recent scrape.

    -

    This also makes it possible to add committee data by alternate processes, particularly useful in cases where scraping a given chamber or state's committees is harder than maintaining a manual roster.

    -

    We will not focus on historical data, intending the committee membership in the file to reflect the membership at a given time. It is possible that in the future we could use git history to try to construct historical records from this.

    -

    This manual data process also leaves the door open for future iterations where some data could be marked in the YAML as to-be-kept so that the membership of a committee could be partially manually curated, but that will be kept to future enhancements.

    -

    API v2

    -

    API v2 will be updated in a backwards-compatible way, such that the existing organizations node can be used to fetch committee data again.

    -

    API v3

    -

    Committees search & detail endpoints will be added to API v3 in the same manner as existing Jurisdictions/People endpoints.

    -

    Rationale

    -

    Restoration of committees has been one of the most requested data sets since their deprecation was deemed necessary. Now that there are resources available to pursue this again it is a high priority addition to our existing data.

    -

    As for the approach itself, it mainly draws on the experience of the earlier iterations of committee data. Drawing upon the expertise gained in the first iteration, we are not attempting a fully automated process as augmentation & deduplication became quite difficult under that approach. The manual approach on the other hand quickly went stale and was not updated in any meaningful way after the initial release. By introducing a new process that combines both of these, it is believed we will trade some initial complexity for an approach that can keep up with the complexities of dealing with committee data.

    -

    Drawbacks

    -
      -
    • This approach is more complicated than it might otherwise be if we felt that relying entirely on scraped data (a la bills) was viable for committees. As a result it will require some additional upfront infrastructure work to achieve the hybrid scraped/manual data approach that is desired.
      • -
    • There will be ongoing maintenance costs to the new infrastructure and scrapers required for this.
      • -
    • As noted above plan does not account for historical committee information.
      • -
    -

    Backwards Compatibility Note: The new committee scraping code will not be backwards compatible with the old. The API and website will be updated in a way that doesn't disturb any public interfaces.

    -

    Implementation Plan

    -

    James will lead infrastructure implementation.

    -

    Several team members, and hopefully some community members, will help to contribute updated committee scrapers.

    - -

    This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.

    - - -
    -
    -
    - - - - Back to top - - -
    - - - -
    -
    -
    -
    - - - - - - - - \ No newline at end of file diff --git a/enhancement-proposals/005-dedupe-key/index.html b/enhancement-proposals/005-dedupe-key/index.html deleted file mode 100644 index 0324fe3..0000000 --- a/enhancement-proposals/005-dedupe-key/index.html +++ /dev/null @@ -1,1164 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - OSEP #5: Replacing pupa_id with dedupe_key - Open States - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - - - - Skip to content - - -
    -
    - -
    - - - - -
    - - -
    - -
    - - - - - - -
    -
    - - - -
    -
    -
    - - - - -
    -
    -
    - - - -
    -
    -
    - - - -
    -
    -
    - - -
    -
    - - - - - - - -

    OSEP #5: Replacing pupa_id with dedupe_key

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Author(s)@jamesturk
    Implementer(s)@jamesturk
    StatusAccepted
    Draft PR(s)https://github.com/openstates/enhancement-proposals/pull/19
    Approval PR(s)https://github.com/openstates/enhancement-proposals/pull/TBD
    Created2021-04-01
    Updated2021-04-01
    -
    -

    Abstract

    -

    There are cases where the automated import pipeline cannot distinguish between two similar objects and relies upon a hint from the scraper to decide if an object should be treated as unique or not. -Right now this is done by setting pupa_id, which is then stored in the pupa_identifier table with a join to the object in question. -This logic is not well understood, and the naming was poorly thought out. -This is a small OSEP that will fix this, making other deduplication work in the future easier to perform.

    -

    Specification

    -

    Scraped VoteEvents will gain a new field 'dedupe_key' which will be optional and function identically to pupa_id on import.

    -

    Rationale

    -

    The current name was never great, and as the last real reference to pupa makes it even more confusing to new people that aren't aware of the history.

    -

    Fixing this code now will make other import-side improvements to come easier.

    -

    Drawbacks

    -

    Very few, a little disruptive to alter a bunch of scrapers at once, but the plan allows doing it incrementally if needed.

    -

    This will be a minor change to the scrape output, which is the main reason this rises to the level of OSEP.

    -

    This will not have any public facing impact.

    -

    Implementation Plan

    -

    This needs to be done in several steps to avoid disruption:

    -
      -
    • addition of VoteEvent.dedupe_key in database
      • -
    • write a migration script that copies data from the pupa_identifier table to VoteEvent.dedupe_key and drops the old pupa_identifier table
      • -
    • write updated version of openstates-core which supports pupa_id in the scrape output but saves to dedupe_key on the backend
      • -
    • when ready, near-simultaneous release of openstates-core and run of migration script
      • -
    • update scrapers to not use pupa_id anymore
      • -
    • once all scrapers updated, release updated version of openstates-core which no longer supports pupa_id in the scrape output
      • -
    -

    James will handle all of this, once approved it should take 1-2 hours.

    - -

    This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.

    - - -
    -
    -
    - - - - Back to top - - -
    - - - -
    -
    -
    -
    - - - - - - - - \ No newline at end of file diff --git a/enhancement-proposals/006-new-people-offices/index.html b/enhancement-proposals/006-new-people-offices/index.html deleted file mode 100644 index 436e5a6..0000000 --- a/enhancement-proposals/006-new-people-offices/index.html +++ /dev/null @@ -1,1187 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - OSEP #6: New People Offices Schema - Open States - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - - - - Skip to content - - -
    -
    - -
    - - - - -
    - - -
    - -
    - - - - - - -
    -
    - - - -
    -
    -
    - - - - -
    -
    -
    - - - -
    -
    -
    - - - -
    -
    -
    - - -
    -
    - - - - - - - -

    OSEP #6: New People Offices Schema

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Author(s)@jamesturk
    Implementer(s)@jamesturk
    StatusFinal
    Issuehttps://github.com/openstates/enhancement-proposals/issues/26
    Draft PR(s)https://github.com/openstates/enhancement-proposals/pull/27
    Approval PR(s)https://github.com/openstates/enhancement-proposals/pull/TBD
    Created2021-08-13
    Updated2021-10-13
    -
    -

    Abstract

    -

    Make improvements to how office/contact information is represented to allow better modeling of real world scenarios & reduce user confusion.

    -

    Specification

    -

    Currently Open States stores contact information in a highly normalized form: -ContactDetail -- type (address|email|url|fax|text|voice|video|pager|textphone) [1] -- value -- note -- person_id

    -

    This is based on the Popolo spec, which heavily influenced Open Civic Data's modeling. One reason for this was to avoid a bunch of null columns for lesser used values, but in practice, we only use voice, fax, and address. (email was used but then moved to a top level field in early 2020).

    -

    In practice though, we typically tie addresses together "e.g. Capitol Office or District Office". This is done via note, which is now required to be either Capitol Office, District Office, or Primary Office.

    -

    This proposal suggests moving us towards a model that more accurately represents how most states provide (and users use) the data:

    -

    Office -- classification: capitol | district | primary -- address -- voice -- fax -- name (optional)

    -

    With additional validation rules: -- an office must have at least one of address, voice, fax. -- At most one address may have the classification capitol or primary. -- Any number of district addresses may be provided. -- If name is not provided, an office will display as Capitol Office, District Office, or Primary Office as it does now (based on classification)

    -

    Rationale

    -

    As noted above, we currently group address information into Capitol Office & District Office (& occasionally Primary Office). Doing so does not allow us to distinguish between multiple district offices, which are common in certain states (e.g. California). This leads to unclear pairings of data on OpenStates.org as well as an issue in API v3 which attempts to do this grouping for user convenience.

    -

    Additionally, the code to merge scraped people is quite complex and has several conditions that cause it to bail & request manual resolution. These conditions are direct consequences of the current schema.

    -

    For more clear examples of the issues presented, assume a person has the following ContactDetail records: -- type=voice, note=District, value=555-555-1234 -- type=address, note=District, value=123 Main Street -- type=voice, note=District, value=555-555-6666 -- type=address, note=District, value=1600 Pennsylvania Ave

    -

    OpenStates.org will show two district offices, but there is no way to know which number corresponds to which address, so they will be grouped based on database retrieval order.

    -

    API v3 will only show one district office, since the logic there is to attempt to convert ContactDetail records into more usable "offices" which works well for most cases, but the pairing information is not currently available in this case.

    -

    Finally, the people merge code (openstates-core/people) will not be able to merge any changes that affect the District offices since it can not be confident which phone number was added/removed/changed. (Essentially due to missing a unique data path to the changed item, which is not an issue with any other piece of data at present.)

    -

    Drawbacks

    -

    We will lose the flexibility of supporting other contact detail types. Given that they haven't been used thus far makes that feel like a reasonable trade-off.

    -

    This proposal will require consumers of the raw people data to update their code, as the openstates/people repository will need to be updated, with a migration script run to convert existing offices to the new format.

    -

    Implementation Plan

    -

    After approval, a script will be written to convert openstates/people to the new data format. All appropriate data scripts (os-people to-database and merge) will be updated, initially to import the data to both the old ContactDetail table as well as the new Offices table.

    -

    Initially this means that upstream impact will be limited, this will give us time to update OpenStates.org, API v2, and v3.

    -

    OpenStates.org's display logic will be changed to support the new offices data.

    -

    From this point, the GraphQL API will be updated to support both forms with a transformation step to allow backwards compatibility. The new offices key will be added and queryable via GraphQL, whereas the old contactDetails key can be left intact as-is with a backwards-compatibility shim put in place.

    -

    API v3 will be updated to query the database's offices key. Given that it already uses the offices key in responses, the new name & classification key will be the only impact there.

    -

    After all of these changes are implemented, the old ContactDetail table can be dropped from the database & import process.

    - -

    This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.

    - - -
    -
    -
    - - - - Back to top - - -
    - - - -
    -
    -
    -
    - - - - - - - - \ No newline at end of file diff --git a/enhancement-proposals/007-events/index.html b/enhancement-proposals/007-events/index.html deleted file mode 100644 index 9630094..0000000 --- a/enhancement-proposals/007-events/index.html +++ /dev/null @@ -1,1250 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - OSEP #7: Restoring Events Data - Open States - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - - - - Skip to content - - -
    -
    - -
    - - - - -
    - - -
    - -
    - - - - - - -
    -
    - - - -
    -
    -
    - - - - -
    -
    -
    - - - -
    -
    -
    - - - -
    -
    -
    - - -
    -
    - - - - - - - -

    OSEP #7: Restoring Events Data

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Author(s)@jamesturk
    Implementer(s)@jamesturk
    StatusFinal
    Issuehttps://github.com/openstates/enhancement-proposals/issues/34
    Draft PR(s)https://github.com/openstates/enhancement-proposals/pull/TBD
    Approval PR(s)https://github.com/openstates/enhancement-proposals/pull/TBD
    Created2021-08-13
    Updated2021-10-13
    -
    -

    Abstract

    -

    From around 2010-2015 Open States scraped "events" data, mostly data on upcoming hearings and legislative meetings. That data has not been part of Open States' data offerings for quite a while, despite maintained scrapers in many states.

    -

    This proposal would restore events data to Open States' public data offerings.

    -

    Specification

    -

    The existing events scrapers would regain "first class" status, and be run regularly (daily at least).

    -

    The existing schema would be left mostly intact, and Event importers would be restored. Additional scrape jobs would be configured to run event scrapers regularly again & report failures similarly to how bill scrapers are configured today.

    -

    API v3 will be updated to include event data. For now, OpenStates.org & API v2 will not be affected.

    -

    Some changes will be introduced as part of this proposal:

    -

    Schema Changes

    -

    Scraped Events will gain the following optional fields:

    -
      -
    • upstream_id: This can be used to record an upstream identifier, such as a database identifier that can be obtained from the source data. If present it will be used to uniquely identify events. (This is distinct from dedupe_key which has no semantic meaning and can be a constructed value or URL.)
      • -
    -

    Soft Deletes on Import

    -

    To better address downstream user's needs, when future events can not be found in the current scrape nor reconciled via the standard means (dedupe_key, the new upstream_id, nor a match on the main attributes), the future event will be marked as deleted in the database.

    -

    Deleted events will not be returned in the API response by default, but may be explicitly requested by clients.

    -

    Windowing

    -

    In some states we have to select a date range to scrape, which currently leads to ad hoc code (as described by Tim here: https://github.com/openstates/enhancement-proposals/pull/28#issuecomment-898720989)

    -

    To bring some sense of standardization to this, we will standardize on the following parameters:

    -
      -
    • today (default: today's date, can be overridden for testing purposes)
      • -
    • days_before (default: 30)
      • -
    • days_after (default: 90)
      • -
    -

    All of which can be overridden on the command line. A scraper that supports this interface will call a utility function with -these variables and obtain a start_date & end_date centered around the today value. The obtained start/end dates should then be used by the scraper to window its request to the source.

    -

    Helper Methods

    -

    The openstates.scrape.Event object will gain a couple of helper functions:

    -
      -
    • Event.add_bill which will add an agenda item with configurable placeholder text (if one does not exist already) and associate a bill with that item. This will serve as a workaround to associate bills with events that do not have an appropriate agenda item already.
      • -
    • Event.add_media_link will gain a classification parameter to classify the type of media being added. Options will include: 'video', 'audio', 'hosted video', 'hosted audio', 'youtube'.
      • -
    -

    Rationale

    -

    This is valuable data, and with renewed development resources, it has again become feasible to support maintaining it.

    -

    Drawbacks

    -

    The main drawback here is just in terms of resource allocation, supporting this across states will be significant work. There are no backwards compatibility concerns to address.

    -

    Implementation Plan

    -

    Tim has already been maintaining the events scrapers within the openstates-scrapers repository. Now they will be maintained by other members of the team as well.

    -

    The additional work to support the import of events & access via the API will be done by James or others under his supervision.

    - -

    This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.

    - - -
    -
    -
    - - - - Back to top - - -
    - - - -
    -
    -
    -
    - - - - - - - - \ No newline at end of file diff --git a/enhancement-proposals/008-active-sessions/index.html b/enhancement-proposals/008-active-sessions/index.html deleted file mode 100644 index 9b593b7..0000000 --- a/enhancement-proposals/008-active-sessions/index.html +++ /dev/null @@ -1,1153 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - OSEP #8: Active Sessions - Open States - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - - - - Skip to content - - -
    -
    - -
    - - - - -
    - - -
    - -
    - - - - - - -
    -
    - - - -
    -
    -
    - - - - -
    -
    -
    - - - -
    -
    -
    - - - -
    -
    -
    - - -
    -
    - - - - - - - -

    OSEP #8: Active Sessions

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Author(s)@jamesturk
    Implementer(s)@jamesturk
    StatusFinal
    Issuehttps://github.com/openstates/enhancement-proposals/issues/29
    Draft PR(s)https://github.com/openstates/enhancement-proposals/pull/30
    Approval PR(s)https://github.com/openstates/enhancement-proposals/pull/TBD
    Created2021-08-16
    Updated2021-11-08
    -
    -

    Abstract

    -

    This proposal would add a flag to sessions indicating whether or not it is considered active or not. This information would be used for determining which sessions to scrape by default, and could be used within the openstates.org UI as well.

    -

    Specification

    -

    The Legislative Session schema will be updated to have a required field named active to indicate if the session is currently receiving updates. Other portions of the code will be updated to use this field as needed, most notably the code that selects which session to run if no session is given on the command line, and the OpenStates.org search logic. This field will also be surfaced in the APIs so that downstream consumers can make decisions based upon it as they wish.

    -

    Additional care will be taken to add validation rules that either error out or warn loudly if the flag is potentially set incorrectly.

    -

    Rationale

    -

    This helps address two major problems that are present right now: -- When scrapers are run, only the latest (by order of appearance in the metadata) session is scraped. Due to complex special session rules, that means that the active session information winds up within the task-definitions repository, where we add additional tasks for one-off re-scrapes, special sessions, etc. This also means when running locally, special care must be taken to run with the correct session=XYZ argument. -- When doing things like full text search, that should default to the current session- we use the same rule (latest in the metadata) whereas it might be more beneficial to search within multiple sessions or a session that is not the latest but is more likely to contain the bill(s) being searched for.

    -

    The main alternative considered was the idea to use begin & end dates instead (see https://github.com/openstates/enhancement-proposals/issues/29). It was pointed out that there are a lot of special cases which would render this solution ineffective, for example: -- IL governor sometimes signs bills from the previous session even after the new one started, this year I think that happened as late as march 2022 after 2021 regular session close. -- VA as of August 2021 is in a state where the Regular session is getting updates (from the executive) despite being adjourned. Special #1 is finished, and Special #2 is actively meeting. -- Prefiles will need to be considered active before their start date to be scraped.

    -

    These cases make using start & end date fraught, as we'd wind up needing to manipulate them to our needs or still falling back to overrides.

    -

    Drawbacks

    -

    This will be an extra piece of data to manually maintain across each state. It is somewhat redundant with the begin & end date (but not entirely, as noted above in Rationale).

    -

    It is not unlikely that we'll run into issues where we forget to update the flag & continue to scrape old sessions. As noted in the specification, special care will be taken to try to add checks for this behavior that run alongside the session check code that runs at scraper start.

    -

    Implementation Plan

    -

    James will take the lead on adding this to openstates-core: -- schema support for LegislativeSession.active -- expose & document new flag in APIs -- additional checks at scraper start that the active session list is sane -- an automated update across all jurisdictions to set active sessions (defaulting to use current last-is-active logic) -- fix os-update command to use new logic -- fix OpenStates.org search to use active sessions by default instead of current logic which assesses latest session with any bills

    - -

    This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.

    - - -
    -
    -
    - - - - Back to top - - -
    - - - -
    -
    -
    -
    - - - - - - - - \ No newline at end of file diff --git a/enhancement-proposals/index.html b/enhancement-proposals/index.html deleted file mode 100644 index 14a6fdb..0000000 --- a/enhancement-proposals/index.html +++ /dev/null @@ -1,938 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - enhancement-proposals - Open States - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - - - - Skip to content - - -
    -
    - -
    - - - - -
    - - -
    - -
    - - - - - - -
    -
    - - - -
    -
    -
    - - - - -
    -
    -
    - - - -
    -
    -
    - - - -
    -
    -
    - - -
    -
    - - - - - - - -

    enhancement-proposals

    -

    Proposals to enhance data or functionality within the Open States project.

    -

    See OSEP #1

    -

    You can open an issue or use template.md to get started on your own proposal.

    - - -
    -
    -
    - - - - Back to top - - -
    - - - -
    -
    -
    -
    - - - - - - - - \ No newline at end of file diff --git a/enhancement-proposals/template/index.html b/enhancement-proposals/template/index.html deleted file mode 100644 index ba9d24e..0000000 --- a/enhancement-proposals/template/index.html +++ /dev/null @@ -1,1042 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - OSEP #: [TITLE] - Open States - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - - - - Skip to content - - -
    -
    - -
    - - - - -
    - - -
    - -
    - - - - - - -
    -
    - - - -
    -
    -
    - - - - -
    -
    -
    - - - -
    -
    -
    - - - -
    -
    -
    - - -
    -
    - - - - - - - -

    OSEP #: [TITLE]

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Author(s)TODO
    Implementer(s)TODO
    StatusDraft
    Issuehttps://github.com/openstates/enhancement-proposals/issues/TBD
    Draft PR(s)https://github.com/openstates/enhancement-proposals/pull/TBD
    Approval PR(s)https://github.com/openstates/enhancement-proposals/pull/TBD
    CreatedTODO
    UpdatedTODO
    -
    -

    Abstract

    -

    TODO: Write a short summary of what you are hoping to achieve.

    -

    Specification

    -

    TODO: Describe how the proposal will work.

    -

    Rationale

    -

    TODO: Explain the reason for this in detail. Discuss alternatives considered.

    -

    Drawbacks

    -

    TODO: Will this affect backwards compatibility in scrapers, the API, or the website? -(It is ok to leave this blank in a draft if you aren't sure.)

    -

    Implementation Plan

    -

    TODO: How will this be done? Are you volunteering to do it? Do you want someone else to do it? -(It is ok to leave this blank in a draft if you aren't sure.)

    - -

    This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.

    - - -
    -
    -
    - - - - Back to top - - -
    - - - -
    -
    -
    -
    - - - - - - - - \ No newline at end of file diff --git a/index.html b/index.html index 2ac9d8d..143b16e 100644 --- a/index.html +++ b/index.html @@ -784,8 +784,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -798,8 +798,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -812,8 +812,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -826,8 +826,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -840,8 +840,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -854,8 +854,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -868,8 +868,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -882,8 +882,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/openstates.org/scheduled-tasks/index.html b/openstates.org/scheduled-tasks/index.html index ddd40d7..b721ee8 100644 --- a/openstates.org/scheduled-tasks/index.html +++ b/openstates.org/scheduled-tasks/index.html @@ -716,8 +716,8 @@
  • - - OSEP #1: Purpose & Process + + None
    • @@ -730,8 +730,8 @@
  • - - OSEP #2: Legal Citations + + None
    • @@ -744,8 +744,8 @@
  • - - OSEP #3: Manual People Data Tools + + None
    • @@ -758,8 +758,8 @@
  • - - OSEP #4: Committee Data + + None
    • @@ -772,8 +772,8 @@
  • - - OSEP #5: Replacing pupa_id with dedupe_key + + None
    • @@ -786,8 +786,8 @@
  • - - OSEP #6: New People Offices Schema + + None
    • @@ -800,8 +800,8 @@
  • - - OSEP #7: Restoring Events Data + + None
    • @@ -814,8 +814,8 @@
  • - - OSEP #8: Active Sessions + + None
    • diff --git a/search/search_index.json b/search/search_index.json index 49029c1..1a92086 100644 --- a/search/search_index.json +++ b/search/search_index.json @@ -1 +1 @@ -{"config":{"indexing":"full","lang":["en"],"min_search_length":3,"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"Introduction \u00b6 Welcome to the Documentation for Open States. The sections below cover contributing to scrapers & data, and how to use the API. About Open States \u00b6 Open States strives to improve civic engagement at the state level by providing data and tools regarding state legislatures. We aim to serve members of the public, activist groups, journalists, and researchers with better data on what is happening in their state capital, and to provide tools to reduce barriers to participation and increase engagement. The project aggregates legislative information from all 50 states, Washington, D.C., and Puerto Rico. This information is then standardized, cleaned, and published to the public via OpenStates.org, a powerful API, and bulk downloads. All of this work is done in the open , and depends heavily on contributors striving to make this important resource available. Thank you for your interest in our community, whether you are looking to use our bulk data or APIs or interested in contributing data or code, we're glad to have you here. Communication \u00b6 When joining a new community, it can be tough to figure out where to ask questions, provide feedback, or help out. Don't worry! As long as you're respectful and follow our Code of Conduct , we're happy to have you! Here are some guidelines regarding the best way to get in touch or contribute. Do note that Open States is a volunteer-powered project, and all of the core developers have day jobs; we're excited to talk to you, but it may sometimes take a bit of time to get back to you. Recommendations \u00b6 Want to ask a general question, have a conversation, or keep up with the community? The best place is our Discussion Forum . This is the main place for Open States discussion. The core team and many other contributors are present there, and we're usually able to answer questions in a timely fashion. Have a private question, or a security concern? Email contact@openstates.org ; only the administrative team can see these. Have you found an error or issue in the Open States data? Have a technical issue not related to the data itself? File an issue on our bug tracker . And before you do, quickly check whether anyone else there has already reported the same bug. Want to contribute to the project more regularly? We also have a Slack that you can join. It is not the best place for asking questions better handled by filing an issue or opening a conversation on GitHub, but you're welcome to join if you're interested in being a part of the community . Discouraged Methods of Communication \u00b6 Please avoid using these channels to get in touch with us: Personal email addresses of Open States developers Please respect our boundaries & refrain from contacting any of the developers directly, unless we ask you to do so. Twitter (or any other social media) We mainly use the @openstates twitter account to make announcements, and don't have the resources to provide technical support or other feedback on Twitter.","title":"Introduction"},{"location":"#introduction","text":"Welcome to the Documentation for Open States. The sections below cover contributing to scrapers & data, and how to use the API.","title":"Introduction"},{"location":"#about-open-states","text":"Open States strives to improve civic engagement at the state level by providing data and tools regarding state legislatures. We aim to serve members of the public, activist groups, journalists, and researchers with better data on what is happening in their state capital, and to provide tools to reduce barriers to participation and increase engagement. The project aggregates legislative information from all 50 states, Washington, D.C., and Puerto Rico. This information is then standardized, cleaned, and published to the public via OpenStates.org, a powerful API, and bulk downloads. All of this work is done in the open , and depends heavily on contributors striving to make this important resource available. Thank you for your interest in our community, whether you are looking to use our bulk data or APIs or interested in contributing data or code, we're glad to have you here.","title":"About Open States"},{"location":"#communication","text":"When joining a new community, it can be tough to figure out where to ask questions, provide feedback, or help out. Don't worry! As long as you're respectful and follow our Code of Conduct , we're happy to have you! Here are some guidelines regarding the best way to get in touch or contribute. Do note that Open States is a volunteer-powered project, and all of the core developers have day jobs; we're excited to talk to you, but it may sometimes take a bit of time to get back to you.","title":"Communication"},{"location":"#recommendations","text":"Want to ask a general question, have a conversation, or keep up with the community? The best place is our Discussion Forum . This is the main place for Open States discussion. The core team and many other contributors are present there, and we're usually able to answer questions in a timely fashion. Have a private question, or a security concern? Email contact@openstates.org ; only the administrative team can see these. Have you found an error or issue in the Open States data? Have a technical issue not related to the data itself? File an issue on our bug tracker . And before you do, quickly check whether anyone else there has already reported the same bug. Want to contribute to the project more regularly? We also have a Slack that you can join. It is not the best place for asking questions better handled by filing an issue or opening a conversation on GitHub, but you're welcome to join if you're interested in being a part of the community .","title":"Recommendations"},{"location":"#discouraged-methods-of-communication","text":"Please avoid using these channels to get in touch with us: Personal email addresses of Open States developers Please respect our boundaries & refrain from contacting any of the developers directly, unless we ask you to do so. Twitter (or any other social media) We mainly use the @openstates twitter account to make announcements, and don't have the resources to provide technical support or other feedback on Twitter.","title":"Discouraged Methods of Communication"},{"location":"code-of-conduct/","text":"Code of Conduct \u00b6 Our Pledge \u00b6 We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. Our Standards \u00b6 Examples of behavior that contributes to a positive environment for our community include: Demonstrating empathy and kindness toward other people Being respectful of differing opinions, viewpoints, and experiences Giving and gracefully accepting constructive feedback Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behavior include: The use of sexualized language or imagery, and sexual attention or advances of any kind Trolling, insulting or derogatory comments, and personal or political attacks Public or private harassment Publishing others' private information, such as a physical or email address, without their explicit permission Other conduct which could reasonably be considered inappropriate in a professional setting Enforcement Responsibilities \u00b6 Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. Scope \u00b6 This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Enforcement \u00b6 Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at contact@openstates.org . All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter of any incident. Enforcement Guidelines \u00b6 Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: 1. Correction \u00b6 Community Impact : Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. Consequence : A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. 2. Warning \u00b6 Community Impact : A violation through a single incident or series of actions. Consequence : A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. 3. Temporary Ban \u00b6 Community Impact : A serious violation of community standards, including sustained inappropriate behavior. Consequence : A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. 4. Permanent Ban \u00b6 Community Impact : Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. Consequence : A permanent ban from any sort of public interaction within the project community. Attribution \u00b6 This Code of Conduct is adapted from the Contributor Covenant , version 2.0, available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html . Community Impact Guidelines were inspired by Mozilla's code of conduct enforcement ladder . For answers to common questions about this code of conduct, see the FAQ at https://www.contributor-covenant.org/faq . Translations are available at https://www.contributor-covenant.org/translations .","title":"Code of Conduct"},{"location":"code-of-conduct/#code-of-conduct","text":"","title":"Code of Conduct"},{"location":"code-of-conduct/#our-pledge","text":"We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.","title":"Our Pledge"},{"location":"code-of-conduct/#our-standards","text":"Examples of behavior that contributes to a positive environment for our community include: Demonstrating empathy and kindness toward other people Being respectful of differing opinions, viewpoints, and experiences Giving and gracefully accepting constructive feedback Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behavior include: The use of sexualized language or imagery, and sexual attention or advances of any kind Trolling, insulting or derogatory comments, and personal or political attacks Public or private harassment Publishing others' private information, such as a physical or email address, without their explicit permission Other conduct which could reasonably be considered inappropriate in a professional setting","title":"Our Standards"},{"location":"code-of-conduct/#enforcement-responsibilities","text":"Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate.","title":"Enforcement Responsibilities"},{"location":"code-of-conduct/#scope","text":"This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event.","title":"Scope"},{"location":"code-of-conduct/#enforcement","text":"Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at contact@openstates.org . All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter of any incident.","title":"Enforcement"},{"location":"code-of-conduct/#enforcement-guidelines","text":"Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct:","title":"Enforcement Guidelines"},{"location":"code-of-conduct/#1-correction","text":"Community Impact : Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. Consequence : A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested.","title":"1. Correction"},{"location":"code-of-conduct/#2-warning","text":"Community Impact : A violation through a single incident or series of actions. Consequence : A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban.","title":"2. Warning"},{"location":"code-of-conduct/#3-temporary-ban","text":"Community Impact : A serious violation of community standards, including sustained inappropriate behavior. Consequence : A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban.","title":"3. Temporary Ban"},{"location":"code-of-conduct/#4-permanent-ban","text":"Community Impact : Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. Consequence : A permanent ban from any sort of public interaction within the project community.","title":"4. Permanent Ban"},{"location":"code-of-conduct/#attribution","text":"This Code of Conduct is adapted from the Contributor Covenant , version 2.0, available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html . Community Impact Guidelines were inspired by Mozilla's code of conduct enforcement ladder . For answers to common questions about this code of conduct, see the FAQ at https://www.contributor-covenant.org/faq . Translations are available at https://www.contributor-covenant.org/translations .","title":"Attribution"},{"location":"api-v2/","text":"GraphQL API \u00b6 API keys are required. You can register for an API key and once activated, you'll pass your API key via the X-API-KEY header. You can also check out our introductory blog post for more details. Basics \u00b6 This is a GraphQL API, and some of the concepts may seem unfamiliar at first. There is in essence, only one endpoint: https://openstates.org/graphql . This endpoint, when accessed in a browser, will provide an interface that allows you to experiment with queries in the browser, it features autocomplete and a way to browse the full graph (click the \\'Docs\\' link in the upper right corner). A GraphQL query mirrors the structure of the data that you\\'d like to obtain. For example, to obtain a list of legislators you\\'d pass something like: { people { edges { node { name } } } } Note If you are using the API programatically it is recommended you send the data as part of the POST body, e.g.: curl -X POST https://openstates.org/graphql -d \"query={people{edges{node{name}}}}\" Of course, if you try this you'll see it doesn't work since there are some basic limits on how much data you can request at once. We paginate with the first , last , before and after parameters to a root node. So let's try that again: { people(first: 3) { edges { node { name } } } } And you'd get back JSON like: { \"data\": { \"people\": { \"edges\": [ { \"node\": { \"name\": \"Lydia Brasch\" } }, { \"node\": { \"name\": \"Matt Williams\" } }, { \"node\": { \"name\": \"Merv Riepe\" } } ] } } } Ah, much better. Nodes also can take other parameters to filter the returned content. Let's try the \"name\" filter which restricts our search to people named Lydia: { people(first: 3, name: \"Lydia\") { edges { node { name } } } } Results in: { \"data\": { \"people\": { \"edges\": [ { \"node\": { \"name\": \"Lydia Brasch\" } }, { \"node\": { \"name\": \"Lydia Graves Chassaniol\" } }, { \"node\": { \"name\": \"Lydia C. Blume\" } } ] } } } It is also possible to request data from multiple root nodes at once, for example: { people(first: 1) { edges { node { name } } } bills(first: 1) { edges { node { title } } } } Would give back something like: { \"data\": { \"people\": { \"edges\": [ { \"node\": { \"name\": \"Lydia Brasch\" } } ] }, \"bills\": { \"edges\": [ { \"node\": { \"title\": \"Criminal Law - Animal Abuse Emergency Compensation Fund - Establishment\" } } ] } } } You may notice something here, that you get back just the data you need. This is extremely powerful, and lets you do the equivalent of many traditional API calls in a single query. Full-fledged Example \u00b6 Let's take a look at a more useful example: { bill(jurisdiction: \"New York\", session: \"2017-2018\", identifier: \"S 5772\") { title actions { description date } votes { edges { node { counts { value option } votes { voterName voter { id contactDetails { value note type } } option } } } } sources { url } createdAt updatedAt } } There's a lot going on there, let's break it down: bill(jurisdiction: \"New York\", session: \"2017-2018\", identifier: \"S 5772\") { We're hitting the bill root node, which takes 3 parameters. This should get us to a single bill from New York. title This is going to give us the title, just like we saw before. actions { description date } Here we're going into a child node, in this case all of the actions taken on the bill. For each action we're requesting a the date & description. votes { edges { node { Here too we're going into a child node, but note that this time we use that \"edges\" and \"node\" pattern that we see on root level nodes. Certain child nodes in the API have the ability to be paginated or further limited, and votes happen to be one of them. In this case however we're not making use of that so we'll just ignore this. (A full discussion of this pattern is out of scope but check out the Relay pagination specification for more detail for more.) counts { value option } votes { voterName voter { id contactDetails { value note type } } option } } Here we grab a few more fields, including child nodes of each vote on our Bill. First, we get a list of counts (essentially pairs of outcomes + numbers e.g. (yes, 31), (no, 5)) We also get individual legislator votes by name, and we traverse into another object to get the Open States ID and contact details for the voter. (Don't sweat the exact data model here, there will be more on the structure once we get to the actual graph documentation.) sources { url } createdAt updatedAt And back up at the top level, we grab a few more pieces of information about the Bill. And now you've seen a glimpse of the power of this API. We were able to get back theexact fields we wanted on a bill, contact information on the legislators that have voted on the bill, and more. Our result looks like this: { \"data\": { \"bill\": { \"title\": \"Relates to bureaus of administrative adjudication\", \"actions\": [ { \"description\": \"REFERRED TO LOCAL GOVERNMENT\", \"date\": \"2017-04-28\" }, { \"description\": \"COMMITTEE DISCHARGED AND COMMITTED TO RULES\", \"date\": \"2017-06-19\" }, { \"description\": \"ORDERED TO THIRD READING CAL.1896\", \"date\": \"2017-06-19\" }, { \"description\": \"RECOMMITTED TO RULES\", \"date\": \"2017-06-21\" } ], \"votes\": { \"edges\": [ { \"node\": { \"counts\": [ { \"value\": 25, \"option\": \"yes\" }, { \"value\": 0, \"option\": \"no\" }, { \"value\": 0, \"option\": \"other\" } ], \"votes\": [ { \"voterName\": \"John J. Bonacic\", \"voter\": { \"id\": \"ocd-person/da013cd5-dc67-4e65-a310-73aa32ad1f7c\" \"contactDetails\": [ { \"value\": \"bonacic@nysenate.gov\", \"note\": \"Capitol Office\", \"type\": \"email\" }, { \"value\": \"Room 503\\nAlbany, NY 12247\", \"note\": \"District Office\", \"type\": \"address\" }, { \"value\": \"518-455-3181\", \"note\": \"District Office\", \"type\": \"voice\" }, ...etc... ] }, \"option\": \"yes\" }, { \"voterName\": \"Neil D. Breslin\", \"voter\": { \"id\": \"ocd-person/4b710aee-1b99-42e0-90e2-d41338e8c5df\" \"contactDetails\": [ ...etc... ], }, \"option\": \"yes\" }, { \"voterName\": \"David Carlucci\", \"voter\": { \"id\": \"ocd-person/1b0feab9-02a7-4bcc-b089-3ab23286da68\" \"contactDetails\": [ ...etc... ], }, \"option\": \"yes\" }, ] } }, ...etc... ] }, \"sources\": [ { \"url\": \"http://legislation.nysenate.gov/api/3/bills/2017-2018/S5772?summary=&detail=\" }, { \"url\": \"http://www.nysenate.gov/legislation/bills/2017/S5772\" }, { \"url\": \"http://assembly.state.ny.us/leg/?default_fld=&bn=S5772&Summary=Y&Actions=Y&Text=Y\" } ], \"createdAt\": \"2017-07-15 05:08:15.848526+00:00\", \"updatedAt\": \"2017-07-15 05:08:15.848541+00:00\" } } }","title":"GraphQL API"},{"location":"api-v2/#graphql-api","text":"API keys are required. You can register for an API key and once activated, you'll pass your API key via the X-API-KEY header. You can also check out our introductory blog post for more details.","title":"GraphQL API"},{"location":"api-v2/#basics","text":"This is a GraphQL API, and some of the concepts may seem unfamiliar at first. There is in essence, only one endpoint: https://openstates.org/graphql . This endpoint, when accessed in a browser, will provide an interface that allows you to experiment with queries in the browser, it features autocomplete and a way to browse the full graph (click the \\'Docs\\' link in the upper right corner). A GraphQL query mirrors the structure of the data that you\\'d like to obtain. For example, to obtain a list of legislators you\\'d pass something like: { people { edges { node { name } } } } Note If you are using the API programatically it is recommended you send the data as part of the POST body, e.g.: curl -X POST https://openstates.org/graphql -d \"query={people{edges{node{name}}}}\" Of course, if you try this you'll see it doesn't work since there are some basic limits on how much data you can request at once. We paginate with the first , last , before and after parameters to a root node. So let's try that again: { people(first: 3) { edges { node { name } } } } And you'd get back JSON like: { \"data\": { \"people\": { \"edges\": [ { \"node\": { \"name\": \"Lydia Brasch\" } }, { \"node\": { \"name\": \"Matt Williams\" } }, { \"node\": { \"name\": \"Merv Riepe\" } } ] } } } Ah, much better. Nodes also can take other parameters to filter the returned content. Let's try the \"name\" filter which restricts our search to people named Lydia: { people(first: 3, name: \"Lydia\") { edges { node { name } } } } Results in: { \"data\": { \"people\": { \"edges\": [ { \"node\": { \"name\": \"Lydia Brasch\" } }, { \"node\": { \"name\": \"Lydia Graves Chassaniol\" } }, { \"node\": { \"name\": \"Lydia C. Blume\" } } ] } } } It is also possible to request data from multiple root nodes at once, for example: { people(first: 1) { edges { node { name } } } bills(first: 1) { edges { node { title } } } } Would give back something like: { \"data\": { \"people\": { \"edges\": [ { \"node\": { \"name\": \"Lydia Brasch\" } } ] }, \"bills\": { \"edges\": [ { \"node\": { \"title\": \"Criminal Law - Animal Abuse Emergency Compensation Fund - Establishment\" } } ] } } } You may notice something here, that you get back just the data you need. This is extremely powerful, and lets you do the equivalent of many traditional API calls in a single query.","title":"Basics"},{"location":"api-v2/#full-fledged-example","text":"Let's take a look at a more useful example: { bill(jurisdiction: \"New York\", session: \"2017-2018\", identifier: \"S 5772\") { title actions { description date } votes { edges { node { counts { value option } votes { voterName voter { id contactDetails { value note type } } option } } } } sources { url } createdAt updatedAt } } There's a lot going on there, let's break it down: bill(jurisdiction: \"New York\", session: \"2017-2018\", identifier: \"S 5772\") { We're hitting the bill root node, which takes 3 parameters. This should get us to a single bill from New York. title This is going to give us the title, just like we saw before. actions { description date } Here we're going into a child node, in this case all of the actions taken on the bill. For each action we're requesting a the date & description. votes { edges { node { Here too we're going into a child node, but note that this time we use that \"edges\" and \"node\" pattern that we see on root level nodes. Certain child nodes in the API have the ability to be paginated or further limited, and votes happen to be one of them. In this case however we're not making use of that so we'll just ignore this. (A full discussion of this pattern is out of scope but check out the Relay pagination specification for more detail for more.) counts { value option } votes { voterName voter { id contactDetails { value note type } } option } } Here we grab a few more fields, including child nodes of each vote on our Bill. First, we get a list of counts (essentially pairs of outcomes + numbers e.g. (yes, 31), (no, 5)) We also get individual legislator votes by name, and we traverse into another object to get the Open States ID and contact details for the voter. (Don't sweat the exact data model here, there will be more on the structure once we get to the actual graph documentation.) sources { url } createdAt updatedAt And back up at the top level, we grab a few more pieces of information about the Bill. And now you've seen a glimpse of the power of this API. We were able to get back theexact fields we wanted on a bill, contact information on the legislators that have voted on the bill, and more. Our result looks like this: { \"data\": { \"bill\": { \"title\": \"Relates to bureaus of administrative adjudication\", \"actions\": [ { \"description\": \"REFERRED TO LOCAL GOVERNMENT\", \"date\": \"2017-04-28\" }, { \"description\": \"COMMITTEE DISCHARGED AND COMMITTED TO RULES\", \"date\": \"2017-06-19\" }, { \"description\": \"ORDERED TO THIRD READING CAL.1896\", \"date\": \"2017-06-19\" }, { \"description\": \"RECOMMITTED TO RULES\", \"date\": \"2017-06-21\" } ], \"votes\": { \"edges\": [ { \"node\": { \"counts\": [ { \"value\": 25, \"option\": \"yes\" }, { \"value\": 0, \"option\": \"no\" }, { \"value\": 0, \"option\": \"other\" } ], \"votes\": [ { \"voterName\": \"John J. Bonacic\", \"voter\": { \"id\": \"ocd-person/da013cd5-dc67-4e65-a310-73aa32ad1f7c\" \"contactDetails\": [ { \"value\": \"bonacic@nysenate.gov\", \"note\": \"Capitol Office\", \"type\": \"email\" }, { \"value\": \"Room 503\\nAlbany, NY 12247\", \"note\": \"District Office\", \"type\": \"address\" }, { \"value\": \"518-455-3181\", \"note\": \"District Office\", \"type\": \"voice\" }, ...etc... ] }, \"option\": \"yes\" }, { \"voterName\": \"Neil D. Breslin\", \"voter\": { \"id\": \"ocd-person/4b710aee-1b99-42e0-90e2-d41338e8c5df\" \"contactDetails\": [ ...etc... ], }, \"option\": \"yes\" }, { \"voterName\": \"David Carlucci\", \"voter\": { \"id\": \"ocd-person/1b0feab9-02a7-4bcc-b089-3ab23286da68\" \"contactDetails\": [ ...etc... ], }, \"option\": \"yes\" }, ] } }, ...etc... ] }, \"sources\": [ { \"url\": \"http://legislation.nysenate.gov/api/3/bills/2017-2018/S5772?summary=&detail=\" }, { \"url\": \"http://www.nysenate.gov/legislation/bills/2017/S5772\" }, { \"url\": \"http://assembly.state.ny.us/leg/?default_fld=&bn=S5772&Summary=Y&Actions=Y&Text=Y\" } ], \"createdAt\": \"2017-07-15 05:08:15.848526+00:00\", \"updatedAt\": \"2017-07-15 05:08:15.848541+00:00\" } } }","title":"Full-fledged Example"},{"location":"api-v2/changelog/","text":"Changelog \u00b6 Changelog for Open States GraphQL API: v2.6 (March 2021) \u00b6 added preliminary support for federal jurisdiction added Jurisdiction.classification node Jurisdictions are now filterable by classification (municipal, state, country) v2.5 (July 2020) \u00b6 added Jurisdiction.lastScrapedAt, openstates/issues#32 v2.4 (April 2020) \u00b6 removed unused fields from graph (organization.links, organization.other_names) v2.3 (August 2019) \u00b6 add experimental full text search via searchQuery parameter to bills node v2.2 (June 2019) \u00b6 add openstatesUrl to bills query speed improvments v2.1 (Feb 2019) \u00b6 fix lat-lon behavior to limit to active memberships improve handling of retired legislators fix type of maximum_memberships bill version ordering is now consistent v2.0 (January 2019) \u00b6 bugfix for maximum_memberships type bugfix for versions field improve tests Beta Release (November 2018) \u00b6 API Keys are now required consider classification when using current_memberships fix geo filtering add openstatesUrl to Bill node for ease of linkage to OpenStates.org add Person.oldMemberships as analog to currentMemberships add actionSince filter to bills node fix 500 errors/optimization when using GraphQL fragments addition of basic protection for excessive queries add totalCount to assist in pagination add Organization.currentMemberships Preview Release 1 (May 2018) \u00b6 fix for people pagination add updatedSince for people add sponsor argument for bills node allow votes to take pagination parameters allow traversing to votes from person Preview Release 0 (Dec 2017) \u00b6 Initial draft release of the API, no backwards-compatibility guarantee made.","title":"Changelog"},{"location":"api-v2/changelog/#changelog","text":"Changelog for Open States GraphQL API:","title":"Changelog"},{"location":"api-v2/changelog/#v26-march-2021","text":"added preliminary support for federal jurisdiction added Jurisdiction.classification node Jurisdictions are now filterable by classification (municipal, state, country)","title":"v2.6 (March 2021)"},{"location":"api-v2/changelog/#v25-july-2020","text":"added Jurisdiction.lastScrapedAt, openstates/issues#32","title":"v2.5 (July 2020)"},{"location":"api-v2/changelog/#v24-april-2020","text":"removed unused fields from graph (organization.links, organization.other_names)","title":"v2.4 (April 2020)"},{"location":"api-v2/changelog/#v23-august-2019","text":"add experimental full text search via searchQuery parameter to bills node","title":"v2.3 (August 2019)"},{"location":"api-v2/changelog/#v22-june-2019","text":"add openstatesUrl to bills query speed improvments","title":"v2.2 (June 2019)"},{"location":"api-v2/changelog/#v21-feb-2019","text":"fix lat-lon behavior to limit to active memberships improve handling of retired legislators fix type of maximum_memberships bill version ordering is now consistent","title":"v2.1 (Feb 2019)"},{"location":"api-v2/changelog/#v20-january-2019","text":"bugfix for maximum_memberships type bugfix for versions field improve tests","title":"v2.0 (January 2019)"},{"location":"api-v2/changelog/#beta-release-november-2018","text":"API Keys are now required consider classification when using current_memberships fix geo filtering add openstatesUrl to Bill node for ease of linkage to OpenStates.org add Person.oldMemberships as analog to currentMemberships add actionSince filter to bills node fix 500 errors/optimization when using GraphQL fragments addition of basic protection for excessive queries add totalCount to assist in pagination add Organization.currentMemberships","title":"Beta Release (November 2018)"},{"location":"api-v2/changelog/#preview-release-1-may-2018","text":"fix for people pagination add updatedSince for people add sponsor argument for bills node allow votes to take pagination parameters allow traversing to votes from person","title":"Preview Release 1 (May 2018)"},{"location":"api-v2/changelog/#preview-release-0-dec-2017","text":"Initial draft release of the API, no backwards-compatibility guarantee made.","title":"Preview Release 0 (Dec 2017)"},{"location":"api-v2/examples/","text":"Examples \u00b6 Get basic information for all legislatures \u00b6 See in GraphiQL { jurisdictions { edges { node { name legislativeSessions { edges { node { name } } } legislature: organizations(classification: \"legislature\", first: 1) { edges { node { name classification children(first: 5) { edges { node { name classification } } } } } } } } } } Get overview of a legislature\\'s structure \u00b6 See in GraphiQL { jurisdiction(name: \"North Dakota\") { name url legislativeSessions { edges { node { name identifier } } } organizations(classification: \"legislature\", first: 1) { edges { node { id name children(first: 20) { edges { node { name } } } } } } } } Search for bills that match a given condition \u00b6 See in GraphiQL { search_1: bills(first: 5, jurisdiction: \"New York\", session: \"2017-2018\", chamber: \"lower\", classification: \"resolution\", updatedSince: \"2017-01-15\") { edges { node { id identifier title classification updatedAt createdAt legislativeSession { identifier jurisdiction { name } } actions { date description classification } documents { date note links { url } } versions { date note links { url } } sources { url note } } } } } Get all information on a particular bill \u00b6 See in GraphiQL { b1: bill(jurisdiction: \"Hawaii\", session: \"2017 Regular Session\", identifier: \"HB 475\") { id identifier title classification updatedAt createdAt legislativeSession { identifier jurisdiction { name } } actions { date description classification } documents { date note links { url } } versions { date note links { url } } sources { url note } } b2: bill(id: \"ocd-bill/9c24aaa2-6acc-43ad-883b-ae9f677062e9\") { id identifier title classification updatedAt createdAt legislativeSession { identifier jurisdiction { name } } actions { date description classification } documents { date note links { url } } versions { date note links { url } } sources { url note } } } Get information about a specific legislator \u00b6 See in GraphiQL { person(id:\"ocd-person/dd05bd23-fe49-4e65-bfff-62db997e56e0\"){ name contactDetails { note type value } otherNames { name } sources { url } currentMemberships { organization { name } } } } Get legislators for a given state/chamber \u00b6 See in GraphiQL ocd-organization/ddf820b5-5246-46b3-a807-99b5914ad39f is the id of the Alabama Senate chamber. { people(memberOf:\"ocd-organization/ddf820b5-5246-46b3-a807-99b5914ad39f\", first: 100) { edges { node { name party: currentMemberships(classification:\"party\") { organization { name } } links { url } sources { url } chamber: currentMemberships(classification:[\"upper\", \"lower\"]) { post { label } organization { name classification parent { name } } } } } } } Search for legislators that represent a given area \u00b6 See in GraphQL { people(latitude: 40.7460022, longitude: -73.9584642, first: 100) { edges { node { name chamber: currentMemberships(classification:[\"upper\", \"lower\"]) { post { label } organization { name classification parent { name } } } } } } }","title":"Examples"},{"location":"api-v2/examples/#examples","text":"","title":"Examples"},{"location":"api-v2/examples/#get-basic-information-for-all-legislatures","text":"See in GraphiQL { jurisdictions { edges { node { name legislativeSessions { edges { node { name } } } legislature: organizations(classification: \"legislature\", first: 1) { edges { node { name classification children(first: 5) { edges { node { name classification } } } } } } } } } }","title":"Get basic information for all legislatures"},{"location":"api-v2/examples/#get-overview-of-a-legislatures-structure","text":"See in GraphiQL { jurisdiction(name: \"North Dakota\") { name url legislativeSessions { edges { node { name identifier } } } organizations(classification: \"legislature\", first: 1) { edges { node { id name children(first: 20) { edges { node { name } } } } } } } }","title":"Get overview of a legislature\\'s structure"},{"location":"api-v2/examples/#search-for-bills-that-match-a-given-condition","text":"See in GraphiQL { search_1: bills(first: 5, jurisdiction: \"New York\", session: \"2017-2018\", chamber: \"lower\", classification: \"resolution\", updatedSince: \"2017-01-15\") { edges { node { id identifier title classification updatedAt createdAt legislativeSession { identifier jurisdiction { name } } actions { date description classification } documents { date note links { url } } versions { date note links { url } } sources { url note } } } } }","title":"Search for bills that match a given condition"},{"location":"api-v2/examples/#get-all-information-on-a-particular-bill","text":"See in GraphiQL { b1: bill(jurisdiction: \"Hawaii\", session: \"2017 Regular Session\", identifier: \"HB 475\") { id identifier title classification updatedAt createdAt legislativeSession { identifier jurisdiction { name } } actions { date description classification } documents { date note links { url } } versions { date note links { url } } sources { url note } } b2: bill(id: \"ocd-bill/9c24aaa2-6acc-43ad-883b-ae9f677062e9\") { id identifier title classification updatedAt createdAt legislativeSession { identifier jurisdiction { name } } actions { date description classification } documents { date note links { url } } versions { date note links { url } } sources { url note } } }","title":"Get all information on a particular bill"},{"location":"api-v2/examples/#get-information-about-a-specific-legislator","text":"See in GraphiQL { person(id:\"ocd-person/dd05bd23-fe49-4e65-bfff-62db997e56e0\"){ name contactDetails { note type value } otherNames { name } sources { url } currentMemberships { organization { name } } } }","title":"Get information about a specific legislator"},{"location":"api-v2/examples/#get-legislators-for-a-given-statechamber","text":"See in GraphiQL ocd-organization/ddf820b5-5246-46b3-a807-99b5914ad39f is the id of the Alabama Senate chamber. { people(memberOf:\"ocd-organization/ddf820b5-5246-46b3-a807-99b5914ad39f\", first: 100) { edges { node { name party: currentMemberships(classification:\"party\") { organization { name } } links { url } sources { url } chamber: currentMemberships(classification:[\"upper\", \"lower\"]) { post { label } organization { name classification parent { name } } } } } } }","title":"Get legislators for a given state/chamber"},{"location":"api-v2/examples/#search-for-legislators-that-represent-a-given-area","text":"See in GraphQL { people(latitude: 40.7460022, longitude: -73.9584642, first: 100) { edges { node { name chamber: currentMemberships(classification:[\"upper\", \"lower\"]) { post { label } organization { name classification parent { name } } } } } } }","title":"Search for legislators that represent a given area"},{"location":"api-v2/other/","text":"Other Notes \u00b6 There are a few other things to be aware of while using the API: Explore the Graph \u00b6 GraphQL is still quite new, so we figured it might be good to provide some helpful tips on how to think about the data and how you'll use the API. First, it is probably well worth your time to play around in GraphiQL to explore the API and data. It was heavily used when developing the API and writing tests, and is a very powerful tool, particularly when you make use of the self-documenting nature of the graph. When you're thinking about how to query don't necessarily try to replicate your old API calls exactly. For example, perhaps you were grabbing all bills that met a given criteria and then grabbing all sponsors contact details. This can now be done in one call by traversing from the bills-root root node into the BillSponsorshipNode and then up to the PersonNode and finally to the ContactDetailNode This may sound complex at first, but once you get the hang of it, it really does unlock a ton of power and will make your apps more powerful and efficient. Pagination \u00b6 In several places (such as the bills-root and BillNode 's votes ) we mention that nodes are paginated. What this means in practice is that instead of getting back the underlying node type, say BillNode , directly, you'll get back BillConnectionNode or similar. (In practice there are connection node types for each paginated type, but all work the same way in our case.) Arguments \u00b6 Each paginated endpoint accepts any of four parameters: first - given an integer N, only return the first N nodes last - given an integer N, only return the last N nodes after - combined with first , will return first N nodes after a given \"cursor\" before - combined with last , will return last N nodes before a given \"cursor\" So typically you'd paginate using first , obtaining a cursor, and then calling the API again with a combination of first and after . The same process could be carried out with last and before to paginate in reverse. Responses \u00b6 Let's take a look at everything that pagination makes available: { bills(first:20) { edges { node { title } cursor } pageInfo { hasNextPage hasPreviousPage endCursor startCursor } totalCount } } You'll see that the connection node has three nodes: edges , pageInfo , and totalCount edges - a list of objects that each have a node and cursor attribute: node - the underlying node type, in our case BillNode cursor - an opaque cursor for this particular item, it can be used with the before and after parameters each paginated node accepts as arguments. pageInfo - a list of helpful pieces of information about this page: hasNextPage - boolean that is true if there is another page after this hasPreviousPage - boolean that is true if there is a page before this endCursor - last cursor in the set of edges, can be used with after to paginate forward startCursor - first cursor in the set of edges, can be used with before to paginate backwards totalCount - total number of objects available from this connection In Practice \u00b6 Let's say you want to get all of the people matching a given criteria: You'd start with a query for all people matching your criteria, ensuring to set the page size to no greater than the maximum: { people(memberOf: \"Some Organization\", first: 100) { edges { node { name } } pageInfo { hasNextPage endCursor } } } Let's say we got back a list of 100 edges and our pageInfo object looked like: { \"hasNextPage\": true, \"endCursor\": \"ZXJyYXlxb20uZWN0aW9uOjA=\" } So you'd make another call: { people(memberOf: \"Some Organization\", first: 100, after:\"ZXJyYXlxb20uZWN0aW9uOjA=\" ) { edges { node { name } } pageInfo { hasNextPage endCursor } } } And let's say in this case you got back only 75 edges, and our pageInfo object looks like: { \"hasNextPage\": false, \"endCursor\": \"AXjYylxX2bu1wxa9uunnb=\" } We'd stop iteration at this point, of course, if hasNextPage had been true, we'd continue on until it wasn't. Renaming fields \u00b6 A really useful trick that is often overlooked is that you can rename fields when retrieving them, for example: { republicans: people(memberOf: \"Republican\", first: 5) { edges { node { full_name: name } } } } Would give back: { \"data\": { \"republicans\": { \"edges\": [ { \"node\": { \"full_name\": \"Michelle Udall\" } }, { \"node\": { \"full_name\": \"Kimberly Yee\" } }, { \"node\": { \"full_name\": \"Regina E. Cobb\" } }, { \"node\": { \"full_name\": \"Michelle B. Ugenti-Rita\" } }, { \"node\": { \"full_name\": \"David Livingston\" } } ] } } } Note that we're both renaming a top-level node here as well as a piece of data within the query. You can also use this to query the same root node twice (doing so without renaming isn't allowed since it results in a name conflict). For example: { republicans: people(memberOf: \"Republican\", first: 5) { edges { node { full_name: name } } } democrats: people(memberOf: \"Democratic\", first: 5) { edges { node { full_name: name } } } } Fuzzy Date Format {#date-format} \u00b6 Unless otherwise noted (most notably createdAt and updatedAt all date objects are \"fuzzy\". Instead of being expressed as an exact date, it is possible a given date takes any of the following formats: YYYY YYYY-MM YYYY-MM-DD YYYY-MM-DD HH:MM:SS (if times are allowed) Action/Vote times are all assumed to be in the state capitol's time zone. Times related to our updates such as updatedAt and createdAt are in UTC. Name Matching \u00b6 In several places such as bill sponsorships and votes you'll notice that we have a raw string representing a person or organization as well as a place for a link to the appropriate OrganizationNode or PersonNode . Because of the way we collect the data from states, we always collect the raw data and later make an attempt to (via a mix of automated matching and manual fixes) connect the reference with data we've already collected. In many cases these linkages will not be provided, but with some upcoming new tools to help us improve this matching we'll be able to dramatically improve the number of matched entities in the near future.","title":"Other Notes"},{"location":"api-v2/other/#other-notes","text":"There are a few other things to be aware of while using the API:","title":"Other Notes"},{"location":"api-v2/other/#explore-the-graph","text":"GraphQL is still quite new, so we figured it might be good to provide some helpful tips on how to think about the data and how you'll use the API. First, it is probably well worth your time to play around in GraphiQL to explore the API and data. It was heavily used when developing the API and writing tests, and is a very powerful tool, particularly when you make use of the self-documenting nature of the graph. When you're thinking about how to query don't necessarily try to replicate your old API calls exactly. For example, perhaps you were grabbing all bills that met a given criteria and then grabbing all sponsors contact details. This can now be done in one call by traversing from the bills-root root node into the BillSponsorshipNode and then up to the PersonNode and finally to the ContactDetailNode This may sound complex at first, but once you get the hang of it, it really does unlock a ton of power and will make your apps more powerful and efficient.","title":"Explore the Graph"},{"location":"api-v2/other/#pagination","text":"In several places (such as the bills-root and BillNode 's votes ) we mention that nodes are paginated. What this means in practice is that instead of getting back the underlying node type, say BillNode , directly, you'll get back BillConnectionNode or similar. (In practice there are connection node types for each paginated type, but all work the same way in our case.)","title":"Pagination"},{"location":"api-v2/other/#arguments","text":"Each paginated endpoint accepts any of four parameters: first - given an integer N, only return the first N nodes last - given an integer N, only return the last N nodes after - combined with first , will return first N nodes after a given \"cursor\" before - combined with last , will return last N nodes before a given \"cursor\" So typically you'd paginate using first , obtaining a cursor, and then calling the API again with a combination of first and after . The same process could be carried out with last and before to paginate in reverse.","title":"Arguments"},{"location":"api-v2/other/#responses","text":"Let's take a look at everything that pagination makes available: { bills(first:20) { edges { node { title } cursor } pageInfo { hasNextPage hasPreviousPage endCursor startCursor } totalCount } } You'll see that the connection node has three nodes: edges , pageInfo , and totalCount edges - a list of objects that each have a node and cursor attribute: node - the underlying node type, in our case BillNode cursor - an opaque cursor for this particular item, it can be used with the before and after parameters each paginated node accepts as arguments. pageInfo - a list of helpful pieces of information about this page: hasNextPage - boolean that is true if there is another page after this hasPreviousPage - boolean that is true if there is a page before this endCursor - last cursor in the set of edges, can be used with after to paginate forward startCursor - first cursor in the set of edges, can be used with before to paginate backwards totalCount - total number of objects available from this connection","title":"Responses"},{"location":"api-v2/other/#in-practice","text":"Let's say you want to get all of the people matching a given criteria: You'd start with a query for all people matching your criteria, ensuring to set the page size to no greater than the maximum: { people(memberOf: \"Some Organization\", first: 100) { edges { node { name } } pageInfo { hasNextPage endCursor } } } Let's say we got back a list of 100 edges and our pageInfo object looked like: { \"hasNextPage\": true, \"endCursor\": \"ZXJyYXlxb20uZWN0aW9uOjA=\" } So you'd make another call: { people(memberOf: \"Some Organization\", first: 100, after:\"ZXJyYXlxb20uZWN0aW9uOjA=\" ) { edges { node { name } } pageInfo { hasNextPage endCursor } } } And let's say in this case you got back only 75 edges, and our pageInfo object looks like: { \"hasNextPage\": false, \"endCursor\": \"AXjYylxX2bu1wxa9uunnb=\" } We'd stop iteration at this point, of course, if hasNextPage had been true, we'd continue on until it wasn't.","title":"In Practice"},{"location":"api-v2/other/#renaming-fields","text":"A really useful trick that is often overlooked is that you can rename fields when retrieving them, for example: { republicans: people(memberOf: \"Republican\", first: 5) { edges { node { full_name: name } } } } Would give back: { \"data\": { \"republicans\": { \"edges\": [ { \"node\": { \"full_name\": \"Michelle Udall\" } }, { \"node\": { \"full_name\": \"Kimberly Yee\" } }, { \"node\": { \"full_name\": \"Regina E. Cobb\" } }, { \"node\": { \"full_name\": \"Michelle B. Ugenti-Rita\" } }, { \"node\": { \"full_name\": \"David Livingston\" } } ] } } } Note that we're both renaming a top-level node here as well as a piece of data within the query. You can also use this to query the same root node twice (doing so without renaming isn't allowed since it results in a name conflict). For example: { republicans: people(memberOf: \"Republican\", first: 5) { edges { node { full_name: name } } } democrats: people(memberOf: \"Democratic\", first: 5) { edges { node { full_name: name } } } }","title":"Renaming fields"},{"location":"api-v2/other/#fuzzy-date-format-date-format","text":"Unless otherwise noted (most notably createdAt and updatedAt all date objects are \"fuzzy\". Instead of being expressed as an exact date, it is possible a given date takes any of the following formats: YYYY YYYY-MM YYYY-MM-DD YYYY-MM-DD HH:MM:SS (if times are allowed) Action/Vote times are all assumed to be in the state capitol's time zone. Times related to our updates such as updatedAt and createdAt are in UTC.","title":"Fuzzy Date Format {#date-format}"},{"location":"api-v2/other/#name-matching","text":"In several places such as bill sponsorships and votes you'll notice that we have a raw string representing a person or organization as well as a place for a link to the appropriate OrganizationNode or PersonNode . Because of the way we collect the data from states, we always collect the raw data and later make an attempt to (via a mix of automated matching and manual fixes) connect the reference with data we've already collected. In many cases these linkages will not be provided, but with some upcoming new tools to help us improve this matching we'll be able to dramatically improve the number of matched entities in the near future.","title":"Name Matching"},{"location":"api-v2/root-nodes/","text":"Root Nodes \u00b6 As seen in the introduction, when constructing a query you will start your query at one (or more) root nodes. The following root nodes are available: jurisdictions \u00b6 Get a list of all jurisdictions. This will return a list of JurisdictionNode objects, one for each state (plus Puerto Rico and DC). Pagination : This endpoint accepts the usual pagination parameters, but pagination is not required. people \u00b6 Get a list of all people matching certain criteria. This will return a list of PersonNode objects, one for each person matching your query. Pagination : This endpoint accepts the usual pagination parameters, and you must limit your results to no more than 100 using either the \"first\" or \"last\" parameter. Parameters \u00b6 name Limit response to people who's name contains the provided string. Includes partial matches & case-insensitive matches. memberOf Limit response to people that have a currently active membership record for an organization. The value passed to memberOf can be an ocd-organization ID or a name (e.g. 'Republican' or 'Nebraska Legislature'). everMemberOf Limit response to people that have any recorded membership record for an organization. Operates as a superset of memberOf. Specifying memberOf and everMemberOf in the same query is invalid. district When specifying either memberOf or everMemberOf, limits to people who's membership represented the district with a given label. (e.g. memberOf: \"Nebraska Legislature\", district: \"7\") Specifying district without memberOf or everMemberOf is invalid. latitude and longitude Limit to people that are currently representing the district(s) containing the point specified by the provided coordinates. Must be specified together. bills \u00b6 Get a list of all bills matching certain criteria. This will return a list of BillNode objects, one for each person matching your query. Pagination : This endpoint accepts the usual pagination parameters, and you must limit your results to no more than 100 using either the \"first\" or \"last\" parameter. Parameters \u00b6 jurisdiction Limit to bills associated with given jurisdiction, parameter can either be a human-readable jurisdiction name or an ocd-jurisdiction ID. chamber Limit to bills originating in a given chamber. (e.g. upper, lower, legislature) session Limit to bills originating in a given legislative session. This parameter should be the desired session's identifier . (See LegislativeSessionNode ). classification Limit to bills with a given classification (e.g. \"bill\" or \"resolution\") subject Limit to bills with a given subject (e.g. \"Agriculture\") searchQuery Limit to bills that contain a given term. (Experimental until 2020!) updatedSince Limit to bills that have had data updated since a given time (UTC). Time should be in the format YYYY-MM-DD[THH:MM:SS]. actionsSince Limit to bills that have had actions since a given time (UTC). Time should be in the format YYYY-MM-DD. jurisdiction \u00b6 Look up a single jurisdiction by name or ID. This will return a single JurisdictionNode object with the provided name or ID parameter. Parameters \u00b6 name The human-readable name of the jurisdiction, such as 'New Hampshire'. id The ocd-jurisdiction ID of the desired jurisdiction, such as 'ocd-jurisdiction/country:us/state:nh'. You are required to provide one of the two available parameters. person \u00b6 Look up a single person by ocd-person ID. This will return a single PersonNode by ID. Parameters \u00b6 id ocd-person ID for the desired individual. organization \u00b6 Look up a single organization by ocd-organization ID. This will return a single OrganizationNode by ID. Parameters \u00b6 id ocd-organization ID for the desired individual. bill \u00b6 Look up a single bill by ID, URL, or (jurisdiction, session, identifier) combo. This will return a single BillNode object with the specified bill. Parameters \u00b6 id The ocd-bill ID of the desired bill, such as 'ocd-jurisdiction/country:us/state:nh'. openstatesUrl The URL of the desired bill, such as ' https://openstates.org/nc/bills/2019/HB760/ '. jurisdiction , session , identifier Must be specified together to fully identify a bill. As is true elsewhere, jurisdiction may be specified by name (New Hampshire) or ocd-jurisdiction ID (ocd-jurisdiction/country:us/state:nh). Session is specified by legislative session identifier (e.g. 2018 or 49). Identifier is the exact identifier of the desired bill, such as \"HB 327\". You are required to provide one either id or the other parameters to fully specify a bill. Use bills if you are looking for something more broad.","title":"Root Nodes"},{"location":"api-v2/root-nodes/#root-nodes","text":"As seen in the introduction, when constructing a query you will start your query at one (or more) root nodes. The following root nodes are available:","title":"Root Nodes"},{"location":"api-v2/root-nodes/#jurisdictions","text":"Get a list of all jurisdictions. This will return a list of JurisdictionNode objects, one for each state (plus Puerto Rico and DC). Pagination : This endpoint accepts the usual pagination parameters, but pagination is not required.","title":"jurisdictions"},{"location":"api-v2/root-nodes/#people","text":"Get a list of all people matching certain criteria. This will return a list of PersonNode objects, one for each person matching your query. Pagination : This endpoint accepts the usual pagination parameters, and you must limit your results to no more than 100 using either the \"first\" or \"last\" parameter.","title":"people"},{"location":"api-v2/root-nodes/#parameters","text":"name Limit response to people who's name contains the provided string. Includes partial matches & case-insensitive matches. memberOf Limit response to people that have a currently active membership record for an organization. The value passed to memberOf can be an ocd-organization ID or a name (e.g. 'Republican' or 'Nebraska Legislature'). everMemberOf Limit response to people that have any recorded membership record for an organization. Operates as a superset of memberOf. Specifying memberOf and everMemberOf in the same query is invalid. district When specifying either memberOf or everMemberOf, limits to people who's membership represented the district with a given label. (e.g. memberOf: \"Nebraska Legislature\", district: \"7\") Specifying district without memberOf or everMemberOf is invalid. latitude and longitude Limit to people that are currently representing the district(s) containing the point specified by the provided coordinates. Must be specified together.","title":"Parameters"},{"location":"api-v2/root-nodes/#bills","text":"Get a list of all bills matching certain criteria. This will return a list of BillNode objects, one for each person matching your query. Pagination : This endpoint accepts the usual pagination parameters, and you must limit your results to no more than 100 using either the \"first\" or \"last\" parameter.","title":"bills"},{"location":"api-v2/root-nodes/#parameters_1","text":"jurisdiction Limit to bills associated with given jurisdiction, parameter can either be a human-readable jurisdiction name or an ocd-jurisdiction ID. chamber Limit to bills originating in a given chamber. (e.g. upper, lower, legislature) session Limit to bills originating in a given legislative session. This parameter should be the desired session's identifier . (See LegislativeSessionNode ). classification Limit to bills with a given classification (e.g. \"bill\" or \"resolution\") subject Limit to bills with a given subject (e.g. \"Agriculture\") searchQuery Limit to bills that contain a given term. (Experimental until 2020!) updatedSince Limit to bills that have had data updated since a given time (UTC). Time should be in the format YYYY-MM-DD[THH:MM:SS]. actionsSince Limit to bills that have had actions since a given time (UTC). Time should be in the format YYYY-MM-DD.","title":"Parameters"},{"location":"api-v2/root-nodes/#jurisdiction","text":"Look up a single jurisdiction by name or ID. This will return a single JurisdictionNode object with the provided name or ID parameter.","title":"jurisdiction"},{"location":"api-v2/root-nodes/#parameters_2","text":"name The human-readable name of the jurisdiction, such as 'New Hampshire'. id The ocd-jurisdiction ID of the desired jurisdiction, such as 'ocd-jurisdiction/country:us/state:nh'. You are required to provide one of the two available parameters.","title":"Parameters"},{"location":"api-v2/root-nodes/#person","text":"Look up a single person by ocd-person ID. This will return a single PersonNode by ID.","title":"person"},{"location":"api-v2/root-nodes/#parameters_3","text":"id ocd-person ID for the desired individual.","title":"Parameters"},{"location":"api-v2/root-nodes/#organization","text":"Look up a single organization by ocd-organization ID. This will return a single OrganizationNode by ID.","title":"organization"},{"location":"api-v2/root-nodes/#parameters_4","text":"id ocd-organization ID for the desired individual.","title":"Parameters"},{"location":"api-v2/root-nodes/#bill","text":"Look up a single bill by ID, URL, or (jurisdiction, session, identifier) combo. This will return a single BillNode object with the specified bill.","title":"bill"},{"location":"api-v2/root-nodes/#parameters_5","text":"id The ocd-bill ID of the desired bill, such as 'ocd-jurisdiction/country:us/state:nh'. openstatesUrl The URL of the desired bill, such as ' https://openstates.org/nc/bills/2019/HB760/ '. jurisdiction , session , identifier Must be specified together to fully identify a bill. As is true elsewhere, jurisdiction may be specified by name (New Hampshire) or ocd-jurisdiction ID (ocd-jurisdiction/country:us/state:nh). Session is specified by legislative session identifier (e.g. 2018 or 49). Identifier is the exact identifier of the desired bill, such as \"HB 327\". You are required to provide one either id or the other parameters to fully specify a bill. Use bills if you are looking for something more broad.","title":"Parameters"},{"location":"api-v2/types/","text":"Data Types \u00b6 Starting at the base nodes, data in the API is represented as interconnected nodes of various types. This page provides an overview of the nodes. Another good way to get acquainted with the layout is to use the GraphiQL browser (click Docs in the upper right corner). Jurisdictions & Sessions \u00b6 JurisdictionNode \u00b6 A Jurisdiction is the Open Civic Data term for the top level divisions of the US. Open States is comprised of 52 jurisdictions, one for each state, and two more for D.C. and Puerto Rico. Each JurisdictionNode has the following attributes & nodes available: id - ocd-jurisdiction identifier, these are permanent identifiers assigned to each Jurisdiction name - human-readable name for the jurisdiction (e.g. Kansas) url - URL of official website for jurisdiction featureFlags - reserved for future use legislativeSessions - Paginated list (see pagination ) of LegislativeSessionNode belonging to this jurisdiction's legislature. organizations - Paginated list of OrganizationNode belonging to this jurisdiction. it is also possible to filter the list of children using the classification parameter lastScrapedAt - Time when last scrape finished. See also: Open Civic Data Jurisdiction reference LegislativeSessionNode \u00b6 A legislative session is a convening of the legislature, either a primary or special session. Each LegislativeSessionNode has the following attributes and nodes available: jurisdiction - JurisdictionNode which this session belongs to. identifier - short identifier by which this session is referred to (e.g. 2017s1 or 121) name - formal name of session (e.g. \"2017 Special Session #1\" or \"121st Session\" classification - \"primary\" or \"special\" startDate - start date of session if known endDate - end date of session if known DivisionNode \u00b6 Divisions represent particular geopolitical boundaries. Divisions exist for states as well as their component districts and are tied closely to political geographies. id - Open Civic Data Division ID name - human-readable name for the division redirect - link to another DivisionNode, only present if division has been replaced country - country code (will be \"us\") for all Open States divisions createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object People & Organizations \u00b6 PersonNode \u00b6 People, typically legislators and their associated metadata. Note that most fields are optional beyond name as often we don't have a reliable given/family name or birthDate for instance. id - Open Civic Data Person ID name - primary name for the person sortName - alternate name to sort by (if known) familyName - hereditary name, essentially a \"last name\" (if known) givenName - essentially a \"first name\" (if known) image - full URL to official image of legislator birthDate - see date-format deathDate - see date-format identifiers - list of other known identifiers, IdentifierNode otherNames - list of other known names, NameNode links - official URLs relating to this person, LinkNode contactDetails - ways to contact this person (via email, phone, etc.), contactdetailnode currentMemberships - currently active memberships MembershipNode can be filtered with the classification parameter to only get memberships to certain types of OrganizationNode oldMemberships - inactive memberships MembershipNode can be filtered with the classification parameter to only get memberships to certain types of OrganizationNode sources - URLs which were used in compiling Open States' information on this subject, [LinkNode] createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Popolo's person Open Civic Data OCDEP 5 OrganizationNode \u00b6 Organizations that comprise the state legislatures and their associated metdata. A typical bicameral legislature is comprised of a top-level organization (classification=legislature), two chambers (classification=upper & lower), and any number of committees (classification=committee). Each Organization is comprised of the following attributes and nodes: id - Open Civic Data Organization ID name - primary name for the person image - full URL to official image for organization classification - the type of organization as described above foundingDate - see date-format dissolutionDate - see date-format parent - parent OrganizationNode if one exists children - paginated list of child OrganizationNode objects it is also possible to filter the list of children using the classification parameter currentMemberships - list of all current members of this Organization identifiers - list of other known identifiers for this organization, IdentifierNode otherNames - list of other known names for this organization, NameNode links - official URLs relating to this person, LinkNode sources - URLs which were used in compiling Open States' information on this subject, [LinkNode] createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Popolo's organization Open Civic Data OCDEP 5 MembershipNode \u00b6 A MembershipNode represents a connection between a personnode and a organizationnode . A membership may optionally also reference a particular postnode , such as a particular seat within a given chamber. Each membership has the following attributes and nodes: id - Open Civic Data Membership ID personName the raw name of the person that the membership describes (see name-matching person - personnode organization - organizationnode post - postnode label - label assigned to this membership role - role fulfilled by this membership startDate - start date of membership if known endDate - end date of membership if known createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Popolo's membership Open Civic Data OCDEP 5 PostNode \u00b6 A PostNode represents a given position within an organization. The most common example would be a seat such as Maryland's 4th House Seat. It is worth noting that some seats can have multiple active memberships at once, as noted in maximumMemberships . Each post has the following attributes and nodes: id - Open Civic Data Post ID label - label assigned to this post (e.g. 3) role - role fulfilled by this membership (e.g. 'member') division - related divisionnode if this role has a relevant division startDate - start date of membership if known endDate - end date of membership if known maximumMemberships - typically 1, but set higher in the case of multi-member districts createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Popolo's post Open Civic Data OCDEP 5 Bills & Votes \u00b6 BillNode \u00b6 A BillNode represents any legislative instrument such as a bill or resolution. Each node has the following attributes and nodes available: id - Internal ocd-bill identifier for this bill. legislativeSession - link to LegislativeSessionNode this bill is from identifier - primary identifier for this bill (e.g. HB 264) title - primary title for this bill fromOrganization - organization (typically upper or lower chamber) primarily associated with this bill classification - list of one or more bill types such as \"bill\" or \"resolution\" subject - list of zero or more subjects assigned by the state abstracts - list of abstracts provided by the state, BillAbstractNode otherTitles - list of other titles provided by the state, BillTitleNode otherIdentifiers - list of other identifiers provided by the state, BillIdentifierNode actions - list of actions (such as introduction, amendment, passage, etc.) that have been taken on the bill, BillActionNode sponsorships - list of bill sponsors, BillSponsorshipNode relatedBills - list of related bills as provided by the state, RelatedBillNode versions - list of bill versions as provided by the state, BillDocumentNode documents - list of related documents (e.g. legal analysis, fiscal notes, etc.) as provided by the state, BillDocumentNode votes - paginated list of VoteEventNode related to the bill sources - URLs which were used in compiling Open States' information on this subject, linknode openstatesUrl - URL to bill page on OpenStates.org createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object BillAbstractNode \u00b6 Represents an official abstract for a bill, each BillAbstractNode has the following attributes: abstract - the abstract itself note - optional note about origin/purpose of abstract date - optional date associated with abstract BillTitleNode \u00b6 Represents an alternate title for a bill, each BillTitleNode has the following attributes: title - the alternate title note - optional note about origin/purpose of this title BillIdentifierNode \u00b6 Represents an alternate identifier for a bill, each BillIdentifierNode has the following attributes: identifier - the alternate identifier scheme - a name for the identifier scheme note - optional note about origin/purpose of this identifier BillActionNode \u00b6 Represents an action taken on a bill, each BillActionNode has the following attributes and nodes: organization - OrganizationNode where this action originated, will typically be either upper or lower chamber, or perhaps legislature as a whole. description - text describing the action as provided by the jurisdiction. date - date action took place (see date-format ) classification - list of zero or more normalized action types (see action-categorization ) order - integer by which actions can be sorted, not intended for display purposes extras - JSON string providing extra information about this action vote - if there is a known associated vote, pointer to the relevant VoteEventNode relatedEntities - a list of RelatedEntityNode with known entities referenced in this action RelatedEntityNode \u00b6 Represents an entity that is related to a BillActionNode . name - raw (source-provided) name of entity entityType - either organization or person organization - if entityType is 'organization', the resolved OrganizationNode person - if entityType is 'person', the resolved PersonNode See name-matching for details on how name relates to organiation and person . BillSponsorshipNode \u00b6 Represents a sponsor of a bill. name - raw (source-provided) name of sponsoring person or organization entityType - either organization or person organization - if entityType is 'organization', the resolved OrganizationNode person - if entityType is 'person', the resolved PersonNode primary - boolean, true if sponsorship is considered by the jurisdiction to be \"primary\" (note: in many states multiple primary sponsors may exist) classification - jurisdiction-provided type of sponsorship, such as \"author\" or \"cosponsor\". These meanings typically vary across states, which is why we provide primary as a sort of indicator of the degree of sponsorship indicated. See name-matching for details on how name relates to organiation and person . RelatedBillNode \u00b6 Represents relationships between bills. identifier - identifier of related bill (e.g. SB 401) legislativeSession - identifier of related session (in same jurisdiction) relationType - type of relationship such as \"companion\", \"prior-session\", \"replaced-by\", or \"replaces\" relatedBill - if the related bill is found to exist in our data, link to the BillNode BillDocumentNode \u00b6 Representation of documents and versions on bills. A given document can have multiple links representing different manifestations (e.g. HTML, PDF, DOC) of the same content. note - note describing the purpose of the document or version (e.g. Final Printing) date - optional date associated with the document links - list of one or more MimetypeLinkNode with actual URLs to bills. MimetypeLinkNode \u00b6 Represents a single manifestation of a particular document. mediaType - media type (aka MIME type) such as application/pdf or text/html url - URL to official copy of the bill text - text describing this particular manifestation (e.g. PDF) VoteEventNode \u00b6 Represents a vote taken on a bill. id - Internal ocd-vote identifier for this bill. identifier - Identifier used by jurisdiction to uniquely identify the vote. motionText - Text of the motion being voted upon, such as \"motion to pass the bill as amended.\" motionClassification - List with zero or more classifications for this motion, such as \"passage\" or \"veto-override\" startDate - Date on which the vote took place. (see date-format result - Outcome of the vote, 'pass' or 'fail'. organization - Related OrganizationNode where vote took place. billAction - Optional linked BillActionNode . votes - List of PersonVoteNode for each individual's recorded vote. (May not be present depending on jurisdiction.) counts - List of VoteCountNode with sums of each outcome (e.g. yea/nay/abstain). sources - URLs which were used in compiling Open States' information on this subject, [LinkNode] createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Open Civic Data vote format . PersonVoteNode \u00b6 Represents an individual person's vote (e.g. yea or nay) on a given bill. option - Option chosen by this individual. (yea, nay, abstain, other, etc.) voterName - Raw name of voter as provided by jurisdiction. voter - Resolved PersonNode representing voter. (See name-matching note - Note attached to this vote, sometimes used for explaining an \"other\" vote. VoteCountNode \u00b6 Represents the sum of votes for a given option . option - Option in question. (yea, nay, abstain, other, etc.) value - Number of individuals voting this way. Other Nodes \u00b6 IdentifierNode \u00b6 Represents an alternate identifier, each with the following attributes: identifier - the alternate identifier scheme - a name for the identifier scheme NameNode \u00b6 Represents an alterante name, each with the following attributes: name - the alternate name note - note about usage/origin of this alternate name startDate - date at which this name began being valid (blank if unknown) endDate - date at which this name stopped being valid (blank if unknown or still active) LinkNode \u00b6 Represents a single link associated with a person or used as a source. url - URL text - text describing the use of this particular URL ContactDetailNode \u00b6 Used to represent a contact method for a given person. type - type of contact detail (e.g. voice, email, address, etc.) value - actual phone number, email address, etc. note - used to group contact data by location (e.g. Home Address, Office Address) label - human-readable label for this contact detail","title":"Data Types"},{"location":"api-v2/types/#data-types","text":"Starting at the base nodes, data in the API is represented as interconnected nodes of various types. This page provides an overview of the nodes. Another good way to get acquainted with the layout is to use the GraphiQL browser (click Docs in the upper right corner).","title":"Data Types"},{"location":"api-v2/types/#jurisdictions-sessions","text":"","title":"Jurisdictions & Sessions"},{"location":"api-v2/types/#jurisdictionnode","text":"A Jurisdiction is the Open Civic Data term for the top level divisions of the US. Open States is comprised of 52 jurisdictions, one for each state, and two more for D.C. and Puerto Rico. Each JurisdictionNode has the following attributes & nodes available: id - ocd-jurisdiction identifier, these are permanent identifiers assigned to each Jurisdiction name - human-readable name for the jurisdiction (e.g. Kansas) url - URL of official website for jurisdiction featureFlags - reserved for future use legislativeSessions - Paginated list (see pagination ) of LegislativeSessionNode belonging to this jurisdiction's legislature. organizations - Paginated list of OrganizationNode belonging to this jurisdiction. it is also possible to filter the list of children using the classification parameter lastScrapedAt - Time when last scrape finished. See also: Open Civic Data Jurisdiction reference","title":"JurisdictionNode"},{"location":"api-v2/types/#legislativesessionnode","text":"A legislative session is a convening of the legislature, either a primary or special session. Each LegislativeSessionNode has the following attributes and nodes available: jurisdiction - JurisdictionNode which this session belongs to. identifier - short identifier by which this session is referred to (e.g. 2017s1 or 121) name - formal name of session (e.g. \"2017 Special Session #1\" or \"121st Session\" classification - \"primary\" or \"special\" startDate - start date of session if known endDate - end date of session if known","title":"LegislativeSessionNode"},{"location":"api-v2/types/#divisionnode","text":"Divisions represent particular geopolitical boundaries. Divisions exist for states as well as their component districts and are tied closely to political geographies. id - Open Civic Data Division ID name - human-readable name for the division redirect - link to another DivisionNode, only present if division has been replaced country - country code (will be \"us\") for all Open States divisions createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object","title":"DivisionNode"},{"location":"api-v2/types/#people-organizations","text":"","title":"People & Organizations"},{"location":"api-v2/types/#personnode","text":"People, typically legislators and their associated metadata. Note that most fields are optional beyond name as often we don't have a reliable given/family name or birthDate for instance. id - Open Civic Data Person ID name - primary name for the person sortName - alternate name to sort by (if known) familyName - hereditary name, essentially a \"last name\" (if known) givenName - essentially a \"first name\" (if known) image - full URL to official image of legislator birthDate - see date-format deathDate - see date-format identifiers - list of other known identifiers, IdentifierNode otherNames - list of other known names, NameNode links - official URLs relating to this person, LinkNode contactDetails - ways to contact this person (via email, phone, etc.), contactdetailnode currentMemberships - currently active memberships MembershipNode can be filtered with the classification parameter to only get memberships to certain types of OrganizationNode oldMemberships - inactive memberships MembershipNode can be filtered with the classification parameter to only get memberships to certain types of OrganizationNode sources - URLs which were used in compiling Open States' information on this subject, [LinkNode] createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Popolo's person Open Civic Data OCDEP 5","title":"PersonNode"},{"location":"api-v2/types/#organizationnode","text":"Organizations that comprise the state legislatures and their associated metdata. A typical bicameral legislature is comprised of a top-level organization (classification=legislature), two chambers (classification=upper & lower), and any number of committees (classification=committee). Each Organization is comprised of the following attributes and nodes: id - Open Civic Data Organization ID name - primary name for the person image - full URL to official image for organization classification - the type of organization as described above foundingDate - see date-format dissolutionDate - see date-format parent - parent OrganizationNode if one exists children - paginated list of child OrganizationNode objects it is also possible to filter the list of children using the classification parameter currentMemberships - list of all current members of this Organization identifiers - list of other known identifiers for this organization, IdentifierNode otherNames - list of other known names for this organization, NameNode links - official URLs relating to this person, LinkNode sources - URLs which were used in compiling Open States' information on this subject, [LinkNode] createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Popolo's organization Open Civic Data OCDEP 5","title":"OrganizationNode"},{"location":"api-v2/types/#membershipnode","text":"A MembershipNode represents a connection between a personnode and a organizationnode . A membership may optionally also reference a particular postnode , such as a particular seat within a given chamber. Each membership has the following attributes and nodes: id - Open Civic Data Membership ID personName the raw name of the person that the membership describes (see name-matching person - personnode organization - organizationnode post - postnode label - label assigned to this membership role - role fulfilled by this membership startDate - start date of membership if known endDate - end date of membership if known createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Popolo's membership Open Civic Data OCDEP 5","title":"MembershipNode"},{"location":"api-v2/types/#postnode","text":"A PostNode represents a given position within an organization. The most common example would be a seat such as Maryland's 4th House Seat. It is worth noting that some seats can have multiple active memberships at once, as noted in maximumMemberships . Each post has the following attributes and nodes: id - Open Civic Data Post ID label - label assigned to this post (e.g. 3) role - role fulfilled by this membership (e.g. 'member') division - related divisionnode if this role has a relevant division startDate - start date of membership if known endDate - end date of membership if known maximumMemberships - typically 1, but set higher in the case of multi-member districts createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Popolo's post Open Civic Data OCDEP 5","title":"PostNode"},{"location":"api-v2/types/#bills-votes","text":"","title":"Bills & Votes"},{"location":"api-v2/types/#billnode","text":"A BillNode represents any legislative instrument such as a bill or resolution. Each node has the following attributes and nodes available: id - Internal ocd-bill identifier for this bill. legislativeSession - link to LegislativeSessionNode this bill is from identifier - primary identifier for this bill (e.g. HB 264) title - primary title for this bill fromOrganization - organization (typically upper or lower chamber) primarily associated with this bill classification - list of one or more bill types such as \"bill\" or \"resolution\" subject - list of zero or more subjects assigned by the state abstracts - list of abstracts provided by the state, BillAbstractNode otherTitles - list of other titles provided by the state, BillTitleNode otherIdentifiers - list of other identifiers provided by the state, BillIdentifierNode actions - list of actions (such as introduction, amendment, passage, etc.) that have been taken on the bill, BillActionNode sponsorships - list of bill sponsors, BillSponsorshipNode relatedBills - list of related bills as provided by the state, RelatedBillNode versions - list of bill versions as provided by the state, BillDocumentNode documents - list of related documents (e.g. legal analysis, fiscal notes, etc.) as provided by the state, BillDocumentNode votes - paginated list of VoteEventNode related to the bill sources - URLs which were used in compiling Open States' information on this subject, linknode openstatesUrl - URL to bill page on OpenStates.org createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object","title":"BillNode"},{"location":"api-v2/types/#billabstractnode","text":"Represents an official abstract for a bill, each BillAbstractNode has the following attributes: abstract - the abstract itself note - optional note about origin/purpose of abstract date - optional date associated with abstract","title":"BillAbstractNode"},{"location":"api-v2/types/#billtitlenode","text":"Represents an alternate title for a bill, each BillTitleNode has the following attributes: title - the alternate title note - optional note about origin/purpose of this title","title":"BillTitleNode"},{"location":"api-v2/types/#billidentifiernode","text":"Represents an alternate identifier for a bill, each BillIdentifierNode has the following attributes: identifier - the alternate identifier scheme - a name for the identifier scheme note - optional note about origin/purpose of this identifier","title":"BillIdentifierNode"},{"location":"api-v2/types/#billactionnode","text":"Represents an action taken on a bill, each BillActionNode has the following attributes and nodes: organization - OrganizationNode where this action originated, will typically be either upper or lower chamber, or perhaps legislature as a whole. description - text describing the action as provided by the jurisdiction. date - date action took place (see date-format ) classification - list of zero or more normalized action types (see action-categorization ) order - integer by which actions can be sorted, not intended for display purposes extras - JSON string providing extra information about this action vote - if there is a known associated vote, pointer to the relevant VoteEventNode relatedEntities - a list of RelatedEntityNode with known entities referenced in this action","title":"BillActionNode"},{"location":"api-v2/types/#relatedentitynode","text":"Represents an entity that is related to a BillActionNode . name - raw (source-provided) name of entity entityType - either organization or person organization - if entityType is 'organization', the resolved OrganizationNode person - if entityType is 'person', the resolved PersonNode See name-matching for details on how name relates to organiation and person .","title":"RelatedEntityNode"},{"location":"api-v2/types/#billsponsorshipnode","text":"Represents a sponsor of a bill. name - raw (source-provided) name of sponsoring person or organization entityType - either organization or person organization - if entityType is 'organization', the resolved OrganizationNode person - if entityType is 'person', the resolved PersonNode primary - boolean, true if sponsorship is considered by the jurisdiction to be \"primary\" (note: in many states multiple primary sponsors may exist) classification - jurisdiction-provided type of sponsorship, such as \"author\" or \"cosponsor\". These meanings typically vary across states, which is why we provide primary as a sort of indicator of the degree of sponsorship indicated. See name-matching for details on how name relates to organiation and person .","title":"BillSponsorshipNode"},{"location":"api-v2/types/#relatedbillnode","text":"Represents relationships between bills. identifier - identifier of related bill (e.g. SB 401) legislativeSession - identifier of related session (in same jurisdiction) relationType - type of relationship such as \"companion\", \"prior-session\", \"replaced-by\", or \"replaces\" relatedBill - if the related bill is found to exist in our data, link to the BillNode","title":"RelatedBillNode"},{"location":"api-v2/types/#billdocumentnode","text":"Representation of documents and versions on bills. A given document can have multiple links representing different manifestations (e.g. HTML, PDF, DOC) of the same content. note - note describing the purpose of the document or version (e.g. Final Printing) date - optional date associated with the document links - list of one or more MimetypeLinkNode with actual URLs to bills.","title":"BillDocumentNode"},{"location":"api-v2/types/#mimetypelinknode","text":"Represents a single manifestation of a particular document. mediaType - media type (aka MIME type) such as application/pdf or text/html url - URL to official copy of the bill text - text describing this particular manifestation (e.g. PDF)","title":"MimetypeLinkNode"},{"location":"api-v2/types/#voteeventnode","text":"Represents a vote taken on a bill. id - Internal ocd-vote identifier for this bill. identifier - Identifier used by jurisdiction to uniquely identify the vote. motionText - Text of the motion being voted upon, such as \"motion to pass the bill as amended.\" motionClassification - List with zero or more classifications for this motion, such as \"passage\" or \"veto-override\" startDate - Date on which the vote took place. (see date-format result - Outcome of the vote, 'pass' or 'fail'. organization - Related OrganizationNode where vote took place. billAction - Optional linked BillActionNode . votes - List of PersonVoteNode for each individual's recorded vote. (May not be present depending on jurisdiction.) counts - List of VoteCountNode with sums of each outcome (e.g. yea/nay/abstain). sources - URLs which were used in compiling Open States' information on this subject, [LinkNode] createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Open Civic Data vote format .","title":"VoteEventNode"},{"location":"api-v2/types/#personvotenode","text":"Represents an individual person's vote (e.g. yea or nay) on a given bill. option - Option chosen by this individual. (yea, nay, abstain, other, etc.) voterName - Raw name of voter as provided by jurisdiction. voter - Resolved PersonNode representing voter. (See name-matching note - Note attached to this vote, sometimes used for explaining an \"other\" vote.","title":"PersonVoteNode"},{"location":"api-v2/types/#votecountnode","text":"Represents the sum of votes for a given option . option - Option in question. (yea, nay, abstain, other, etc.) value - Number of individuals voting this way.","title":"VoteCountNode"},{"location":"api-v2/types/#other-nodes","text":"","title":"Other Nodes"},{"location":"api-v2/types/#identifiernode","text":"Represents an alternate identifier, each with the following attributes: identifier - the alternate identifier scheme - a name for the identifier scheme","title":"IdentifierNode"},{"location":"api-v2/types/#namenode","text":"Represents an alterante name, each with the following attributes: name - the alternate name note - note about usage/origin of this alternate name startDate - date at which this name began being valid (blank if unknown) endDate - date at which this name stopped being valid (blank if unknown or still active)","title":"NameNode"},{"location":"api-v2/types/#linknode","text":"Represents a single link associated with a person or used as a source. url - URL text - text describing the use of this particular URL","title":"LinkNode"},{"location":"api-v2/types/#contactdetailnode","text":"Used to represent a contact method for a given person. type - type of contact detail (e.g. voice, email, address, etc.) value - actual phone number, email address, etc. note - used to group contact data by location (e.g. Home Address, Office Address) label - human-readable label for this contact detail","title":"ContactDetailNode"},{"location":"api-v3/","text":"API v3 Overview \u00b6 Open States provides a JSON API that can be used to programatically access state legislative information. API Basics \u00b6 The root URL for the API is https://v3.openstates.org/ . API keys are required. You can register for an API key and once activated, you\\'ll pass your API key via the X-API-KEY header or ?apikey query parameter. Auto-generated interactive documentation is available at either: https://v3.openstates.org/docs/ https://v3.openstates.org/redoc/ (whichever you prefer) Issues should be filed at our issue tracker . You can also check out our introductory blog post for more details. Methods \u00b6 Method Description Interactive Docs /jurisdictions Get list of available jurisdictions. try it! /jurisdictions/{jurisdiction_id} Get detailed metadata for a particular jurisdiction. try it! /people List or search people (legislators, governors, etc.) try it! /people.geo Get legislators for a given location. try it! /bills Search bills by various criteria. try it! /bills/ocd-bill/{uuid} Get bill by internal ID. try it! /bills/{jurisdiction}/{session}/{id} Get bill by jurisdiction, session, and ID. try it! /committees Get list of committees by jurisdiction. try it! /committees/{committee_id} Get details on committee by internal ID. try it! /events Get list of events by jurisdiction. try it! /events/{event_id} Get details on event by internal ID. try it! Concepts \u00b6 Jurisdiction The fundamental unit by which data is partitioned is the \\'jurisdiction.\\' If you are just interested in states you can consider the words synonymous for the most part. Jurisdictions include states, DC & Puerto Rico, and municipal governments for which we have limited support. Person A legislator, governor, mayor, etc. Each person possibly has a number of roles, at most one of which is considered \\'current.\\' Bill A proposed piece of legislation, encompasses bills, resolutions, constitutional amendments, etc. A given bill may have any number of votes, sponsorships, actions, etc.","title":"API v3 Overview"},{"location":"api-v3/#api-v3-overview","text":"Open States provides a JSON API that can be used to programatically access state legislative information.","title":"API v3 Overview"},{"location":"api-v3/#api-basics","text":"The root URL for the API is https://v3.openstates.org/ . API keys are required. You can register for an API key and once activated, you\\'ll pass your API key via the X-API-KEY header or ?apikey query parameter. Auto-generated interactive documentation is available at either: https://v3.openstates.org/docs/ https://v3.openstates.org/redoc/ (whichever you prefer) Issues should be filed at our issue tracker . You can also check out our introductory blog post for more details.","title":"API Basics"},{"location":"api-v3/#methods","text":"Method Description Interactive Docs /jurisdictions Get list of available jurisdictions. try it! /jurisdictions/{jurisdiction_id} Get detailed metadata for a particular jurisdiction. try it! /people List or search people (legislators, governors, etc.) try it! /people.geo Get legislators for a given location. try it! /bills Search bills by various criteria. try it! /bills/ocd-bill/{uuid} Get bill by internal ID. try it! /bills/{jurisdiction}/{session}/{id} Get bill by jurisdiction, session, and ID. try it! /committees Get list of committees by jurisdiction. try it! /committees/{committee_id} Get details on committee by internal ID. try it! /events Get list of events by jurisdiction. try it! /events/{event_id} Get details on event by internal ID. try it!","title":"Methods"},{"location":"api-v3/#concepts","text":"Jurisdiction The fundamental unit by which data is partitioned is the \\'jurisdiction.\\' If you are just interested in states you can consider the words synonymous for the most part. Jurisdictions include states, DC & Puerto Rico, and municipal governments for which we have limited support. Person A legislator, governor, mayor, etc. Each person possibly has a number of roles, at most one of which is considered \\'current.\\' Bill A proposed piece of legislation, encompasses bills, resolutions, constitutional amendments, etc. A given bill may have any number of votes, sponsorships, actions, etc.","title":"Concepts"},{"location":"api-v3/changelog/","text":"Changelog \u00b6 2021.11.12 \u00b6 add LegislativeSession.downloads 2021.10.27 \u00b6 add Bill.related_bills include 2021.10.12 \u00b6 added classification field on Person.offices Person.offices now always have all fields present, even if empty 2021.09.24 \u00b6 added experimental event endpoints 2021.08.30 \u00b6 added identifiers parameter to bill search 2021.08.03 \u00b6 added latest_bill_update & latest_people_update to API v3 2021.08.02 \u00b6 addded experimental committee endpoints 2021.04.19 \u00b6 experimental support for Jurisdiction.latest_runs include parameter 2021.03.24 \u00b6 Jurisdiction classification can now be \\'country\\' experimental US federal support 2021.02.23 \u00b6 people district= parameter added 2020.12.21 \u00b6 people name= parameter is now fuzzy-searched, matching API v1 and v2 2020.10.30 \u00b6 move email to a top-level field on Person responses add consistent post ordering 2020.10.21 \u00b6 fix datetime handling for updated_since and created_since filters add list of divisions when including Jurisdiction.organizations 2020.10.13 \u00b6 add updated_asc sort option add rate limiting bugfix for New York jurisdiction lookup (openstates/issues#136) 2020.09.28 \u00b6 set permissive CORS settings bills endpoint updates: added created_since filter, thanks to Donald Wasserman! added sponsor and sponsor_classification filters added sort parameter added useful error message when searching /bills by session without jurisdiction restored missing Bill.from_organization field introduced new fields: Person.openstates_url, Bill.openstates_url 2020.09.14 \u00b6 removed some unused fields from responses removed deprecated government classification from Jurisdiction 2020.09.10 \u00b6 added Jurisdiction.legislative_sessions corrected initial pagination limits for release 2020.09.09 \u00b6 Initial beta release.","title":"Changelog"},{"location":"api-v3/changelog/#changelog","text":"","title":"Changelog"},{"location":"api-v3/changelog/#20211112","text":"add LegislativeSession.downloads","title":"2021.11.12"},{"location":"api-v3/changelog/#20211027","text":"add Bill.related_bills include","title":"2021.10.27"},{"location":"api-v3/changelog/#20211012","text":"added classification field on Person.offices Person.offices now always have all fields present, even if empty","title":"2021.10.12"},{"location":"api-v3/changelog/#20210924","text":"added experimental event endpoints","title":"2021.09.24"},{"location":"api-v3/changelog/#20210830","text":"added identifiers parameter to bill search","title":"2021.08.30"},{"location":"api-v3/changelog/#20210803","text":"added latest_bill_update & latest_people_update to API v3","title":"2021.08.03"},{"location":"api-v3/changelog/#20210802","text":"addded experimental committee endpoints","title":"2021.08.02"},{"location":"api-v3/changelog/#20210419","text":"experimental support for Jurisdiction.latest_runs include parameter","title":"2021.04.19"},{"location":"api-v3/changelog/#20210324","text":"Jurisdiction classification can now be \\'country\\' experimental US federal support","title":"2021.03.24"},{"location":"api-v3/changelog/#20210223","text":"people district= parameter added","title":"2021.02.23"},{"location":"api-v3/changelog/#20201221","text":"people name= parameter is now fuzzy-searched, matching API v1 and v2","title":"2020.12.21"},{"location":"api-v3/changelog/#20201030","text":"move email to a top-level field on Person responses add consistent post ordering","title":"2020.10.30"},{"location":"api-v3/changelog/#20201021","text":"fix datetime handling for updated_since and created_since filters add list of divisions when including Jurisdiction.organizations","title":"2020.10.21"},{"location":"api-v3/changelog/#20201013","text":"add updated_asc sort option add rate limiting bugfix for New York jurisdiction lookup (openstates/issues#136)","title":"2020.10.13"},{"location":"api-v3/changelog/#20200928","text":"set permissive CORS settings bills endpoint updates: added created_since filter, thanks to Donald Wasserman! added sponsor and sponsor_classification filters added sort parameter added useful error message when searching /bills by session without jurisdiction restored missing Bill.from_organization field introduced new fields: Person.openstates_url, Bill.openstates_url","title":"2020.09.28"},{"location":"api-v3/changelog/#20200914","text":"removed some unused fields from responses removed deprecated government classification from Jurisdiction","title":"2020.09.14"},{"location":"api-v3/changelog/#20200910","text":"added Jurisdiction.legislative_sessions corrected initial pagination limits for release","title":"2020.09.10"},{"location":"api-v3/changelog/#20200909","text":"Initial beta release.","title":"2020.09.09"},{"location":"contributing/","text":"Getting Started \u00b6 We're glad to have you joining us, taking a few minutes to read the following pages will help you be a better member of our community: Our Code of Conduct is important to us, and helps us maintain a healthy community. We also have a guide to help you learn where to get help that you should look over. If you are new to open source, or unfamiliar to contributing to open source projects, it might be beneficial to read this guide . We welcome and value all contributions to Open States, including (but not limited to!) code contributions. This guide assumes a basic familiarity with using the command line , git , and Python. If you are unfamiliar with something that is mentioned in this guide, we encourage you to read the linked resources. No matter how experienced you are, it is a good idea to read through this section before diving into Open States' code. No worries if you aren't an expert though, we'll walk you through the steps. And as for Python, if you've written other languages like Javascript or Ruby you'll probably be just fine. Here's a great guide on getting started installing and managing Python versions. Don't be afraid to ask for help either! Project Overview \u00b6 Open States is a fairly large & somewhat complex project comprised of many moving parts with a long history. As you look to contribute, it may be beneficial to understand a little bit about the various components. These repositories make up the core of the project, if you're looking to contribute there's a 95% chance one of these is what you want. openstates-scrapers - Open States' scrapers. All code related to getting information from a website and storing it in the Open States database lives here. What is a scraper? people - Open States people & committee data, maintained as editable YAML files. openstates-core - Open States data model & scraper backend. openstates.org - Powers OpenStates.org website & GraphQL API. api-v3 - Powers API v3 . documentation - you're reading it now . Installing Prerequisites \u00b6 poetry \u00b6 If you're working on the people repo, api-v3 , or want to work on scrapers without Docker , you'll need poetry to build your Python virtual environment . Note If you haven't used poetry before, it is similar to pipenv , pip , and conda in that it manages a Python virtualenv on your behalf. Installing Poetry The official poetry docs recommend installing with: curl -sSL https://install.python-poetry.org | python3 - Then within each repo you check out, be sure to run: poetry install Which will fetch the correct version of dependencies. docker & docker-compose \u00b6 When working on scrapers or openstates.org, you have the option to use Docker. The first thing you will need to do is get a working docker environment on your local machine. We'll do this using Docker. No worries if you aren't familiar with Docker, you'll barely have to touch it beyond what this guide explains. Install Docker and docker-compose (if not already installed on your local system): (a) Installing Docker: On OSX: Docker for Mac On Windows: Docker for Windows On Linux: Use your package manager of choice or follow Docker's instructions . ( Docker Compose is probably already installed by step 1(a) if not, proceed to step 1(b) ) (b) Installing docker-compose: For easy installation on macOS, Windows, and 64-bit Linux. Ensure that Docker and docker-compose are installed locally: $ docker --version Docker version 19.03.4, build 9013bf5 $ docker-compose --version docker-compose version 1.24.1, build 4667896b Of course, your versions will differ, but ensure they are relatively recent to avoid strange issues. pre-commit \u00b6 To help keep the code as managable as possible we strongly recommend you use pre-commit to make sure all commits adhere to our preferred style. See pre-commit's installation instructions Within each repo you check out, run pre-commit install after checking out. It should look something like: $ pre-commit install pre-commit installed at .git/hooks/pre-commit Note If you're running flake8 and black yourself via your editor or similar this isn't strictly necessary, but we find it helps ensure commits don't fail linting. We require all PRs to pass linting! What is linting? Recent Major Work \u00b6 To give a sense of recent priorities, here are major milestones from the past few years: Federal Data & Committee Data - 2021 API v3 - Q3 2020 Legislation Tracking - Q1 2020 Restoration of Historical Legislator Data - Q4 2019 Full Text Search - Q4 2019 2019 Legislative Session Updates - Q1 2019 OpenStates.org 2019 rewrite - Q1 2019 OpenStates GraphQL API - Q4 2018 Scraper Overhaul - Throughout much of 2017 we reworked our scrapers to be more resilient and to use an updated tech stack, replacing the one that powered the site from 2011-2016.","title":"Getting Started"},{"location":"contributing/#getting-started","text":"We're glad to have you joining us, taking a few minutes to read the following pages will help you be a better member of our community: Our Code of Conduct is important to us, and helps us maintain a healthy community. We also have a guide to help you learn where to get help that you should look over. If you are new to open source, or unfamiliar to contributing to open source projects, it might be beneficial to read this guide . We welcome and value all contributions to Open States, including (but not limited to!) code contributions. This guide assumes a basic familiarity with using the command line , git , and Python. If you are unfamiliar with something that is mentioned in this guide, we encourage you to read the linked resources. No matter how experienced you are, it is a good idea to read through this section before diving into Open States' code. No worries if you aren't an expert though, we'll walk you through the steps. And as for Python, if you've written other languages like Javascript or Ruby you'll probably be just fine. Here's a great guide on getting started installing and managing Python versions. Don't be afraid to ask for help either!","title":"Getting Started"},{"location":"contributing/#project-overview","text":"Open States is a fairly large & somewhat complex project comprised of many moving parts with a long history. As you look to contribute, it may be beneficial to understand a little bit about the various components. These repositories make up the core of the project, if you're looking to contribute there's a 95% chance one of these is what you want. openstates-scrapers - Open States' scrapers. All code related to getting information from a website and storing it in the Open States database lives here. What is a scraper? people - Open States people & committee data, maintained as editable YAML files. openstates-core - Open States data model & scraper backend. openstates.org - Powers OpenStates.org website & GraphQL API. api-v3 - Powers API v3 . documentation - you're reading it now .","title":"Project Overview"},{"location":"contributing/#installing-prerequisites","text":"","title":"Installing Prerequisites"},{"location":"contributing/#poetry","text":"If you're working on the people repo, api-v3 , or want to work on scrapers without Docker , you'll need poetry to build your Python virtual environment . Note If you haven't used poetry before, it is similar to pipenv , pip , and conda in that it manages a Python virtualenv on your behalf. Installing Poetry The official poetry docs recommend installing with: curl -sSL https://install.python-poetry.org | python3 - Then within each repo you check out, be sure to run: poetry install Which will fetch the correct version of dependencies.","title":"poetry"},{"location":"contributing/#docker-docker-compose","text":"When working on scrapers or openstates.org, you have the option to use Docker. The first thing you will need to do is get a working docker environment on your local machine. We'll do this using Docker. No worries if you aren't familiar with Docker, you'll barely have to touch it beyond what this guide explains. Install Docker and docker-compose (if not already installed on your local system): (a) Installing Docker: On OSX: Docker for Mac On Windows: Docker for Windows On Linux: Use your package manager of choice or follow Docker's instructions . ( Docker Compose is probably already installed by step 1(a) if not, proceed to step 1(b) ) (b) Installing docker-compose: For easy installation on macOS, Windows, and 64-bit Linux. Ensure that Docker and docker-compose are installed locally: $ docker --version Docker version 19.03.4, build 9013bf5 $ docker-compose --version docker-compose version 1.24.1, build 4667896b Of course, your versions will differ, but ensure they are relatively recent to avoid strange issues.","title":"docker & docker-compose"},{"location":"contributing/#pre-commit","text":"To help keep the code as managable as possible we strongly recommend you use pre-commit to make sure all commits adhere to our preferred style. See pre-commit's installation instructions Within each repo you check out, run pre-commit install after checking out. It should look something like: $ pre-commit install pre-commit installed at .git/hooks/pre-commit Note If you're running flake8 and black yourself via your editor or similar this isn't strictly necessary, but we find it helps ensure commits don't fail linting. We require all PRs to pass linting! What is linting?","title":"pre-commit"},{"location":"contributing/#recent-major-work","text":"To give a sense of recent priorities, here are major milestones from the past few years: Federal Data & Committee Data - 2021 API v3 - Q3 2020 Legislation Tracking - Q1 2020 Restoration of Historical Legislator Data - Q4 2019 Full Text Search - Q4 2019 2019 Legislative Session Updates - Q1 2019 OpenStates.org 2019 rewrite - Q1 2019 OpenStates GraphQL API - Q4 2018 Scraper Overhaul - Throughout much of 2017 we reworked our scrapers to be more resilient and to use an updated tech stack, replacing the one that powered the site from 2011-2016.","title":"Recent Major Work"},{"location":"contributing/documentation/","text":"Documentation \u00b6 If you notice any issues on these docs, or just want to help improve them the source is in the openstates/documentation repository. Checking out \u00b6 Fork and clone the documentation repository: Visit https://github.com/openstates/documentation and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/documentation.git Cloning into 'documentation'.. Building Docs Locally \u00b6 Step 1) Install poetry if you haven't already ( https://python-poetry.org/docs/ ) Step 2) Run poetry install to build virtualenv. Step 3) Run poetry run mkdocs serve to preview changes in browser.","title":"Documentation"},{"location":"contributing/documentation/#documentation","text":"If you notice any issues on these docs, or just want to help improve them the source is in the openstates/documentation repository.","title":"Documentation"},{"location":"contributing/documentation/#checking-out","text":"Fork and clone the documentation repository: Visit https://github.com/openstates/documentation and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/documentation.git Cloning into 'documentation'..","title":"Checking out"},{"location":"contributing/documentation/#building-docs-locally","text":"Step 1) Install poetry if you haven't already ( https://python-poetry.org/docs/ ) Step 2) Run poetry install to build virtualenv. Step 3) Run poetry run mkdocs serve to preview changes in browser.","title":"Building Docs Locally"},{"location":"contributing/local-database/","text":"Running a Local Database \u00b6 If you want to ensure your scraped data imports or work on OpenStates.org or API v3, you'll need a local database. This can be a bit cumbersome, since running Postgres locally varies a lot platform-to-platform, and you'll need to populate it as well. If you're comfortable with Postgres, most of these steps can be easily modified to use your own Postgres instance, but for the remainder of this guide we'll be using a dockerized postgres image. Prerequisites \u00b6 Be sure you've already installed docker and docker-compose , as noted in Installing Prerequisites . You'll need openstates-scrapers checked out, even if you aren't working on scrapers. This repository has the docker-compose.yml config and initialization scripts for the database. If you want to initialize the database for openstates.org work you'll need that project checked out as well. Initialize Database For Scraping \u00b6 Run init-db.sh from within the openstates-scrapers directory: Warning If you've already run this before, running scripts/init-db.sh will reset your database to scratch! openstates-scrapers/$ ./scripts/init-db.sh + unset DATABASE_URL + docker-compose down Removing scrapers_db_1 ... done Removing network openstates-network + docker volume rm openstates-postgres openstates-postgres + docker-compose up -d db Creating network \"openstates-network\" with the default driver Creating volume \"openstates-postgres\" with default driver Creating scrapers_db_1 ... done + sleep 3 + DATABASE_URL=postgis://openstates:openstates@db/openstatesorg + docker-compose run --rm --entrypoint 'poetry run os-initdb' scrape Creating scrapers_scrape_run ... done Operations to perform: Apply all migrations: contenttypes, data Running migrations: Applying contenttypes.0001_initial... OK Applying contenttypes.0002_remove_content_type_name... OK Applying data.0001_initial... OK Applying data.0002_auto_20200422_0028... OK Applying data.0003_auto_20200422_0031... OK ...TRUNCATED... loading WY loading DC loading PR loading US This will populate your database with the tables needed for scraping, as well as some basic static data such as the jurisdiction metadata. If all you want to do is run scrapers and import data into a database for inspection, you're good to go! Initialize Database for OpenStates.org \u00b6 Note You must run openstates-scrapers ' init-db.sh as shown above first! From within the openstates.org directory run docker/init-db.sh : openstates.org/$ ./docker/init-db.sh + unset DATABASE_URL + docker-compose run --rm -e PYTHONPATH=docker/ --entrypoint 'poetry run ./manage.py migrate' django Creating openstatesorg_django_run ... done Operations to perform: Apply all migrations: account, admin, auth, bulk, bundles, contenttypes, dashboards, data, people_admin, profiles, sessions, sites, socialaccount Running migrations: Applying auth.0001_initial... OK ...TRUNCATED... docker-compose run --rm -e PYTHONPATH=docker/ --entrypoint 'poetry run ./manage.py shell -c \"import testdata\"' django This creates the Django-specific tables, and also creates a local API key testkey that can be used for local development. Working with the Local Database \u00b6 The database will persist to disk, so for the most part once you run these steps you're good to go. Starting the Database \u00b6 You'll need to make sure that database is running whenever you're working on scrapers or OpenStates.org locally. You can do that by running docker-compose up -d db from the openstates-scrapers directory. openstates-scrapers$ docker-compose up -d db Starting scrapers_db_1 ... done If it is already running output will look like: openstates-scrapers$ docker-compose up -d db scrapers_db_1 is up-to-date Stopping the Database \u00b6 openstates-scrapers$ docker-compose stop db Stopping scrapers_db_1 ... done Resetting the Database \u00b6 You can always run scripts/init-db.sh to reset your database. This can be good if you have some bad data, or just whenever you'd like a fresh start: openstates-scrapers/$ ./scripts/init-db.sh","title":"Running a Local Database"},{"location":"contributing/local-database/#running-a-local-database","text":"If you want to ensure your scraped data imports or work on OpenStates.org or API v3, you'll need a local database. This can be a bit cumbersome, since running Postgres locally varies a lot platform-to-platform, and you'll need to populate it as well. If you're comfortable with Postgres, most of these steps can be easily modified to use your own Postgres instance, but for the remainder of this guide we'll be using a dockerized postgres image.","title":"Running a Local Database"},{"location":"contributing/local-database/#prerequisites","text":"Be sure you've already installed docker and docker-compose , as noted in Installing Prerequisites . You'll need openstates-scrapers checked out, even if you aren't working on scrapers. This repository has the docker-compose.yml config and initialization scripts for the database. If you want to initialize the database for openstates.org work you'll need that project checked out as well.","title":"Prerequisites"},{"location":"contributing/local-database/#initialize-database-for-scraping","text":"Run init-db.sh from within the openstates-scrapers directory: Warning If you've already run this before, running scripts/init-db.sh will reset your database to scratch! openstates-scrapers/$ ./scripts/init-db.sh + unset DATABASE_URL + docker-compose down Removing scrapers_db_1 ... done Removing network openstates-network + docker volume rm openstates-postgres openstates-postgres + docker-compose up -d db Creating network \"openstates-network\" with the default driver Creating volume \"openstates-postgres\" with default driver Creating scrapers_db_1 ... done + sleep 3 + DATABASE_URL=postgis://openstates:openstates@db/openstatesorg + docker-compose run --rm --entrypoint 'poetry run os-initdb' scrape Creating scrapers_scrape_run ... done Operations to perform: Apply all migrations: contenttypes, data Running migrations: Applying contenttypes.0001_initial... OK Applying contenttypes.0002_remove_content_type_name... OK Applying data.0001_initial... OK Applying data.0002_auto_20200422_0028... OK Applying data.0003_auto_20200422_0031... OK ...TRUNCATED... loading WY loading DC loading PR loading US This will populate your database with the tables needed for scraping, as well as some basic static data such as the jurisdiction metadata. If all you want to do is run scrapers and import data into a database for inspection, you're good to go!","title":"Initialize Database For Scraping"},{"location":"contributing/local-database/#initialize-database-for-openstatesorg","text":"Note You must run openstates-scrapers ' init-db.sh as shown above first! From within the openstates.org directory run docker/init-db.sh : openstates.org/$ ./docker/init-db.sh + unset DATABASE_URL + docker-compose run --rm -e PYTHONPATH=docker/ --entrypoint 'poetry run ./manage.py migrate' django Creating openstatesorg_django_run ... done Operations to perform: Apply all migrations: account, admin, auth, bulk, bundles, contenttypes, dashboards, data, people_admin, profiles, sessions, sites, socialaccount Running migrations: Applying auth.0001_initial... OK ...TRUNCATED... docker-compose run --rm -e PYTHONPATH=docker/ --entrypoint 'poetry run ./manage.py shell -c \"import testdata\"' django This creates the Django-specific tables, and also creates a local API key testkey that can be used for local development.","title":"Initialize Database for OpenStates.org"},{"location":"contributing/local-database/#working-with-the-local-database","text":"The database will persist to disk, so for the most part once you run these steps you're good to go.","title":"Working with the Local Database"},{"location":"contributing/local-database/#starting-the-database","text":"You'll need to make sure that database is running whenever you're working on scrapers or OpenStates.org locally. You can do that by running docker-compose up -d db from the openstates-scrapers directory. openstates-scrapers$ docker-compose up -d db Starting scrapers_db_1 ... done If it is already running output will look like: openstates-scrapers$ docker-compose up -d db scrapers_db_1 is up-to-date","title":"Starting the Database"},{"location":"contributing/local-database/#stopping-the-database","text":"openstates-scrapers$ docker-compose stop db Stopping scrapers_db_1 ... done","title":"Stopping the Database"},{"location":"contributing/local-database/#resetting-the-database","text":"You can always run scripts/init-db.sh to reset your database. This can be good if you have some bad data, or just whenever you'd like a fresh start: openstates-scrapers/$ ./scripts/init-db.sh","title":"Resetting the Database"},{"location":"contributing/openstates-org/","text":"Working On openstates.org \u00b6 openstates.org is the public-facing result of all the work we do. The site is built in Django and includes the web frontend and API. Checking out \u00b6 Fork and clone the openstates.org repository: Visit https://github.com/openstates/openstates.org and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/openstates.org.git Cloning into 'openstates.org'... Be sure to run poetry install to fetch the correct version of dependencies. And remember to install pre-commit : $ pre-commit install pre-commit installed at .git/hooks/pre-commit Getting a working database \u00b6 See Running a Local Database to get your database ready for OpenStates.org. Running Tests \u00b6 You can run the tests for the project via: ./docker/run-tests.sh You can also append standard pytest arguments such as -x to bail on first failure. Example of running just the graphapi tests, bailing on error: ./docker/run-tests.sh graphapi -x Repository overview \u00b6 The project is rather large, with quite a few django apps, here's a quick guide: Django Apps: bulk/ - handles bulk downloads on the website dashboards/ - dashboards for viewing various statistics geo/ - geography services for legislator lookup graphapi/ - powers GraphQL API profiles/ - user and subscription management public/ - public-facing pages (bulk of the site) utils/ - utilities shared by the other applications Other Stuff: ansible/ - the files used to deploy OpenStates.org are here docker/ - special scripts for running tests, etc. within docker openstates/ - core Django settings files static/ - various static assets, including frontend code templates/ - Django templates Running openstates.org \u00b6 Simply running docker-compose up should start django & the database, then browse to http://localhost:8000 and you'll be looking at your own local copy of openstates.org. In a separate terminal window, run npm run build and npm run start to see the site's react and style components. Note If you're running into issues with models not being found or an incorrectly configured virtual environment, running docker-compose build should help to fix it. If you have issues getting your instance up and running, please document the errors you're seeing and reach out . Running outside of Docker \u00b6 It might be desirable to test outside of docker sometimes to bypass caching or other issues that make development within the docker environment difficult. If so, you can install goreman (or any foreman clone) and run goreman start .","title":"Working On openstates.org"},{"location":"contributing/openstates-org/#working-on-openstatesorg","text":"openstates.org is the public-facing result of all the work we do. The site is built in Django and includes the web frontend and API.","title":"Working On openstates.org"},{"location":"contributing/openstates-org/#checking-out","text":"Fork and clone the openstates.org repository: Visit https://github.com/openstates/openstates.org and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/openstates.org.git Cloning into 'openstates.org'... Be sure to run poetry install to fetch the correct version of dependencies. And remember to install pre-commit : $ pre-commit install pre-commit installed at .git/hooks/pre-commit","title":"Checking out"},{"location":"contributing/openstates-org/#getting-a-working-database","text":"See Running a Local Database to get your database ready for OpenStates.org.","title":"Getting a working database"},{"location":"contributing/openstates-org/#running-tests","text":"You can run the tests for the project via: ./docker/run-tests.sh You can also append standard pytest arguments such as -x to bail on first failure. Example of running just the graphapi tests, bailing on error: ./docker/run-tests.sh graphapi -x","title":"Running Tests"},{"location":"contributing/openstates-org/#repository-overview","text":"The project is rather large, with quite a few django apps, here's a quick guide: Django Apps: bulk/ - handles bulk downloads on the website dashboards/ - dashboards for viewing various statistics geo/ - geography services for legislator lookup graphapi/ - powers GraphQL API profiles/ - user and subscription management public/ - public-facing pages (bulk of the site) utils/ - utilities shared by the other applications Other Stuff: ansible/ - the files used to deploy OpenStates.org are here docker/ - special scripts for running tests, etc. within docker openstates/ - core Django settings files static/ - various static assets, including frontend code templates/ - Django templates","title":"Repository overview"},{"location":"contributing/openstates-org/#running-openstatesorg","text":"Simply running docker-compose up should start django & the database, then browse to http://localhost:8000 and you'll be looking at your own local copy of openstates.org. In a separate terminal window, run npm run build and npm run start to see the site's react and style components. Note If you're running into issues with models not being found or an incorrectly configured virtual environment, running docker-compose build should help to fix it. If you have issues getting your instance up and running, please document the errors you're seeing and reach out .","title":"Running openstates.org"},{"location":"contributing/openstates-org/#running-outside-of-docker","text":"It might be desirable to test outside of docker sometimes to bypass caching or other issues that make development within the docker environment difficult. If so, you can install goreman (or any foreman clone) and run goreman start .","title":"Running outside of Docker"},{"location":"contributing/people/","text":"Contributing People Data \u00b6 Person data is maintained in the openstates/people repository. This repository contains YAML files with all the information on given individuals and committees. Info Please note that this portion of the project is in the public domain in the United States with all copyright waived via a CC0 dedication. By contributing you agree to waive all copyright claims. Checking out \u00b6 Fork and clone the people repository: Visit https://github.com/openstates/people and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/people.git Cloning into 'people'.. Build the environment with poetry : $ poetry install Installing dependencies from lock file ... And remember to install pre-commit hooks: $ pre-commit install pre-commit installed at .git/hooks/pre-commit Repository overview \u00b6 The repository consists of a few key components: settings.yml Settings for state legislatures, including the number of seats, and current vacancies. data/ Data files in YAML format on legislators, organized by state & status. You can use the os-people and os-committees commands to manage the data: poetry run os-people --help or poetry run os-committees --help Common tasks \u00b6 Updating legislator data by hand \u00b6 Let's say you call a legislator and find out that they have a new phone number, contribute back! See schema.md for details on the acceptable fields. If you're looking to add a lot of data but unsure where it fits feel free to ask via an issue and we can either amend the schema or make a recommendation. Start a new branch for this work Make the edits you need in the appropriate YAML file. Please keep edits to a minimum (e.g. don't re-order fields) Submit a PR, please describe how you came across this information to expedite review. Retiring a legislator \u00b6 Start a new branch for this work Add an end_date to their most recent role within the appropriate legislator's YAML file Run poetry run os-people retire with the appropriate legislator file(s) from the root directory Review the automatically edited files & submit a PR. Updating an entire state's legislators via a scrape \u00b6 Let's say a North Carolina has had an election & it makes sense to re-scrape everything for that state. Start a new branch for this work (e.g. nc-2021-people-update ) Scrape data using Open States' Scrapers Run poetry run os-people merge nc scrapes/2021-01-01/001 against the generated JSON data from the scrape Manually reconcile remaining changes, will often require some retirements as well. Check that data looks clean with poetry run os-people lint nc --summary commit your changes and prepare a PR. Example of the process: (In this example, we assume the people repo is stored at ~/gitroot/people and the openstates-scrapers repo is stored at ~/gitroot/openstates-scrapers ) :#~/gitroot/openstates-scrapers$ poetry run spatula scrape scrapers_next.de.people.House INFO:scrapers_next.de.people.House:fetching https://legis.delaware.gov/json/House/GetRepresentatives INFO:scrapers_next.de.people.LegDetail:fetching https://legis.delaware.gov/LegislatorDetail?personId = 13589 INFO:scrapers_next.de.people.LegDetail:fetching https://legis.delaware.gov/LegislatorDetail?personId = 332 ... success: wrote 41 objects to _scrapes/2022-06-17/001 :#~/gitroot/openstates-scrapers$ OS_PEOPLE_DIRECTORY = ~/gitroot/people poetry run os-people merge de _scrapes/2022-06-17/001/ analyzing 120 existing people and 41 scraped perfect match perfect match perfect match other_names: append { 'start_date' : '' , 'end_date' : '' , 'name' : 'Gerald L. Brady' } name: Gerald L. Brady = > Charles \"Bud\" M. Freel email: Gerald.Brady@delaware.gov = > Bud.Freel@delaware.gov offices changed from: Capitol Office address = 411 Legislative Ave. Dover, DE 19901 voice = 302 -744-4351 to Capitol Office address = 411 Legislative Avenue Dover, DE 19901 voice = 302 -744-4351 links: append { 'url' : 'https://legis.delaware.gov/LegislatorDetail?personId=24197' , 'note' : 'homepage' } sources: append { 'url' : 'https://legis.delaware.gov/LegislatorDetail?personId=24197' , 'note' : '' } ( m ) erge? ( r ) etire Gerald L. Brady? ( s ) kip? ( a ) bort? Aborted! This example stops at step #4 because the final steps all require manual work. We can use --retirement to automatically retire members during this process, but this requires knowing that all potentially retired members happened at the same time.","title":"Contributing People Data"},{"location":"contributing/people/#contributing-people-data","text":"Person data is maintained in the openstates/people repository. This repository contains YAML files with all the information on given individuals and committees. Info Please note that this portion of the project is in the public domain in the United States with all copyright waived via a CC0 dedication. By contributing you agree to waive all copyright claims.","title":"Contributing People Data"},{"location":"contributing/people/#checking-out","text":"Fork and clone the people repository: Visit https://github.com/openstates/people and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/people.git Cloning into 'people'.. Build the environment with poetry : $ poetry install Installing dependencies from lock file ... And remember to install pre-commit hooks: $ pre-commit install pre-commit installed at .git/hooks/pre-commit","title":"Checking out"},{"location":"contributing/people/#repository-overview","text":"The repository consists of a few key components: settings.yml Settings for state legislatures, including the number of seats, and current vacancies. data/ Data files in YAML format on legislators, organized by state & status. You can use the os-people and os-committees commands to manage the data: poetry run os-people --help or poetry run os-committees --help","title":"Repository overview"},{"location":"contributing/people/#common-tasks","text":"","title":"Common tasks"},{"location":"contributing/people/#updating-legislator-data-by-hand","text":"Let's say you call a legislator and find out that they have a new phone number, contribute back! See schema.md for details on the acceptable fields. If you're looking to add a lot of data but unsure where it fits feel free to ask via an issue and we can either amend the schema or make a recommendation. Start a new branch for this work Make the edits you need in the appropriate YAML file. Please keep edits to a minimum (e.g. don't re-order fields) Submit a PR, please describe how you came across this information to expedite review.","title":"Updating legislator data by hand"},{"location":"contributing/people/#retiring-a-legislator","text":"Start a new branch for this work Add an end_date to their most recent role within the appropriate legislator's YAML file Run poetry run os-people retire with the appropriate legislator file(s) from the root directory Review the automatically edited files & submit a PR.","title":"Retiring a legislator"},{"location":"contributing/people/#updating-an-entire-states-legislators-via-a-scrape","text":"Let's say a North Carolina has had an election & it makes sense to re-scrape everything for that state. Start a new branch for this work (e.g. nc-2021-people-update ) Scrape data using Open States' Scrapers Run poetry run os-people merge nc scrapes/2021-01-01/001 against the generated JSON data from the scrape Manually reconcile remaining changes, will often require some retirements as well. Check that data looks clean with poetry run os-people lint nc --summary commit your changes and prepare a PR. Example of the process: (In this example, we assume the people repo is stored at ~/gitroot/people and the openstates-scrapers repo is stored at ~/gitroot/openstates-scrapers ) :#~/gitroot/openstates-scrapers$ poetry run spatula scrape scrapers_next.de.people.House INFO:scrapers_next.de.people.House:fetching https://legis.delaware.gov/json/House/GetRepresentatives INFO:scrapers_next.de.people.LegDetail:fetching https://legis.delaware.gov/LegislatorDetail?personId = 13589 INFO:scrapers_next.de.people.LegDetail:fetching https://legis.delaware.gov/LegislatorDetail?personId = 332 ... success: wrote 41 objects to _scrapes/2022-06-17/001 :#~/gitroot/openstates-scrapers$ OS_PEOPLE_DIRECTORY = ~/gitroot/people poetry run os-people merge de _scrapes/2022-06-17/001/ analyzing 120 existing people and 41 scraped perfect match perfect match perfect match other_names: append { 'start_date' : '' , 'end_date' : '' , 'name' : 'Gerald L. Brady' } name: Gerald L. Brady = > Charles \"Bud\" M. Freel email: Gerald.Brady@delaware.gov = > Bud.Freel@delaware.gov offices changed from: Capitol Office address = 411 Legislative Ave. Dover, DE 19901 voice = 302 -744-4351 to Capitol Office address = 411 Legislative Avenue Dover, DE 19901 voice = 302 -744-4351 links: append { 'url' : 'https://legis.delaware.gov/LegislatorDetail?personId=24197' , 'note' : 'homepage' } sources: append { 'url' : 'https://legis.delaware.gov/LegislatorDetail?personId=24197' , 'note' : '' } ( m ) erge? ( r ) etire Gerald L. Brady? ( s ) kip? ( a ) bort? Aborted! This example stops at step #4 because the final steps all require manual work. We can use --retirement to automatically retire members during this process, but this requires knowing that all potentially retired members happened at the same time.","title":"Updating an entire state's legislators via a scrape"},{"location":"contributing/scrapers/","text":"Contributing to Scrapers \u00b6 Scrapers are at the core of what Open States does, each state requires several custom scrapers designed to extract bills, legislators, committees, and votes from the state website. All together there are around 200 scrapers, each one essentially independent, which means that there is always more work to do, but fortunately plenty of prior work to learn from. Checking Out \u00b6 Fork and clone the main scraper repository: Visit https://github.com/openstates/openstates-scrapers and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/openstates-scrapers.git Cloning into 'openstates-scrapers'... And remember to install pre-commit hooks: $ pre-commit install pre-commit installed at .git/hooks/pre-commit Be sure to run poetry install to fetch the correct version of dependencies. Warning Before cloning on a Windows computer, you will need to disable line-ending conversion. git config --global core.autocrlf false After cloning and entering the repo, you'll likely want to set global line-ending conversion back to true, and set local conversion to false. Repository Overview \u00b6 At this point you'll have a local openstates-scrapers directory. Within it, you'll find a directory called scrapers , lets take a look at it: $ ls scrapers __init__.py dc in mn nj pr va ak de ks mo nm ri vi al fl ky ms nv sc vt ar ga la mt ny sd wa az hi ma nc oh tn wi ca ia md nd ok tx wv co id me ne or ut wy ct il mi nh pa utils This directory has 50+ python modules, one for each state. Let's look inside one: $ ls scrapers/nc __init__.py bills.py votes.py Some states' directories will differ a bit, but all will have __init__.py and bills.py . The __init__.py file for each state has basic metadata on the state including a list of sessions. Other files contain the scrapers, typically named bills , votes , etc. At the root, you'll also find a directory called scrapers_next . This directory also has python modules for each state. Inside a state, you'll find people and potentially committee scrapers written using spatula . The plan is to port all scrapers to this framework and have scrapers_next replace the scraper directory. Running Your First Scraper \u00b6 Let's run your state's bills scraper (substitute your state for 'nc' below) : $ docker-compose run --rm scrape nc bills --fastmode --scrape The parameters you pass after docker-compose run --rm scrape are passed to os-update . Here we're saying that we're running NC's scrapers, and that we want to do it in \"fast mode\". By default, os-update imports results into a postgres database; the --scrape flag skips that step. The following arguments are optional: To bring up a list of the optional arguments in the CLI use -h , --help \\ -h , --help show this help message and exit\\ --debug open debugger on error\\ --loglevel {LOGLEVEL} set log level. options are: DEBUG|INFO|WARNING|ERROR|CRITICAL (default is INFO) \\ --scrape only run scrape post-scrape step\\ --import only run import post-scrape step\\ --nonstrict skip validation on save\\ --fastmode use cache and turn off throttling\\ --datadir {SCRAPED_DATA_DIR} data directory\\ --cachedir {CACHE_DIR} cache directory\\ -r {SCRAPELIB_RPM} scraper rpm\\ --rpm {SCRAPELIB_RPM} scraper rpm\\ --timeout {SCRAPELIB_TIMEOUT} scraper timeout\\ --no-verify skip tls verification\\ --retries {SCRAPELIB_RETRIES} scraper retries\\ --retry_wait {SCRAPELIB_RETRY_WAIT_SECONDS} scraper retry wait\\ --realtime loads bills in realtime to database, this requires configuring an AWS S3 bucket and using the lambda function: openstates-realtime You'll see the run plan , which is what the update aims to capture; in this case we're scraping the state website's data into JSON files: nc (scrape) bills: {} Then legislative posts and organizations get created, which is mostly boilerplate: 08:46:35 INFO openstates: save jurisdiction North Carolina as jurisdiction_ocd-jurisdiction-country:us-state:nc-government.json 08:46:35 INFO openstates: save organization North Carolina General Assembly as organization_01d6327c-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save organization Executive Office of the Governor as organization_01d63560-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save organization Senate as organization_01d636e6-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 1 as post_01d63a06-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 2 as post_01d63b96-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 3 as post_01d63cea-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 4 as post_01d63e34-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 5 as post_01d63f74-72d2-11e7-8df8-0242ac130003.json And then the actual data scraping begins, defaulting to the most recent legislative session: 08:46:36 INFO openstates: no session specified, using 2017 08:46:36 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/SimpleBillInquiry/displaybills.pl?Session=2017&tab=Chamber&Chamber=Senate 08:46:38 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S1 08:46:39 INFO openstates: save bill SR 1 in 2017 as bill_03c7edb4-72d2-11e7-8df8-0242ac130003.json 08:46:39 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S2 08:46:39 INFO openstates: save bill SJR 2 in 2017 as bill_044a5fc4-72d2-11e7-8df8-0242ac130003.json 08:46:39 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S3 08:46:40 INFO openstates: save bill SB 3 in 2017 as bill_04e8c66e-72d2-11e7-8df8-0242ac130003.json 08:46:40 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S4 08:46:41 INFO openstates: save bill SB 4 in 2017 as bill_05781f08-72d2-11e7-8df8-0242ac130003.json 08:46:41 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S5 Depending on the scraper you run, this part takes a while. Some scrapers can take hours to run depending on the number of bills and speed of the state's website. Note It is often desirable to bail out of running the whole scrape (Ctrl-C) after it has gotten a bit of data, instead of letting it run the entire scrape. To review the data you just fetched, you can browse the _data/nc/ directory and inspect the JSON files. If you're trying to make a small fix this is often sufficient, you can confirm that the scraped data looks correct and move on. Note It is of course possible that the scrape fails. If so, there's a good chance that isn't your fault, especially if it starts to run and then errors out. Scrapers do break, and there's no guarantee North Carolina didn't change their legislator page yesterday, breaking our tutorial here. If that's the case and you think the issue is with the scraper, feel free to get in touch with us or file an issue . At this point you're ready to run scrapers and contribute fixes. Hop onto our GitHub ticket queue , pick an issue to solve, and then submit a Pull Request! Importing Data \u00b6 Optionally, if you'd like to see how your scraped data imports into the database, perhaps to diagnose an issue that is happening after the scrape, pop over to getting a working database to see how to get a local database that you can import data into. Once that's done, make sure that the db image from openstates.org is running: $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 27fe691ad7c5 mdillon/postgis:11-alpine \"docker-entrypoint.s\u2026\" 3 hours ago Up 3 hours 0.0.0.0:5405->5432/tcp openstatesorg_db_1 Your output will vary, but if you don't see something named openstatesorg_db running you should run this command (from the openstates.org directory, not your scraper directory): $ docker-compose up -d db Now, when you want to run imports, you can drop the --scrape portion of the command you've been running. Or if you just want to test the import of already scraped data you can replace it with --import . An import looks something like this: $ docker-compose run --rm scrape fl bills --fast ... (truncated) ... 23:03:34 ERROR openstates: cannot resolve pseudo id to Person: ~{\u201cname\u201d: \u201cGrant, M.\u201c} 23:03:36 ERROR openstates: cannot resolve pseudo id to Person: ~{\u201cname\u201d: \u201cRodrigues, R.\u201c} fl (import) bills: {} import: bill: 0 new 0 updated 2620 noop jurisdiction: 0 new 0 updated 1 noop vote_event: 21 new 12 updated 533 noop The errors about unresolved psuedo-ids can safely be ignored, as long as you see the final run report the data you scraped is available in your database. The number of objects of each type that were created & updated are available for spot checking, as well as the total number of items that were seen that already exactly matched what was in the database. These can be useful stats as you try to see if your local changes to a scraper have the impact you expect. Running Spatula Scrapers \u00b6 Let's run a people scraper: $ poetry run spatula scrape scrapers_next.nc.people.SenList The command to run these scrapers is structured differently, as the parameters are set by giving the exact location of the function you want to run: directory.state.file.function . Note Function names do vary and scrapes for legislators are commonly split by chamber, so make sure to check you're passing the right function in your command. The actual data scraping should look something like: INFO:scrapers_next.nc.people.SenList:fetching https://www.ncleg.gov/Members/MemberTable/S INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/430 INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/431 INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/432 INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/433 INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/434 To review the data you scraped, you can inspect the JSON files in the dated directory within _scrapes/ . Each time you run a scrape, a new numbered folder will be within the dated directory, so you can compare older data to new easily. Note If a scrape fails, it's likely an issue with the scraper. Feel free to get in touch with us or file an issue . Spatula is incredibly powerful with lots of flexibility and useful CLI commands that are worth checking out as well. At this point you're ready to run spatula scrapers and contribute fixes. Hop onto our GitHub ticket queue , pick an issue to solve, and then submit a Pull Request!","title":"Contributing to Scrapers"},{"location":"contributing/scrapers/#contributing-to-scrapers","text":"Scrapers are at the core of what Open States does, each state requires several custom scrapers designed to extract bills, legislators, committees, and votes from the state website. All together there are around 200 scrapers, each one essentially independent, which means that there is always more work to do, but fortunately plenty of prior work to learn from.","title":"Contributing to Scrapers"},{"location":"contributing/scrapers/#checking-out","text":"Fork and clone the main scraper repository: Visit https://github.com/openstates/openstates-scrapers and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/openstates-scrapers.git Cloning into 'openstates-scrapers'... And remember to install pre-commit hooks: $ pre-commit install pre-commit installed at .git/hooks/pre-commit Be sure to run poetry install to fetch the correct version of dependencies. Warning Before cloning on a Windows computer, you will need to disable line-ending conversion. git config --global core.autocrlf false After cloning and entering the repo, you'll likely want to set global line-ending conversion back to true, and set local conversion to false.","title":"Checking Out"},{"location":"contributing/scrapers/#repository-overview","text":"At this point you'll have a local openstates-scrapers directory. Within it, you'll find a directory called scrapers , lets take a look at it: $ ls scrapers __init__.py dc in mn nj pr va ak de ks mo nm ri vi al fl ky ms nv sc vt ar ga la mt ny sd wa az hi ma nc oh tn wi ca ia md nd ok tx wv co id me ne or ut wy ct il mi nh pa utils This directory has 50+ python modules, one for each state. Let's look inside one: $ ls scrapers/nc __init__.py bills.py votes.py Some states' directories will differ a bit, but all will have __init__.py and bills.py . The __init__.py file for each state has basic metadata on the state including a list of sessions. Other files contain the scrapers, typically named bills , votes , etc. At the root, you'll also find a directory called scrapers_next . This directory also has python modules for each state. Inside a state, you'll find people and potentially committee scrapers written using spatula . The plan is to port all scrapers to this framework and have scrapers_next replace the scraper directory.","title":"Repository Overview"},{"location":"contributing/scrapers/#running-your-first-scraper","text":"Let's run your state's bills scraper (substitute your state for 'nc' below) : $ docker-compose run --rm scrape nc bills --fastmode --scrape The parameters you pass after docker-compose run --rm scrape are passed to os-update . Here we're saying that we're running NC's scrapers, and that we want to do it in \"fast mode\". By default, os-update imports results into a postgres database; the --scrape flag skips that step. The following arguments are optional: To bring up a list of the optional arguments in the CLI use -h , --help \\ -h , --help show this help message and exit\\ --debug open debugger on error\\ --loglevel {LOGLEVEL} set log level. options are: DEBUG|INFO|WARNING|ERROR|CRITICAL (default is INFO) \\ --scrape only run scrape post-scrape step\\ --import only run import post-scrape step\\ --nonstrict skip validation on save\\ --fastmode use cache and turn off throttling\\ --datadir {SCRAPED_DATA_DIR} data directory\\ --cachedir {CACHE_DIR} cache directory\\ -r {SCRAPELIB_RPM} scraper rpm\\ --rpm {SCRAPELIB_RPM} scraper rpm\\ --timeout {SCRAPELIB_TIMEOUT} scraper timeout\\ --no-verify skip tls verification\\ --retries {SCRAPELIB_RETRIES} scraper retries\\ --retry_wait {SCRAPELIB_RETRY_WAIT_SECONDS} scraper retry wait\\ --realtime loads bills in realtime to database, this requires configuring an AWS S3 bucket and using the lambda function: openstates-realtime You'll see the run plan , which is what the update aims to capture; in this case we're scraping the state website's data into JSON files: nc (scrape) bills: {} Then legislative posts and organizations get created, which is mostly boilerplate: 08:46:35 INFO openstates: save jurisdiction North Carolina as jurisdiction_ocd-jurisdiction-country:us-state:nc-government.json 08:46:35 INFO openstates: save organization North Carolina General Assembly as organization_01d6327c-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save organization Executive Office of the Governor as organization_01d63560-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save organization Senate as organization_01d636e6-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 1 as post_01d63a06-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 2 as post_01d63b96-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 3 as post_01d63cea-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 4 as post_01d63e34-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 5 as post_01d63f74-72d2-11e7-8df8-0242ac130003.json And then the actual data scraping begins, defaulting to the most recent legislative session: 08:46:36 INFO openstates: no session specified, using 2017 08:46:36 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/SimpleBillInquiry/displaybills.pl?Session=2017&tab=Chamber&Chamber=Senate 08:46:38 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S1 08:46:39 INFO openstates: save bill SR 1 in 2017 as bill_03c7edb4-72d2-11e7-8df8-0242ac130003.json 08:46:39 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S2 08:46:39 INFO openstates: save bill SJR 2 in 2017 as bill_044a5fc4-72d2-11e7-8df8-0242ac130003.json 08:46:39 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S3 08:46:40 INFO openstates: save bill SB 3 in 2017 as bill_04e8c66e-72d2-11e7-8df8-0242ac130003.json 08:46:40 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S4 08:46:41 INFO openstates: save bill SB 4 in 2017 as bill_05781f08-72d2-11e7-8df8-0242ac130003.json 08:46:41 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S5 Depending on the scraper you run, this part takes a while. Some scrapers can take hours to run depending on the number of bills and speed of the state's website. Note It is often desirable to bail out of running the whole scrape (Ctrl-C) after it has gotten a bit of data, instead of letting it run the entire scrape. To review the data you just fetched, you can browse the _data/nc/ directory and inspect the JSON files. If you're trying to make a small fix this is often sufficient, you can confirm that the scraped data looks correct and move on. Note It is of course possible that the scrape fails. If so, there's a good chance that isn't your fault, especially if it starts to run and then errors out. Scrapers do break, and there's no guarantee North Carolina didn't change their legislator page yesterday, breaking our tutorial here. If that's the case and you think the issue is with the scraper, feel free to get in touch with us or file an issue . At this point you're ready to run scrapers and contribute fixes. Hop onto our GitHub ticket queue , pick an issue to solve, and then submit a Pull Request!","title":"Running Your First Scraper"},{"location":"contributing/scrapers/#importing-data","text":"Optionally, if you'd like to see how your scraped data imports into the database, perhaps to diagnose an issue that is happening after the scrape, pop over to getting a working database to see how to get a local database that you can import data into. Once that's done, make sure that the db image from openstates.org is running: $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 27fe691ad7c5 mdillon/postgis:11-alpine \"docker-entrypoint.s\u2026\" 3 hours ago Up 3 hours 0.0.0.0:5405->5432/tcp openstatesorg_db_1 Your output will vary, but if you don't see something named openstatesorg_db running you should run this command (from the openstates.org directory, not your scraper directory): $ docker-compose up -d db Now, when you want to run imports, you can drop the --scrape portion of the command you've been running. Or if you just want to test the import of already scraped data you can replace it with --import . An import looks something like this: $ docker-compose run --rm scrape fl bills --fast ... (truncated) ... 23:03:34 ERROR openstates: cannot resolve pseudo id to Person: ~{\u201cname\u201d: \u201cGrant, M.\u201c} 23:03:36 ERROR openstates: cannot resolve pseudo id to Person: ~{\u201cname\u201d: \u201cRodrigues, R.\u201c} fl (import) bills: {} import: bill: 0 new 0 updated 2620 noop jurisdiction: 0 new 0 updated 1 noop vote_event: 21 new 12 updated 533 noop The errors about unresolved psuedo-ids can safely be ignored, as long as you see the final run report the data you scraped is available in your database. The number of objects of each type that were created & updated are available for spot checking, as well as the total number of items that were seen that already exactly matched what was in the database. These can be useful stats as you try to see if your local changes to a scraper have the impact you expect.","title":"Importing Data"},{"location":"contributing/scrapers/#running-spatula-scrapers","text":"Let's run a people scraper: $ poetry run spatula scrape scrapers_next.nc.people.SenList The command to run these scrapers is structured differently, as the parameters are set by giving the exact location of the function you want to run: directory.state.file.function . Note Function names do vary and scrapes for legislators are commonly split by chamber, so make sure to check you're passing the right function in your command. The actual data scraping should look something like: INFO:scrapers_next.nc.people.SenList:fetching https://www.ncleg.gov/Members/MemberTable/S INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/430 INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/431 INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/432 INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/433 INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/434 To review the data you scraped, you can inspect the JSON files in the dated directory within _scrapes/ . Each time you run a scrape, a new numbered folder will be within the dated directory, so you can compare older data to new easily. Note If a scrape fails, it's likely an issue with the scraper. Feel free to get in touch with us or file an issue . Spatula is incredibly powerful with lots of flexibility and useful CLI commands that are worth checking out as well. At this point you're ready to run spatula scrapers and contribute fixes. Hop onto our GitHub ticket queue , pick an issue to solve, and then submit a Pull Request!","title":"Running Spatula Scrapers"},{"location":"contributing/state-specific/","text":"State-Specific Scraper Info \u00b6 California MySQL \u00b6 California is a unique state that takes a couple of extra steps to get working locally. California provides MySQL dumps of their data, and in order to use those we start up a local MySQL instance and read from that. To download the data for the current session: docker-compose run --rm ca-download (You can append --year YYYY to instead select data for a given year.) This will start a local MySQL image as well, that image will need to stay up for the next step, which is running a scrape like normal: docker-compose run --rm ca-scrape ca bills --fast State API Keys \u00b6 Unfortunately, some states find it necessary to require API Keys (or other credentials) to access their best data. Despite the difficulties this creates for contributors, in the interest of ensuring we have the best possible data we've made the decision that we will use this data where possible. Our policy: We will maintain (when possible) two copies of credentials, one for development and one for production. (Thus minimizing the chance that a mistake made w/ a development key will jeopardize our ability to scrape.) We encourage developers to get an API key of their own, but if necessary we can share our testing key in limited circumstances. Currently only a few states require API keys: New York \u00b6 http://legislation.nysenate.gov/static/docs/html/index.html Request Form: http://legislation.nysenate.gov/ Set in environment prior to running scrape: NEW_YORK_API_KEY Indiana \u00b6 http://docs.api.iga.in.gov/api.html API Key Request Process: Email Bob Amos ( bob.amos@iga.in.gov or bamos@iga.in.gov ), and include your name, address, phone, email address and company. Also indicate that you have read the terms of service at the link above. Set in environment prior to running scrape: INDIANA_API_KEY As a side note, Indiana also requires a user-agent string , so set that in your environment as well, prior to running scrape: USER_AGENT Virginia \u00b6 https://lis.virginia.gov/SiteInformation/ftp.html API Credentials Request Process: To acquire access, please contact the Virginia Legislative Information System help desk at (804) 786-9631 for a user id. Set in environment prior to running scrape: VIRGINIA_FTP_USER , VIRGINIA_FTP_PASSWORD District of Columbia \u00b6 https://lims.dccouncil.us/api/help/index.html API Key Request Form: https://lims.dccouncil.us/developerRegistration Set in environment prior to running scrape: DC_API_KEY","title":"State-Specific Scraper Info"},{"location":"contributing/state-specific/#state-specific-scraper-info","text":"","title":"State-Specific Scraper Info"},{"location":"contributing/state-specific/#california-mysql","text":"California is a unique state that takes a couple of extra steps to get working locally. California provides MySQL dumps of their data, and in order to use those we start up a local MySQL instance and read from that. To download the data for the current session: docker-compose run --rm ca-download (You can append --year YYYY to instead select data for a given year.) This will start a local MySQL image as well, that image will need to stay up for the next step, which is running a scrape like normal: docker-compose run --rm ca-scrape ca bills --fast","title":"California MySQL"},{"location":"contributing/state-specific/#state-api-keys","text":"Unfortunately, some states find it necessary to require API Keys (or other credentials) to access their best data. Despite the difficulties this creates for contributors, in the interest of ensuring we have the best possible data we've made the decision that we will use this data where possible. Our policy: We will maintain (when possible) two copies of credentials, one for development and one for production. (Thus minimizing the chance that a mistake made w/ a development key will jeopardize our ability to scrape.) We encourage developers to get an API key of their own, but if necessary we can share our testing key in limited circumstances. Currently only a few states require API keys:","title":"State API Keys"},{"location":"contributing/state-specific/#new-york","text":"http://legislation.nysenate.gov/static/docs/html/index.html Request Form: http://legislation.nysenate.gov/ Set in environment prior to running scrape: NEW_YORK_API_KEY","title":"New York"},{"location":"contributing/state-specific/#indiana","text":"http://docs.api.iga.in.gov/api.html API Key Request Process: Email Bob Amos ( bob.amos@iga.in.gov or bamos@iga.in.gov ), and include your name, address, phone, email address and company. Also indicate that you have read the terms of service at the link above. Set in environment prior to running scrape: INDIANA_API_KEY As a side note, Indiana also requires a user-agent string , so set that in your environment as well, prior to running scrape: USER_AGENT","title":"Indiana"},{"location":"contributing/state-specific/#virginia","text":"https://lis.virginia.gov/SiteInformation/ftp.html API Credentials Request Process: To acquire access, please contact the Virginia Legislative Information System help desk at (804) 786-9631 for a user id. Set in environment prior to running scrape: VIRGINIA_FTP_USER , VIRGINIA_FTP_PASSWORD","title":"Virginia"},{"location":"contributing/state-specific/#district-of-columbia","text":"https://lims.dccouncil.us/api/help/index.html API Key Request Form: https://lims.dccouncil.us/developerRegistration Set in environment prior to running scrape: DC_API_KEY","title":"District of Columbia"},{"location":"contributing/testing-scrapers/","text":"Testing Scrapers \u00b6 One of the first things people new to the project tend to notice is that there aren't a lot of tests in the scrapers. Over the years we've evolved a de facto policy of somewhat discouraging tests, which is definitely an unusual stance to take and warrants explanation. Intentionally Fragile Scrapers \u00b6 When it comes to scrapers, there are two major types of breakage: 1) the scraper collects bad information and inserts it into the database 2) the scraper encounters an error and quits without importing data Given a choice, the second is greatly preferable. Once bad data makes it into the database, it can be difficult to detect and remove. On the other hand, the second can be triggered to alert us immediately and someone can evaluate the proper fix. The best way to favor the second over first is to write \"intentionally fragile\" scrapers. That is, scrapers that raise an exception when they see unexpected input. While it is possible to try to write a resilient scraper that recovers, by nature these scrapers are more likely to produce the first kind of error, and so we encourage scraper writers to be conservative in what errors are suppressed. Here's an example of an overly permissive scraper: party_abbr = doc.xpath('//span[@class=\"partyabbr\"]) if party_abbr == 'D': party = 'Democratic' elif party_abbr == 'R': party = 'Republican' else: # haven't seen this yet, but let's just keep things moving party = party_abbr The following would be preferred: party_abbr = doc.xpath('//span[@class=\"partyabbr\"]) party = {'D': 'Democratic', 'R': 'Republican'}[party_abbr] This code would raise a KeyError the first time a new party is found. This forces someone to take a look, fix the scraper with an entry for the new party, and then the scraper will be able to run again with correct data. Testing Scrapers Is Hard \u00b6 On most software projects a failing test means that something is broken, and passing tests should mean that things are working just fine. In our experience however, the majority of the \"breaks\" that occur in scrapers are due to upstream site changes. In the past the fragile nature of scrapers has led to people writing a lot of bad tests, which is where our stance of somewhat discouraging tests has come from. An example of a bad test: def extract_name(doc): return doc.xpath('//h2[@class=\"legislatorName\"]').text_content().strip() def test_extract_name(): # probably a snapshot of the page at some point in time EXAMPLE_LEGISLATOR_HTML = '...' doc = lxml.html.fromstring(EXAMPLE_LEGISLATOR_HTML) assert extract_name(doc) == 'Erica Example' With a test like this: As soon as the HTML changes, the scraper will start failing, but the tests will still pass. The scraper will then be updated, breaking the test. The test HTML will be updated, fixing the test. But since the initial scraper breakage isn't predicted by a failing test, this type of test really doesn't serve us any purpose and just results in extra code to maintain every time the scraper needs a slight change. Other Strategies \u00b6 Of course this isn't to say that we just abandon the idea of testing, altogether. If you're more comfortable writing tests, say you're parsing a particularly nasty PDF and want to run it against some test data: a test might make sense there as a way to be confident in your own code, by all means, write a test. We also have some other strategies to help ensure data quality: Validate Scraper Output \u00b6 Scraper output is verified against JSON schemas that protect against common regressions (missing sources, invalid formatted districts, etc.) - most of these tests can be written effectively against scraper output across the board, and in doing so also applies universally across all 50 states. We also aim for our underlying libraries like openstates-core to be as well tested as possible. (To be 100% clear, our lax testing philosophy only applies to site-specific scraper code, not these support libraries.) Run Scrapers Regularly \u00b6 In a sense, the scrapers are tested every night by being run. This is why the intentionally fragile approach is so important; those failures are in essence the same as integration test failures. Of course, this doesn't tell us if the scraper is picking up bad data, etc., but combined with validation we can be fairly confident in our data. Test Utilities \u00b6 One area we can definitely improve upon is our use of (and then thorough testing of) common functions. Right now (largely because of the great variety of authors, etc.) many scrapers do similar things like conversion of party abbreviations and whitespace normalization in slightly different ways. We should be making a push to use common utility functions and thoroughly test those.","title":"Testing Scrapers"},{"location":"contributing/testing-scrapers/#testing-scrapers","text":"One of the first things people new to the project tend to notice is that there aren't a lot of tests in the scrapers. Over the years we've evolved a de facto policy of somewhat discouraging tests, which is definitely an unusual stance to take and warrants explanation.","title":"Testing Scrapers"},{"location":"contributing/testing-scrapers/#intentionally-fragile-scrapers","text":"When it comes to scrapers, there are two major types of breakage: 1) the scraper collects bad information and inserts it into the database 2) the scraper encounters an error and quits without importing data Given a choice, the second is greatly preferable. Once bad data makes it into the database, it can be difficult to detect and remove. On the other hand, the second can be triggered to alert us immediately and someone can evaluate the proper fix. The best way to favor the second over first is to write \"intentionally fragile\" scrapers. That is, scrapers that raise an exception when they see unexpected input. While it is possible to try to write a resilient scraper that recovers, by nature these scrapers are more likely to produce the first kind of error, and so we encourage scraper writers to be conservative in what errors are suppressed. Here's an example of an overly permissive scraper: party_abbr = doc.xpath('//span[@class=\"partyabbr\"]) if party_abbr == 'D': party = 'Democratic' elif party_abbr == 'R': party = 'Republican' else: # haven't seen this yet, but let's just keep things moving party = party_abbr The following would be preferred: party_abbr = doc.xpath('//span[@class=\"partyabbr\"]) party = {'D': 'Democratic', 'R': 'Republican'}[party_abbr] This code would raise a KeyError the first time a new party is found. This forces someone to take a look, fix the scraper with an entry for the new party, and then the scraper will be able to run again with correct data.","title":"Intentionally Fragile Scrapers"},{"location":"contributing/testing-scrapers/#testing-scrapers-is-hard","text":"On most software projects a failing test means that something is broken, and passing tests should mean that things are working just fine. In our experience however, the majority of the \"breaks\" that occur in scrapers are due to upstream site changes. In the past the fragile nature of scrapers has led to people writing a lot of bad tests, which is where our stance of somewhat discouraging tests has come from. An example of a bad test: def extract_name(doc): return doc.xpath('//h2[@class=\"legislatorName\"]').text_content().strip() def test_extract_name(): # probably a snapshot of the page at some point in time EXAMPLE_LEGISLATOR_HTML = '...' doc = lxml.html.fromstring(EXAMPLE_LEGISLATOR_HTML) assert extract_name(doc) == 'Erica Example' With a test like this: As soon as the HTML changes, the scraper will start failing, but the tests will still pass. The scraper will then be updated, breaking the test. The test HTML will be updated, fixing the test. But since the initial scraper breakage isn't predicted by a failing test, this type of test really doesn't serve us any purpose and just results in extra code to maintain every time the scraper needs a slight change.","title":"Testing Scrapers Is Hard"},{"location":"contributing/testing-scrapers/#other-strategies","text":"Of course this isn't to say that we just abandon the idea of testing, altogether. If you're more comfortable writing tests, say you're parsing a particularly nasty PDF and want to run it against some test data: a test might make sense there as a way to be confident in your own code, by all means, write a test. We also have some other strategies to help ensure data quality:","title":"Other Strategies"},{"location":"contributing/testing-scrapers/#validate-scraper-output","text":"Scraper output is verified against JSON schemas that protect against common regressions (missing sources, invalid formatted districts, etc.) - most of these tests can be written effectively against scraper output across the board, and in doing so also applies universally across all 50 states. We also aim for our underlying libraries like openstates-core to be as well tested as possible. (To be 100% clear, our lax testing philosophy only applies to site-specific scraper code, not these support libraries.)","title":"Validate Scraper Output"},{"location":"contributing/testing-scrapers/#run-scrapers-regularly","text":"In a sense, the scrapers are tested every night by being run. This is why the intentionally fragile approach is so important; those failures are in essence the same as integration test failures. Of course, this doesn't tell us if the scraper is picking up bad data, etc., but combined with validation we can be fairly confident in our data.","title":"Run Scrapers Regularly"},{"location":"contributing/testing-scrapers/#test-utilities","text":"One area we can definitely improve upon is our use of (and then thorough testing of) common functions. Right now (largely because of the great variety of authors, etc.) many scrapers do similar things like conversion of party abbreviations and whitespace normalization in slightly different ways. We should be making a push to use common utility functions and thoroughly test those.","title":"Test Utilities"},{"location":"contributing/text-extraction/","text":"Text Extraction \u00b6 The bill scrapers scrape the web and pull down metadata, including links to various versions of the bills. As a later step, we extract the actual text of the bill so that it can be indexed for search and other uses. Checking out \u00b6 Fork and clone the text-extraction repository: Visit https://github.com/openstates/text-extraction and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/text-extraction.git Cloning into 'text-extraction'... And remember to install pre-commit : $ pre-commit install pre-commit installed at .git/hooks/pre-commit Repository overview \u00b6 The text extraction code itself is written as a standalone Python script text_extract.py that uses configuration and utility functions from within extract/ . You'll also notice a directory called raw/ -- this contains a sampling of bills for each state that we can use to test text-extraction. Typically if you're making changes in the repository you'll be editing files within extract/ , we'll come back to that later. Running text_extract \u00b6 Just like in other repositories, we'll use docker-compose to run the code. In this case docker-compose is running text_extract.py , an all-in-one tool that has a few useful subcommands: Usage: text_extract.py [OPTIONS] COMMAND [ARGS]... Options: --help Show this message and exit. Commands: reindex-state rebuild the search index objects for a given state sample obtain a sample of bills to extract text from status print a status table showing the current condition of... test run sample on all states, used for CI update update the saved bill text in the database For the purposes of development, sample and update are the only two commands that you'll need to look at. Let's go ahead and run sample against NC: $ docker-compose run --rm text-extract sample nc raw/nc/2017-HR 924-Edition 1.pdf => text/nc/2017-HR 924-Edition 1.pdf.txt (1507 bytes) raw/nc/2017-HB 1034-Edition 1.pdf => text/nc/2017-HB 1034-Edition 1.pdf.txt (3096 bytes) raw/nc/2019-SB 421-Edition 1.pdf => text/nc/2019-SB 421-Edition 1.pdf.txt (961 bytes) raw/nc/2019-HB 430-Edition 1.pdf => text/nc/2019-HB 430-Edition 1.pdf.txt (4831 bytes) raw/nc/2017-SB 753-Edition 1.pdf => text/nc/2017-SB 753-Edition 1.pdf.txt (719 bytes) raw/nc/2019-HB 788-Edition 1.pdf => text/nc/2019-HB 788-Edition 1.pdf.txt (2674 bytes) raw/nc/2017-SB 373-Filed.pdf => text/nc/2017-SB 373-Filed.pdf.txt (18538 bytes) raw/nc/2019-SB 574-Filed.pdf => text/nc/2019-SB 574-Filed.pdf.txt (1712 bytes) raw/nc/2017-SJR 686-Resolution 2017-12.pdf => text/nc/2017-SJR 686-Resolution 2017-12.pdf.txt (15928 bytes) raw/nc/2017-HB 1007-Filed.pdf => text/nc/2017-HB 1007-Filed.pdf.txt (6248 bytes) nc: processed 10, 0 skipped, 0 missing, 0 empty The exact output and number of bills will vary across states, but should be pretty similar. This command just did a lot: Read in the file raw/nc.csv to get a list of bills to sample. Downloaded those files (assuming this was the first run) to raw/nc/ so future runs will be faster. Used the extraction function(s) defined in extract/__init__.py for NC to extract text from the given documents. Wrote that output to text/nc/ so you can compare. You'll also notice that it helpfully prints the number of bytes of text extracted, this is useful as a first check. Let's go ahead and look at the shortest one, text/nc/2017-SB 753-Edition 1.pdf.txt . (Your run may differ, pick whichever you prefer.) : $ cat \"text/nc/2017-SB 753 Edition 1.pdf.txt\" A BILL TO BE ENTITLED AN ACT PROVIDING THAT THE DEPOSIT OF CURRENCY AND COINS INTO A CASH VAULT THAT PHYSICALLY SECURES THE CASH AND ELECTRONICALLY RECORDS THE DEPOSIT DAILY IN AN OFFICIAL DEPOSITORY BANK QUALIFIES AS A DAILY DEPOSIT UNDER THE LOCAL GOVERNMENT BUDGET AND FISCAL CONTROL ACT FOR FRANKLIN AND WAKE COUNTIES AND THE MUNICIPALITIES IN THOSE COUNTIES. The General Assembly of North Carolina enacts: SECTION 1. Section 2 of S.L. 2011-89 reads as rewritten: \"SECTION 2. This act applies only to the City of Winston-Salem only.Winston-Salem, Franklin County and the municipalities in Franklin County, and Wake County and the municipalities in Wake County.\" SECTION 2. This act is effective when it becomes law. This looks complete, but to check, go ahead and open the equivalent source file, in this case raw/nc/2017-SB 753-Edition 1.pdf and confirm visually that all the text was extracted. Don't worry about formatting, or the preamble, as we'll often exclude that and just aim for the interesting bits of the text. Making changes \u00b6 Let's say that we discover that a state has started publishing their bills in a new format. Perhaps Alabama switches from PDF to HTML. It'd first be good to add some of these new bills to the sample csv, which you can do manually or by invoking sample with the --resample flag.: docker-compose run --rm text-extract sample --resample al Running would result in some warnings being printed and some zero byte files. To actually handle the HTML documents we'd open up extract/__init__.py and find the CONVERSION_FUNCTIONS dictionary, you'll see a line like: CONVERSION_FUNCTIONS = { \"al\": {\"application/pdf\": extract_line_numbered_pdf}, ... The way extraction works is by matching a document found in a scrape to an appropriate function, in this case PDFs will be sent through the extract_line_numbered_pdf function. If the new HTML was wrapped in a given element, perhaps with
    we could just update that line to look like: CONVERSION_FUNCTIONS = { \"al\": { \"application/pdf\": extract_line_numbered_pdf, \"text/html\": extractor_for_element_by_id(\"billtext\"), }, ... And we'd be good to go. Tips & Tricks \u00b6 Functions already exist for common configurations of PDF, HTML, Word Doc, and even OCR. Rarely will you need to write a custom function, always look at the options first. When dealing with PDFs, most are either handled by extract_line_numbered_pdf or extract_sometimes_numbered_pdf , the difference is that \"sometimes numbered PDF\" accounts for cases where 90% or so of bills are numbered, but a few (often resolutions) are not numbered. Formatting Guidelines \u00b6 How far do we go? Should we strip punctuation? Newlines? Whitespace? Section headings? Try not to be too aggressive with punctuation stripping, search indices/etc. can easily strip it later, but it can be handy if someone decides they want to search for things like \"\u00a7 143C-4-8.b\" Ideally leave newlines as-is since it makes looking at changes a lot nicer for humans and stripping newlines out for final products (search/text comparison/etc.) is trivial. Collapsing spaces/etc. is recommended, but not required. Removal of section headers/etc. is fine, but if the only reason you're writing a new function instead of using a common one is to do this, reconsider. When in doubt, ask , you may have encountered something we haven't considered yet and we can discuss the best practice and add it here. Should we include bill digests? There isn't a need to, but it doesn't hurt if separating the two is difficult.","title":"Text Extraction"},{"location":"contributing/text-extraction/#text-extraction","text":"The bill scrapers scrape the web and pull down metadata, including links to various versions of the bills. As a later step, we extract the actual text of the bill so that it can be indexed for search and other uses.","title":"Text Extraction"},{"location":"contributing/text-extraction/#checking-out","text":"Fork and clone the text-extraction repository: Visit https://github.com/openstates/text-extraction and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/text-extraction.git Cloning into 'text-extraction'... And remember to install pre-commit : $ pre-commit install pre-commit installed at .git/hooks/pre-commit","title":"Checking out"},{"location":"contributing/text-extraction/#repository-overview","text":"The text extraction code itself is written as a standalone Python script text_extract.py that uses configuration and utility functions from within extract/ . You'll also notice a directory called raw/ -- this contains a sampling of bills for each state that we can use to test text-extraction. Typically if you're making changes in the repository you'll be editing files within extract/ , we'll come back to that later.","title":"Repository overview"},{"location":"contributing/text-extraction/#running-text_extract","text":"Just like in other repositories, we'll use docker-compose to run the code. In this case docker-compose is running text_extract.py , an all-in-one tool that has a few useful subcommands: Usage: text_extract.py [OPTIONS] COMMAND [ARGS]... Options: --help Show this message and exit. Commands: reindex-state rebuild the search index objects for a given state sample obtain a sample of bills to extract text from status print a status table showing the current condition of... test run sample on all states, used for CI update update the saved bill text in the database For the purposes of development, sample and update are the only two commands that you'll need to look at. Let's go ahead and run sample against NC: $ docker-compose run --rm text-extract sample nc raw/nc/2017-HR 924-Edition 1.pdf => text/nc/2017-HR 924-Edition 1.pdf.txt (1507 bytes) raw/nc/2017-HB 1034-Edition 1.pdf => text/nc/2017-HB 1034-Edition 1.pdf.txt (3096 bytes) raw/nc/2019-SB 421-Edition 1.pdf => text/nc/2019-SB 421-Edition 1.pdf.txt (961 bytes) raw/nc/2019-HB 430-Edition 1.pdf => text/nc/2019-HB 430-Edition 1.pdf.txt (4831 bytes) raw/nc/2017-SB 753-Edition 1.pdf => text/nc/2017-SB 753-Edition 1.pdf.txt (719 bytes) raw/nc/2019-HB 788-Edition 1.pdf => text/nc/2019-HB 788-Edition 1.pdf.txt (2674 bytes) raw/nc/2017-SB 373-Filed.pdf => text/nc/2017-SB 373-Filed.pdf.txt (18538 bytes) raw/nc/2019-SB 574-Filed.pdf => text/nc/2019-SB 574-Filed.pdf.txt (1712 bytes) raw/nc/2017-SJR 686-Resolution 2017-12.pdf => text/nc/2017-SJR 686-Resolution 2017-12.pdf.txt (15928 bytes) raw/nc/2017-HB 1007-Filed.pdf => text/nc/2017-HB 1007-Filed.pdf.txt (6248 bytes) nc: processed 10, 0 skipped, 0 missing, 0 empty The exact output and number of bills will vary across states, but should be pretty similar. This command just did a lot: Read in the file raw/nc.csv to get a list of bills to sample. Downloaded those files (assuming this was the first run) to raw/nc/ so future runs will be faster. Used the extraction function(s) defined in extract/__init__.py for NC to extract text from the given documents. Wrote that output to text/nc/ so you can compare. You'll also notice that it helpfully prints the number of bytes of text extracted, this is useful as a first check. Let's go ahead and look at the shortest one, text/nc/2017-SB 753-Edition 1.pdf.txt . (Your run may differ, pick whichever you prefer.) : $ cat \"text/nc/2017-SB 753 Edition 1.pdf.txt\" A BILL TO BE ENTITLED AN ACT PROVIDING THAT THE DEPOSIT OF CURRENCY AND COINS INTO A CASH VAULT THAT PHYSICALLY SECURES THE CASH AND ELECTRONICALLY RECORDS THE DEPOSIT DAILY IN AN OFFICIAL DEPOSITORY BANK QUALIFIES AS A DAILY DEPOSIT UNDER THE LOCAL GOVERNMENT BUDGET AND FISCAL CONTROL ACT FOR FRANKLIN AND WAKE COUNTIES AND THE MUNICIPALITIES IN THOSE COUNTIES. The General Assembly of North Carolina enacts: SECTION 1. Section 2 of S.L. 2011-89 reads as rewritten: \"SECTION 2. This act applies only to the City of Winston-Salem only.Winston-Salem, Franklin County and the municipalities in Franklin County, and Wake County and the municipalities in Wake County.\" SECTION 2. This act is effective when it becomes law. This looks complete, but to check, go ahead and open the equivalent source file, in this case raw/nc/2017-SB 753-Edition 1.pdf and confirm visually that all the text was extracted. Don't worry about formatting, or the preamble, as we'll often exclude that and just aim for the interesting bits of the text.","title":"Running text_extract"},{"location":"contributing/text-extraction/#making-changes","text":"Let's say that we discover that a state has started publishing their bills in a new format. Perhaps Alabama switches from PDF to HTML. It'd first be good to add some of these new bills to the sample csv, which you can do manually or by invoking sample with the --resample flag.: docker-compose run --rm text-extract sample --resample al Running would result in some warnings being printed and some zero byte files. To actually handle the HTML documents we'd open up extract/__init__.py and find the CONVERSION_FUNCTIONS dictionary, you'll see a line like: CONVERSION_FUNCTIONS = { \"al\": {\"application/pdf\": extract_line_numbered_pdf}, ... The way extraction works is by matching a document found in a scrape to an appropriate function, in this case PDFs will be sent through the extract_line_numbered_pdf function. If the new HTML was wrapped in a given element, perhaps with
    we could just update that line to look like: CONVERSION_FUNCTIONS = { \"al\": { \"application/pdf\": extract_line_numbered_pdf, \"text/html\": extractor_for_element_by_id(\"billtext\"), }, ... And we'd be good to go.","title":"Making changes"},{"location":"contributing/text-extraction/#tips-tricks","text":"Functions already exist for common configurations of PDF, HTML, Word Doc, and even OCR. Rarely will you need to write a custom function, always look at the options first. When dealing with PDFs, most are either handled by extract_line_numbered_pdf or extract_sometimes_numbered_pdf , the difference is that \"sometimes numbered PDF\" accounts for cases where 90% or so of bills are numbered, but a few (often resolutions) are not numbered.","title":"Tips & Tricks"},{"location":"contributing/text-extraction/#formatting-guidelines","text":"How far do we go? Should we strip punctuation? Newlines? Whitespace? Section headings? Try not to be too aggressive with punctuation stripping, search indices/etc. can easily strip it later, but it can be handy if someone decides they want to search for things like \"\u00a7 143C-4-8.b\" Ideally leave newlines as-is since it makes looking at changes a lot nicer for humans and stripping newlines out for final products (search/text comparison/etc.) is trivial. Collapsing spaces/etc. is recommended, but not required. Removal of section headers/etc. is fine, but if the only reason you're writing a new function instead of using a common one is to do this, reconsider. When in doubt, ask , you may have encountered something we haven't considered yet and we can discuss the best practice and add it here. Should we include bill digests? There isn't a need to, but it doesn't hurt if separating the two is difficult.","title":"Formatting Guidelines"},{"location":"contributing/writing-a-committee-scraper/","text":"Writing a Committee Scraper \u00b6 For Those Completely New to Writing Scrapers \u00b6 What do we mean when we use the term \u201cweb scraping\u201d? What is a scraper? How do you write a web scraper? An overview from the source above: When you run the code for web scraping, a request is sent to the URL that you have mentioned. As a response to the request, the server sends the data and allows you to read the HTML or XML page. The code then parses the HTML or XML page, finds the data and extracts it. To extract data using web scraping with python, you need to follow these basic steps: Find the URL that you want to scrape Find the data you want to extract Write the code Run the code and extract the data Store the data in the required format For those new to Open States: \u00b6 What is Open States? What is an open source project? How do you contribute to Open States? What projects are we looking for contributions on? Committee Scrapers Other issues Writing Committee Scrapers \u00b6 Background: What are legislative committees? \u00b6 This information isn\u2019t necessary for writing scrapers , but if you\u2019re interested in more context or want to know why committee data is important, this is where you\u2019ll find it! Legislative committees are groups of legislators selected by House and Senate leadership to consider legislation concerning a certain subject or set of subjects (ex: Appropriations, Environment & Natural Resources, Criminal Justice & Public Safety). While most state legislatures have separate House and Senate committees, a few states (CT, ME, MA, and NE) have just one set of committees. Most legislators serve on between 2 and 5 committees each session. Standing committees are permanent committees that are created at the beginning of the legislative session in the House, Senate, or Joint Rules. Other committee types (select committees, interim committees, study committees) are time-limited and often created by legislation or resolutions. Because the vast majority of legislation goes through the committee process, and because the committee membership has such an outsized role on whether that legislation lives or dies, it's incredibly important that people have accurate and up to date information about committees, their membership, and their work. Writing a Committee Scraper, Step by Step \u00b6 How do I set up my environment? Make sure you've walked through our installation prerequisities Where should my code live? All committee scrapers live in the scrapers_next directory of the openstates_scrapers repository When you're ready to commit your code, this is where your new committee scraper will live too! What are the steps for working on my committee scraper locally? Fork the openstates-scrapers repository ( what is a fork ?) Find the state you want to write a committee scraper for under the scrapers_next directory Create a committees.py file inside of that state's directory Write your committee scraper code! What information does a committee scraper need to grab? Name of the committee Chamber (upper/lower/etc) Classification (committee, subcommittee) Parent (if it is a subcommittee) Sources (home page for the list of committees, specific page for that committee, etc) Members (name and role on the committee) Example of complete data: { \"name\" : \"Revenue\" , \"chamber\" : \"upper\" , \"classification\" : \"committee\" , \"parent\" : null , \"sources\" : [ { \"url\" : \"https://www.ilga.gov/senate/committees/default.asp\" , \"note\" : \"homepage\" }, { \"url\" : \"https://www.ilga.gov/senate/committees/members.asp?CommitteeID=2688\" , \"note\" : \"\" }], \"links\" : [], \"other_names\" : [], \"members\" : [ { \"name\" : \"Mattie Hunter\" , \"role\" : \"Chair\" , \"person_id\" : null }, { \"name\" : \"Steve Stadelman\" , \"role\" : \"Vice-Chair\" , \"person_id\" : null }, { \"name\" : \"Robert F. Martwick\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Cristina H. Pacione-Zayas\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Robert Peters\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Elgie R. Sims, Jr.\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Donald P. DeWitte\" , \"role\" : \"Minority\" , \"person_id\" : null }, { \"name\" : \"Dale Fowler\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Jil Tracy\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Sally J. Turner\" , \"role\" : \"Member\" , \"person_id\" : null }], \"extras\" : {} } What does a complete committee scraper look like? An example committee scraper The way committee scrapers look will vary depending on how the website for the state is set up. Most of them will have a list of committees for the House, a list of committees for the Senate, and individual pages for each committee that includes details about membership and/or subcommittees I know what information I want to grab, but how do I grab it? Grab the information you need using selectors (<- a guide to selectors) Getting the selector/XPath on chrome: Right click the item you want to grab Click \u201cinspect\u201d A panel will pop up and the element that you\u2019re inspecting should be highlighted. Click on the three dots on the left side of the highlight Hover over \u201ccopy\u201d and click on either selector (for a CSS selector) or XPath to copy it There aren\u2019t any rules about whether XPath or CSS selectors are better. Use whichever helps you grab the information you need! General Scraper Writing Tips \u00b6 What are helpful tools for writing scrapers? The most helpful tool that you can use to write a committee scraper is the python package spatula A walkthrough on how to write a scraper using spatula How do I run a scraper? Follow the instructions here . If you haven\u2019t read the Getting Started , make sure you do that first! The command for running a spatula scraper is poetry run spatula scrape {directory your python file is in}.{state your scraper is for}.committees.{function (optional)} Example: To run the entire committee scraper for Missouri, the command would be poetry run spatula scrape scrapers_next.mo.committees Help! My code doesn't work! Stuck? Think it through and ask us for help! What\u2019s not working? What do you think should happen? What\u2019s actually happening? Exception handling can be great for weird cases, like a specific committee\u2019s information being unavailable Spatula lets you easily skip over weird cases Debug and quickly test if your CSS or Xpath selector is accurate with spatula test for a single page Before running spatula scrape, you can test the logic of what you\u2019ve already written (and essentially get a quick \u201cpreview\u201d) in the terminal. This is especially helpful for making sure you\u2019re using the correct CSS or Xpath selector for an element. You can set an example_source as a default URL to keep running spatula scrape on Example use case: After scraping a list of committees on one page, I now need specific information about each committee. Each committee has its own webpage (that we navigate to from that list of committees), and the format for each webpage is fairly similar (this is most likely the case!). I can set the example_source as a single committee\u2019s more-detailed webpage, and as I write that part of the scraper, I can keep running spatula test to see in my terminal that I\u2019m grabbing the correct information from that specific webpage. This should save time and prevent some headaches!","title":"Writing a Committee Scraper"},{"location":"contributing/writing-a-committee-scraper/#writing-a-committee-scraper","text":"","title":"Writing a Committee Scraper"},{"location":"contributing/writing-a-committee-scraper/#for-those-completely-new-to-writing-scrapers","text":"What do we mean when we use the term \u201cweb scraping\u201d? What is a scraper? How do you write a web scraper? An overview from the source above: When you run the code for web scraping, a request is sent to the URL that you have mentioned. As a response to the request, the server sends the data and allows you to read the HTML or XML page. The code then parses the HTML or XML page, finds the data and extracts it. To extract data using web scraping with python, you need to follow these basic steps: Find the URL that you want to scrape Find the data you want to extract Write the code Run the code and extract the data Store the data in the required format","title":"For Those Completely New to Writing Scrapers"},{"location":"contributing/writing-a-committee-scraper/#for-those-new-to-open-states","text":"What is Open States? What is an open source project? How do you contribute to Open States? What projects are we looking for contributions on? Committee Scrapers Other issues","title":"For those new to Open States:"},{"location":"contributing/writing-a-committee-scraper/#writing-committee-scrapers","text":"","title":"Writing Committee Scrapers"},{"location":"contributing/writing-a-committee-scraper/#background-what-are-legislative-committees","text":"This information isn\u2019t necessary for writing scrapers , but if you\u2019re interested in more context or want to know why committee data is important, this is where you\u2019ll find it! Legislative committees are groups of legislators selected by House and Senate leadership to consider legislation concerning a certain subject or set of subjects (ex: Appropriations, Environment & Natural Resources, Criminal Justice & Public Safety). While most state legislatures have separate House and Senate committees, a few states (CT, ME, MA, and NE) have just one set of committees. Most legislators serve on between 2 and 5 committees each session. Standing committees are permanent committees that are created at the beginning of the legislative session in the House, Senate, or Joint Rules. Other committee types (select committees, interim committees, study committees) are time-limited and often created by legislation or resolutions. Because the vast majority of legislation goes through the committee process, and because the committee membership has such an outsized role on whether that legislation lives or dies, it's incredibly important that people have accurate and up to date information about committees, their membership, and their work.","title":"Background: What are legislative committees?"},{"location":"contributing/writing-a-committee-scraper/#writing-a-committee-scraper-step-by-step","text":"How do I set up my environment? Make sure you've walked through our installation prerequisities Where should my code live? All committee scrapers live in the scrapers_next directory of the openstates_scrapers repository When you're ready to commit your code, this is where your new committee scraper will live too! What are the steps for working on my committee scraper locally? Fork the openstates-scrapers repository ( what is a fork ?) Find the state you want to write a committee scraper for under the scrapers_next directory Create a committees.py file inside of that state's directory Write your committee scraper code! What information does a committee scraper need to grab? Name of the committee Chamber (upper/lower/etc) Classification (committee, subcommittee) Parent (if it is a subcommittee) Sources (home page for the list of committees, specific page for that committee, etc) Members (name and role on the committee) Example of complete data: { \"name\" : \"Revenue\" , \"chamber\" : \"upper\" , \"classification\" : \"committee\" , \"parent\" : null , \"sources\" : [ { \"url\" : \"https://www.ilga.gov/senate/committees/default.asp\" , \"note\" : \"homepage\" }, { \"url\" : \"https://www.ilga.gov/senate/committees/members.asp?CommitteeID=2688\" , \"note\" : \"\" }], \"links\" : [], \"other_names\" : [], \"members\" : [ { \"name\" : \"Mattie Hunter\" , \"role\" : \"Chair\" , \"person_id\" : null }, { \"name\" : \"Steve Stadelman\" , \"role\" : \"Vice-Chair\" , \"person_id\" : null }, { \"name\" : \"Robert F. Martwick\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Cristina H. Pacione-Zayas\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Robert Peters\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Elgie R. Sims, Jr.\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Donald P. DeWitte\" , \"role\" : \"Minority\" , \"person_id\" : null }, { \"name\" : \"Dale Fowler\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Jil Tracy\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Sally J. Turner\" , \"role\" : \"Member\" , \"person_id\" : null }], \"extras\" : {} } What does a complete committee scraper look like? An example committee scraper The way committee scrapers look will vary depending on how the website for the state is set up. Most of them will have a list of committees for the House, a list of committees for the Senate, and individual pages for each committee that includes details about membership and/or subcommittees I know what information I want to grab, but how do I grab it? Grab the information you need using selectors (<- a guide to selectors) Getting the selector/XPath on chrome: Right click the item you want to grab Click \u201cinspect\u201d A panel will pop up and the element that you\u2019re inspecting should be highlighted. Click on the three dots on the left side of the highlight Hover over \u201ccopy\u201d and click on either selector (for a CSS selector) or XPath to copy it There aren\u2019t any rules about whether XPath or CSS selectors are better. Use whichever helps you grab the information you need!","title":"Writing a Committee Scraper, Step by Step"},{"location":"contributing/writing-a-committee-scraper/#general-scraper-writing-tips","text":"What are helpful tools for writing scrapers? The most helpful tool that you can use to write a committee scraper is the python package spatula A walkthrough on how to write a scraper using spatula How do I run a scraper? Follow the instructions here . If you haven\u2019t read the Getting Started , make sure you do that first! The command for running a spatula scraper is poetry run spatula scrape {directory your python file is in}.{state your scraper is for}.committees.{function (optional)} Example: To run the entire committee scraper for Missouri, the command would be poetry run spatula scrape scrapers_next.mo.committees Help! My code doesn't work! Stuck? Think it through and ask us for help! What\u2019s not working? What do you think should happen? What\u2019s actually happening? Exception handling can be great for weird cases, like a specific committee\u2019s information being unavailable Spatula lets you easily skip over weird cases Debug and quickly test if your CSS or Xpath selector is accurate with spatula test for a single page Before running spatula scrape, you can test the logic of what you\u2019ve already written (and essentially get a quick \u201cpreview\u201d) in the terminal. This is especially helpful for making sure you\u2019re using the correct CSS or Xpath selector for an element. You can set an example_source as a default URL to keep running spatula scrape on Example use case: After scraping a list of committees on one page, I now need specific information about each committee. Each committee has its own webpage (that we navigate to from that list of committees), and the format for each webpage is fairly similar (this is most likely the case!). I can set the example_source as a single committee\u2019s more-detailed webpage, and as I write that part of the scraper, I can keep running spatula test to see in my terminal that I\u2019m grabbing the correct information from that specific webpage. This should save time and prevent some headaches!","title":"General Scraper Writing Tips"},{"location":"data/","text":"Understanding the Data \u00b6 Open States data adheres to a schema that has evolved over our 11+ years of working with legislative data. Our goal is to provide as much uniformity across states as possible while still allowing for the wide diversity of legislative processes between the states. These docs both catalog the schema and attempt to explain some of those choices, particularly where they might be surprising. Main Concepts \u00b6 The main concepts are defined below. You'll notice these concepts mostly correspond to the v2 GraphQL root nodes. Concept Definition Jurisdiction Essentially just another word for \"State\" in our context. (Includes DC and Puerto Rico.) Session A period of time in a legislature where the same members serve together, typically punctuated by elections. All bills in a session will be uniquely numbered. (e.g. HB 1 in the 2017 session is typically not the same bill as in the 2019 session) Bill Represents all types of legislation whether it is a bill, resolution, etc. Vote A vote among members of the legislature, typically an entire chamber but can also be a committee vote. Person Any person that is associated with the legislature. Organization A generic term used to represent a few different concepts: legislatures, chambers, committees, and political parties. Post A particular role within an organization, typically used to represent a seat in the legislature. (e.g. the District 4 post in the North Carolina Senate Organization) Membership Ties a Person to a Post for a duration of time.","title":"Understanding the Data"},{"location":"data/#understanding-the-data","text":"Open States data adheres to a schema that has evolved over our 11+ years of working with legislative data. Our goal is to provide as much uniformity across states as possible while still allowing for the wide diversity of legislative processes between the states. These docs both catalog the schema and attempt to explain some of those choices, particularly where they might be surprising.","title":"Understanding the Data"},{"location":"data/#main-concepts","text":"The main concepts are defined below. You'll notice these concepts mostly correspond to the v2 GraphQL root nodes. Concept Definition Jurisdiction Essentially just another word for \"State\" in our context. (Includes DC and Puerto Rico.) Session A period of time in a legislature where the same members serve together, typically punctuated by elections. All bills in a session will be uniquely numbered. (e.g. HB 1 in the 2017 session is typically not the same bill as in the 2019 session) Bill Represents all types of legislation whether it is a bill, resolution, etc. Vote A vote among members of the legislature, typically an entire chamber but can also be a committee vote. Person Any person that is associated with the legislature. Organization A generic term used to represent a few different concepts: legislatures, chambers, committees, and political parties. Post A particular role within an organization, typically used to represent a seat in the legislature. (e.g. the District 4 post in the North Carolina Senate Organization) Membership Ties a Person to a Post for a duration of time.","title":"Main Concepts"},{"location":"data/categorization/","text":"Categorization \u00b6 One of the ways that we add value to the data we provide is by attempting to classify bills, actions, and votes across states. This allows us to let states use their own names for these things, but for us to try to provide some mapping to a common simplified view of the legislative process. Bill Types \u00b6 State legislatures deal with more than bills. Despite the name of the bill objects in our data we take in all types of legislation that a state might produce. Generally looking at the bill_id will help you determine the type of legislation, but to make things easier across states we provide a type field on bills. This field is a list with one (or more) of the following values: Common Values: bill resolution joint resolution concurrent resolution constitutional amendment Some states also make use of additional types such as 'contract', 'nomination', 'memorial' and more. Action Types \u00b6 Although most states follow very similar parlimentary procedure the names that their bill status systems use for various actions almost never match up. To make analysis and the building of certain types of tools easier we attempt to classify common actions. In using our data you'll find these values in the classification field of actions. filing - bill is filed (where this is a separate action from introduction) introduction - introduced, typically the first action reading-1 - first reading (often same as introduction) reading-2 - second reading reading-3 - third reading (often same as passage) passage - bill is passed by the chamber failure - bill fails to proceed from the chamber withdrawal - bill is withdrawn substitution - a substitution is made to the bill text deferral - consideration of the bill was deferred receipt - a bill was received by another chamber referral - a bill was sent somewhere for consideration referral-committee - a bill was sent to a committee for consideration became-law - the bill became law (through signature or inaction) amendment-introduction - an amendment is introduced amendment-passage - an amendment passes amendment-withdrawal - an amendment is withdrawn amendment-failure - an amendment fails to pass amendment-amendment - an amendment is amended amendment-deferral - consideration of an amendment is deferred committee-passage - the bill passes the current committee (unknown outcome, typically favorble) committee-passage-favorable - the bill passes the current committee favorably committee-passage-unfavorable - the bill passes the current committee with an unfavorable report committee-failure - the bill fails to advance out of committee executive-receipt - the bill is sent to the governor executive-signature - the governor signs the bill executive-veto - the governor vetos the bill executive-veto-line-item - the governor uses a line-item veto to strike part of a bill veto-override-passage - a veto override vote occurred and succeeded veto-override-failure - a veto override vote occurred and failed Vote Types \u00b6 Similarly to actions, we make an effort to categorize the motion being voted upon. You'll find these values in the categorization field on VoteEvents. Possible values: bill-passage - This is a vote to pass (either out of committee or a chamber) amendment-passage - Vote on amending a bill veto-override - Vote to override an executive veto","title":"Categorization"},{"location":"data/categorization/#categorization","text":"One of the ways that we add value to the data we provide is by attempting to classify bills, actions, and votes across states. This allows us to let states use their own names for these things, but for us to try to provide some mapping to a common simplified view of the legislative process.","title":"Categorization"},{"location":"data/categorization/#bill-types","text":"State legislatures deal with more than bills. Despite the name of the bill objects in our data we take in all types of legislation that a state might produce. Generally looking at the bill_id will help you determine the type of legislation, but to make things easier across states we provide a type field on bills. This field is a list with one (or more) of the following values: Common Values: bill resolution joint resolution concurrent resolution constitutional amendment Some states also make use of additional types such as 'contract', 'nomination', 'memorial' and more.","title":"Bill Types"},{"location":"data/categorization/#action-types","text":"Although most states follow very similar parlimentary procedure the names that their bill status systems use for various actions almost never match up. To make analysis and the building of certain types of tools easier we attempt to classify common actions. In using our data you'll find these values in the classification field of actions. filing - bill is filed (where this is a separate action from introduction) introduction - introduced, typically the first action reading-1 - first reading (often same as introduction) reading-2 - second reading reading-3 - third reading (often same as passage) passage - bill is passed by the chamber failure - bill fails to proceed from the chamber withdrawal - bill is withdrawn substitution - a substitution is made to the bill text deferral - consideration of the bill was deferred receipt - a bill was received by another chamber referral - a bill was sent somewhere for consideration referral-committee - a bill was sent to a committee for consideration became-law - the bill became law (through signature or inaction) amendment-introduction - an amendment is introduced amendment-passage - an amendment passes amendment-withdrawal - an amendment is withdrawn amendment-failure - an amendment fails to pass amendment-amendment - an amendment is amended amendment-deferral - consideration of an amendment is deferred committee-passage - the bill passes the current committee (unknown outcome, typically favorble) committee-passage-favorable - the bill passes the current committee favorably committee-passage-unfavorable - the bill passes the current committee with an unfavorable report committee-failure - the bill fails to advance out of committee executive-receipt - the bill is sent to the governor executive-signature - the governor signs the bill executive-veto - the governor vetos the bill executive-veto-line-item - the governor uses a line-item veto to strike part of a bill veto-override-passage - a veto override vote occurred and succeeded veto-override-failure - a veto override vote occurred and failed","title":"Action Types"},{"location":"data/categorization/#vote-types","text":"Similarly to actions, we make an effort to categorize the motion being voted upon. You'll find these values in the categorization field on VoteEvents. Possible values: bill-passage - This is a vote to pass (either out of committee or a chamber) amendment-passage - Vote on amending a bill veto-override - Vote to override an executive veto","title":"Vote Types"},{"location":"data/session-naming/","text":"Session Naming \u00b6 States name their sessions drastically differently, and sometimes inconsistently even within their own site. (49th vs 2008 Regular Session). As our goal is to help smooth these inconsistencies we put forward this guide to naming sessions within state metadata. (See https://github.com/sunlightlabs/openstates/issues/81 for discussion on the topic) Default Session Names \u00b6 The sessions list within terms is dangerous to change as all bill data is keyed off it. As a rule these should be short and generally useful for the scraper to make the appropriate decisions on what data to scrape. If a state calls its 1st special session in 2010 '2010E1' this is a perfectly acceptable name for the session in the metadata. Similarly 49th-regular, 2009-Special-B, etc. are fine names. Generally names with spaces should be avoided simply for ease of construction of URLs, etc. In states where spaces are already in use it is fine to continue to use them. The one caveat is that if a state uses a unique ID that has no bearing on the session itself such as '7323' for the 2011 session, this should not be used. Instead add some mapping that maps a session name that is descriptive to their internal ids. Session Display Names \u00b6 Because the most convenient name to refer to a session is often far from what a user might expect to see upon opening a mobile application, the session_details dict supports a display_name key. Suitable display names are descriptive but also short and obey a given style. General Rules \u00b6 All sessions should be in title case. Fewer than 20 characters is highly preferable. Months should be abbreviated to 3 letters (Jan., Feb., Jun., Dec.) Ordinals \u00b6 If no special sessions are present: [Ordinal] Legislature If special sessions are present: [Ordinal] Regular Session [Ordinal], [Ordinal] Special Session Examples: 82nd Legislature 82nd Regular Session 82nd, 3rd Special Session Years \u00b6 [Year/Year-Range] Regular Session [Year/Year-Range], [Ordinal] Special Session [Mon. Year] Special Session Examples: 2010 Regular Session 2011-2012, 4th Special Session Dec. 2011 Special Session","title":"Session Naming"},{"location":"data/session-naming/#session-naming","text":"States name their sessions drastically differently, and sometimes inconsistently even within their own site. (49th vs 2008 Regular Session). As our goal is to help smooth these inconsistencies we put forward this guide to naming sessions within state metadata. (See https://github.com/sunlightlabs/openstates/issues/81 for discussion on the topic)","title":"Session Naming"},{"location":"data/session-naming/#default-session-names","text":"The sessions list within terms is dangerous to change as all bill data is keyed off it. As a rule these should be short and generally useful for the scraper to make the appropriate decisions on what data to scrape. If a state calls its 1st special session in 2010 '2010E1' this is a perfectly acceptable name for the session in the metadata. Similarly 49th-regular, 2009-Special-B, etc. are fine names. Generally names with spaces should be avoided simply for ease of construction of URLs, etc. In states where spaces are already in use it is fine to continue to use them. The one caveat is that if a state uses a unique ID that has no bearing on the session itself such as '7323' for the 2011 session, this should not be used. Instead add some mapping that maps a session name that is descriptive to their internal ids.","title":"Default Session Names"},{"location":"data/session-naming/#session-display-names","text":"Because the most convenient name to refer to a session is often far from what a user might expect to see upon opening a mobile application, the session_details dict supports a display_name key. Suitable display names are descriptive but also short and obey a given style.","title":"Session Display Names"},{"location":"data/session-naming/#general-rules","text":"All sessions should be in title case. Fewer than 20 characters is highly preferable. Months should be abbreviated to 3 letters (Jan., Feb., Jun., Dec.)","title":"General Rules"},{"location":"data/session-naming/#ordinals","text":"If no special sessions are present: [Ordinal] Legislature If special sessions are present: [Ordinal] Regular Session [Ordinal], [Ordinal] Special Session Examples: 82nd Legislature 82nd Regular Session 82nd, 3rd Special Session","title":"Ordinals"},{"location":"data/session-naming/#years","text":"[Year/Year-Range] Regular Session [Year/Year-Range], [Ordinal] Special Session [Mon. Year] Special Session Examples: 2010 Regular Session 2011-2012, 4th Special Session Dec. 2011 Special Session","title":"Years"},{"location":"enhancement-proposals/","text":"enhancement-proposals \u00b6 Proposals to enhance data or functionality within the Open States project. See OSEP #1 You can open an issue or use template.md to get started on your own proposal.","title":"enhancement-proposals"},{"location":"enhancement-proposals/#enhancement-proposals","text":"Proposals to enhance data or functionality within the Open States project. See OSEP #1 You can open an issue or use template.md to get started on your own proposal.","title":"enhancement-proposals"},{"location":"enhancement-proposals/001-purpose-and-process/","text":"OSEP #1: Purpose & Process \u00b6 Author(s) @jamesturk Implementer(s) @jamesturk Status Final Draft PR(s) n/a Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/1 Created 2021-02-11 Updated 2021-08-17 Abstract \u00b6 Provide a formal mechanism by which major changes to Open States can be agreed upon & made. Specification \u00b6 Format \u00b6 A proposal should have this structure: Short title (e.g. \"Adding Effective Dates\"), which is also in the filename (e.g. 007-adding-effective-dates.md ). Metadata : listed at the top of the file, to include: Author(s), Implementer(s) - these can be multiple project members, implementer can be blank initially. Created, Updated : Updated should be updated when substantial changes are adopted. Status : Draft, Accepted, Rejected, Withdrawn, Final, Superseded Issue : To better track the current status of various proposals, all proposals must have a linked issue with the appropriate status tag. Draft PR : Link to Draft PR discussion. Approval PR : Link to Approval PR discussion. Abstract - Concise summary of the proposal. Specification - Detailed description of how the proposal should work. Rationale - Explanation of why this would be a good idea. The rationale should discuss considered alternatives and why they were rejected. Drawbacks - A discussion of backwards compatibility, maintenance cost, etc. (Can be omitted or very simple in drafts.) Implementation Plan - An estimate of what it would take to implement this. (Can be omitted or very simple in drafts.) Copyright - Each proposal must end with a CC0 dedication. Issue Status \u00b6 Process \u00b6 Drafting A Proposal Submitting a proposal will take a little bit more planning than just opening a GitHub issue, but you can start there by opening a \"pre-draft\" issue if desired. Issues on this repository can be used as a form of proto-proposal in the early planning phase. No specific format is required, but these can be useful to gauge interest in a formal proposal if desired. Small non-breaking changes and minor site & API enhancements will continue to be filed as typical issues in openstates/issues but larger feature requests and similar will be moved to this repository with a note explaining this process. When ready, draft proposals begin as PRs against this repository. Once a proposal is deemed on-topic and in the above format, it will be accepted as a draft proposal. Approval Phase Once accepted as a draft, a full discussion will ensue. This repository should serve as the system of record for this discussion. (i.e. If a conversation requires a Slack conversation or similar, the results of that conversation should be attached to the relevant issue/PR) If fields like implementer, drawbacks, and implementation plan were not specified in the draft, they must be determined before a proposal can be accepted. After any requisite back & forth, the active core team will submit their decision. All active core developers can review, with a goal of consensus, but in cases of indecision the project lead will make the final decision. The status will then be adjusted to Accepted, Rejected, or Withdrawn. Implementation Phase Once accepted, work can begin as needed. The proposal can & will likely be edited during this time. Once the work is reviewed and accepted, the proposal's status will be updated to Final. Rationale \u00b6 Both over the years, and right now in GitHub issues & discussions there are a lot of proposals to improve the project in various ways. These are typically handled via a combination of GitHub issues, Slack conversations, and individual whims. By introducing a formal but lightweight Open States Enhancement Proposal (OSEP) process, I am hoping to achieve three outcomes: Structure the conversation around improvements so that new contributors and core members alike can understand the scope of a proposed change. Provide a means by which consensus can be gathered publicly, in a timely manner to reduce stalled decisions so that fewer issues are left in the \"yeah.. we really should figure out how to X\" state. Available development time will still be an obstacle at times, but at least when time is available it will be clear what can be worked on. Keep a record of failed proposals so we retain institutional memory why some things aren't done. This can reduce the same conversation happening when new contributors arrive, and also let us revisit old decisions to see if the context around old decisions still holds or if there's need to re-evaluate. History \u00b6 There used to be a process surrounding OCDEPs by which the core schema/etc. could be modified, but that process was cumbersome and slow and wasn't reflective of the needs of Open States as the spec derived there was aiming at a common denominator between various team's needs. Most other improvements have come after either lengthy conversation or on a whim, typically James'. (Both of these come with their share of issues, hence the proposal.) Alternatives could be to continue with the status quo or just using GitHub issues ad hoc. Both feel inferior to having a documented process around this. In August 2021, this proposal was updated to spell out the use of GitHub issues to track the current status of proposals. Drawbacks \u00b6 n/a Implementation Plan \u00b6 If you're reading this, the repository is already created. Documentation and openstates/issues repo will need to be updated to point people to this process. Copyright \u00b6 This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"OSEP #1: Purpose & Process"},{"location":"enhancement-proposals/001-purpose-and-process/#osep-1-purpose-process","text":"Author(s) @jamesturk Implementer(s) @jamesturk Status Final Draft PR(s) n/a Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/1 Created 2021-02-11 Updated 2021-08-17","title":"OSEP #1: Purpose & Process"},{"location":"enhancement-proposals/001-purpose-and-process/#abstract","text":"Provide a formal mechanism by which major changes to Open States can be agreed upon & made.","title":"Abstract"},{"location":"enhancement-proposals/001-purpose-and-process/#specification","text":"","title":"Specification"},{"location":"enhancement-proposals/001-purpose-and-process/#format","text":"A proposal should have this structure: Short title (e.g. \"Adding Effective Dates\"), which is also in the filename (e.g. 007-adding-effective-dates.md ). Metadata : listed at the top of the file, to include: Author(s), Implementer(s) - these can be multiple project members, implementer can be blank initially. Created, Updated : Updated should be updated when substantial changes are adopted. Status : Draft, Accepted, Rejected, Withdrawn, Final, Superseded Issue : To better track the current status of various proposals, all proposals must have a linked issue with the appropriate status tag. Draft PR : Link to Draft PR discussion. Approval PR : Link to Approval PR discussion. Abstract - Concise summary of the proposal. Specification - Detailed description of how the proposal should work. Rationale - Explanation of why this would be a good idea. The rationale should discuss considered alternatives and why they were rejected. Drawbacks - A discussion of backwards compatibility, maintenance cost, etc. (Can be omitted or very simple in drafts.) Implementation Plan - An estimate of what it would take to implement this. (Can be omitted or very simple in drafts.) Copyright - Each proposal must end with a CC0 dedication.","title":"Format"},{"location":"enhancement-proposals/001-purpose-and-process/#issue-status","text":"","title":"Issue Status"},{"location":"enhancement-proposals/001-purpose-and-process/#process","text":"Drafting A Proposal Submitting a proposal will take a little bit more planning than just opening a GitHub issue, but you can start there by opening a \"pre-draft\" issue if desired. Issues on this repository can be used as a form of proto-proposal in the early planning phase. No specific format is required, but these can be useful to gauge interest in a formal proposal if desired. Small non-breaking changes and minor site & API enhancements will continue to be filed as typical issues in openstates/issues but larger feature requests and similar will be moved to this repository with a note explaining this process. When ready, draft proposals begin as PRs against this repository. Once a proposal is deemed on-topic and in the above format, it will be accepted as a draft proposal. Approval Phase Once accepted as a draft, a full discussion will ensue. This repository should serve as the system of record for this discussion. (i.e. If a conversation requires a Slack conversation or similar, the results of that conversation should be attached to the relevant issue/PR) If fields like implementer, drawbacks, and implementation plan were not specified in the draft, they must be determined before a proposal can be accepted. After any requisite back & forth, the active core team will submit their decision. All active core developers can review, with a goal of consensus, but in cases of indecision the project lead will make the final decision. The status will then be adjusted to Accepted, Rejected, or Withdrawn. Implementation Phase Once accepted, work can begin as needed. The proposal can & will likely be edited during this time. Once the work is reviewed and accepted, the proposal's status will be updated to Final.","title":"Process"},{"location":"enhancement-proposals/001-purpose-and-process/#rationale","text":"Both over the years, and right now in GitHub issues & discussions there are a lot of proposals to improve the project in various ways. These are typically handled via a combination of GitHub issues, Slack conversations, and individual whims. By introducing a formal but lightweight Open States Enhancement Proposal (OSEP) process, I am hoping to achieve three outcomes: Structure the conversation around improvements so that new contributors and core members alike can understand the scope of a proposed change. Provide a means by which consensus can be gathered publicly, in a timely manner to reduce stalled decisions so that fewer issues are left in the \"yeah.. we really should figure out how to X\" state. Available development time will still be an obstacle at times, but at least when time is available it will be clear what can be worked on. Keep a record of failed proposals so we retain institutional memory why some things aren't done. This can reduce the same conversation happening when new contributors arrive, and also let us revisit old decisions to see if the context around old decisions still holds or if there's need to re-evaluate.","title":"Rationale"},{"location":"enhancement-proposals/001-purpose-and-process/#history","text":"There used to be a process surrounding OCDEPs by which the core schema/etc. could be modified, but that process was cumbersome and slow and wasn't reflective of the needs of Open States as the spec derived there was aiming at a common denominator between various team's needs. Most other improvements have come after either lengthy conversation or on a whim, typically James'. (Both of these come with their share of issues, hence the proposal.) Alternatives could be to continue with the status quo or just using GitHub issues ad hoc. Both feel inferior to having a documented process around this. In August 2021, this proposal was updated to spell out the use of GitHub issues to track the current status of proposals.","title":"History"},{"location":"enhancement-proposals/001-purpose-and-process/#drawbacks","text":"n/a","title":"Drawbacks"},{"location":"enhancement-proposals/001-purpose-and-process/#implementation-plan","text":"If you're reading this, the repository is already created. Documentation and openstates/issues repo will need to be updated to point people to this process.","title":"Implementation Plan"},{"location":"enhancement-proposals/001-purpose-and-process/#copyright","text":"This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"Copyright"},{"location":"enhancement-proposals/002-legal-citations/","text":"OSEP #2: Legal Citations \u00b6 Author(s) Tim Showers Implementer(s) Tim Showers Status Accepted Issue https://github.com/openstates/enhancement-proposals/issues/33 Draft PR(s) https://github.com/openstates/enhancement-proposals/pull/9/ Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Created 2020-02-17 Updated TODO Abstract \u00b6 A number of jurisdictions offer metadata to link bills to the sections of the legal code that they will alter, or to the chaptered laws. We should modify the bills model to allow scraping this data in a structured way. Note : There are many different legal structures that can be altered by bills. In this document, \"legal code\" will be used as shorthand for \"collection of laws or rules modified by a bill\" such as statutes, codes, slip laws, public laws, register entries, constitutions, etc. Specification \u00b6 Add an underlying data structure for one or more legal citations to the bill model. These will be accessed via a new method on the bill object: add_citation() Legal Citation: \u00b6 Structure: \u00b6 A list of 0+ Citation dict s publication - string - The affected publication. e.g. \"Minnesota Statutes\", \"California Public Utilities Code\", \"DC Register\", \"Constitution of Nevada\". Note that these cover a wide variety of different types of law and rule making. citation - string - The reference to the (sub)section of the publication. Formats and abbreviations vary widely. type - enum [ proposed , chapter , final , other ] - whether the citation is a part of a pending bill, a chapter law, or a final affected section of some publication after the bill as been made into law. The list of proposed citations may not match the final list due to changes between bill versions. See the Chapter Citations section of rationale below for further discussion of chapter laws. other would cover corner cases such as a constitutional amendment that was passed by the legislature, but required a referendum as well. effective - optional datetimeoptional - effective date expiration - optional datetimeoptional - expiration date url - optional string - Link to the URL of the affected code Examples \u00b6 2019 WY HB 4 Chaptered to CH0024 of 2019, Effective 7/1/2019 bill . add_citation ( \"Wyoming Chapter Laws of 2019\" , \"CH0024\" , type = \"chapter\" , effective = datetime ( 2019 , 07 , 01 ) ) 2020 MN HF 4285 Chaptered to \"Chapter 89\", modified a number of Statutes, effective 08/01/20 bill . add_citation ( \"Minnesota Session Laws, 2020\" , \"Chapter 89\" , type = \"chapter\" , effective = datetime ( 2020 , 08 , 01 ), url = \"https://www.revisor.mn.gov/laws/2020/0/Session+Law/Chapter/89/\" , ) bill . add_citation ( \"Minnesota Statutes 2018\" , \"Chapter 27, Section 27\" , type = \"final\" , effective = datetime ( 2020 , 08 , 01 ) url = \"https://www.revisor.mn.gov/laws/2020/0/Session+Law/Chapter/89/\" ) bill . add_citation ( \"Minnesota Statutes 2018\" , \"Chapter 27, Section 13\" , type = \"final\" , effective = datetime ( 2020 , 08 , 01 ) url = \"https://www.revisor.mn.gov/laws/2020/0/Session+Law/Chapter/89/\" ) bill . add_citation ( \"Minnesota Statutes 2018\" , \"Chapter 27, Section 17\" , type = \"final\" , effective = datetime ( 2020 , 08 , 01 ) url = \"https://www.revisor.mn.gov/laws/2020/0/Session+Law/Chapter/89/\" ) 23rd Council DC 997 which results in a DC register entry in vol 67. bill . add_citation ( \"DC Register\" , \"Vol 67 and Page 14429\" , type = \"final\" , expires = datetime ( 2021 , 03 , 06 ) ) MO 2020 HJR 104 a (failed) constitutional amendment. bill . add_citation ( \"Constitution of Missouri\" , \"Article X Section 6\" , type = \"proposed\" , ) Rationale \u00b6 We want to be able to catch this both proactively -- \"Tell me if a bill is introduced that would alter the California corporations code\", and historically -- \"Who sponsored the bill that made (X) a misdemeanor instead of a felony?\". This also allows us to provide interesting session overview data -- \"What laws were changed as a result of the 2019 session?\", \"What legislator introduced the most changes?\". Chapter Citations: \u00b6 Many jurisdictions keep a running list of all the bills that have become law in a session, often with effective dates and redlines to aid legal researchers. These are generally known as 'chaptered laws', and may or may not also link to the state codes. Sometimes these are a holding area for laws to folded into state code(s) at a later date for something like an annual printing, but not always. Some states (MN) chapter things like budgets that aren't statutory, so there's a chapter law without a corresponding change to a legal code. Note that the \"Chaptered Laws\" of a given session, and a legal code that might have structural elements called \"chapters\" aren't necessarily related. So 2019/CH24 may not modify chapter 24 of some title of a legal code. Example chapter list from MN Granularity of data \u00b6 A given citation may be provided at multiple levels of granularity; for example: \"Minnesota Statutes 2018, section 31\" vs \"Minnesota Statutes 2018, section 31A.10\", \"Minnesota Statutes 2018, section 31A.14\", \"Minnesota Statutes 2018, section 31B.10-12\" There's often a comparatively simple first-pass approach that will yield the first result, while the second result requires a full-featured legal citation parser. The second approach also generally requires downloading and processing bill texts, which we prefer to keep seperate from the main bill scrapers for operational reasons. Here we leave the granularity up the individual scraper and contributor, allowing for maximum flexibility at the cost of some uniformity. We want to collect the 'low hanging fruit' of references that are on pages we already routinely scrape or could easily collect without exploding the volume of scraper requests. If a future contributor wants to add a more fully featured legal parser to one or more scrapers, we would need to balance code complexity and scrape time against the value of the collected data. This could potentially be integrated via a new module, or a processing pipeline step that lived seperately from the existing scrapers. Further discussion of this issue can be found the initial draft's pull request . Formatting as open-text \u00b6 In line with the differing formats provided by jurisdictions, consumers have differing opinions on the most granular citations that are appropriate, and how they should be formatted. A structured approach was considered, for example: {'chapter': '12', 'section': '23A', 'subsection': '3', 'paragraph': '1'} One consumer maybe interested in the specific paragraph for redline purposes, while another may want to point to the subsection or even section for providing context around the change. The large number of possible formats, and inconsistent naming ('title' lives at different levels across jursidictions, for example) makes writing parsers and validators for this impractical, particularly if end consumers are likely to re-parse this data anyway. Requiring 'parsed' citations also significantly raises the barrier to entry to adding this data to a scraper, and likely increases scraper failures as new citation formats are encountered. Given these drawbacks, an unstructured string is a more effective format here. See the Formatting section in drawbacks for further details. Drawbacks \u00b6 Data Availability -- The availability of this data is limited. See Note[1] below for a non-exhaustive survey. Formatting - State legal citation formats are extremely varied. See Note[2] for some examples. Rather than try to standardize them, I propose we just take the states citations as is and leave followup steps to the consumer. Codes/Laws are confusing. States handle rulemaking in a variety of formats -- Constitutions, Statutes, Codes, Administrative Registers, Administrative Rulemaking, etc. Many states also split their lawmaking into multiple publications, e.g. \"criminal code\", \"corporations code\", \"environmental code\", etc. Rather than try to handle that extreme complexity for a few jurisdictions, plaintext citations have us punt it to the end consumer. Proposed legal citations don't seem to be available as structured data in any states right now so this would require regexing bill text which has historically not been a thing we've done in scrapers. I think it's still better to have the ability in the model. Backwards Compatibility Issues: None, except for one-time a mass 'updated at' change in the affected states as the scrapers are updated. Implementation Plan \u00b6 Tim adds the model to openstates-core, with some help from James on questions. James updates the poetry requirements and rolls new images, and does database migrations. Scraper writers add the functionality to states as desired. Tim would take the first shot at the known states. Notes \u00b6 States that offer chapters post hoc -- DC, GA, US, WY, MN Projects that try to parse legal citations: law-identifier , citation-regexes Copyright \u00b6 This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"OSEP #2: Legal Citations"},{"location":"enhancement-proposals/002-legal-citations/#osep-2-legal-citations","text":"Author(s) Tim Showers Implementer(s) Tim Showers Status Accepted Issue https://github.com/openstates/enhancement-proposals/issues/33 Draft PR(s) https://github.com/openstates/enhancement-proposals/pull/9/ Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Created 2020-02-17 Updated TODO","title":"OSEP #2: Legal Citations"},{"location":"enhancement-proposals/002-legal-citations/#abstract","text":"A number of jurisdictions offer metadata to link bills to the sections of the legal code that they will alter, or to the chaptered laws. We should modify the bills model to allow scraping this data in a structured way. Note : There are many different legal structures that can be altered by bills. In this document, \"legal code\" will be used as shorthand for \"collection of laws or rules modified by a bill\" such as statutes, codes, slip laws, public laws, register entries, constitutions, etc.","title":"Abstract"},{"location":"enhancement-proposals/002-legal-citations/#specification","text":"Add an underlying data structure for one or more legal citations to the bill model. These will be accessed via a new method on the bill object: add_citation()","title":"Specification"},{"location":"enhancement-proposals/002-legal-citations/#legal-citation","text":"","title":"Legal Citation:"},{"location":"enhancement-proposals/002-legal-citations/#structure","text":"A list of 0+ Citation dict s publication - string - The affected publication. e.g. \"Minnesota Statutes\", \"California Public Utilities Code\", \"DC Register\", \"Constitution of Nevada\". Note that these cover a wide variety of different types of law and rule making. citation - string - The reference to the (sub)section of the publication. Formats and abbreviations vary widely. type - enum [ proposed , chapter , final , other ] - whether the citation is a part of a pending bill, a chapter law, or a final affected section of some publication after the bill as been made into law. The list of proposed citations may not match the final list due to changes between bill versions. See the Chapter Citations section of rationale below for further discussion of chapter laws. other would cover corner cases such as a constitutional amendment that was passed by the legislature, but required a referendum as well. effective - optional datetimeoptional - effective date expiration - optional datetimeoptional - expiration date url - optional string - Link to the URL of the affected code","title":"Structure:"},{"location":"enhancement-proposals/002-legal-citations/#examples","text":"2019 WY HB 4 Chaptered to CH0024 of 2019, Effective 7/1/2019 bill . add_citation ( \"Wyoming Chapter Laws of 2019\" , \"CH0024\" , type = \"chapter\" , effective = datetime ( 2019 , 07 , 01 ) ) 2020 MN HF 4285 Chaptered to \"Chapter 89\", modified a number of Statutes, effective 08/01/20 bill . add_citation ( \"Minnesota Session Laws, 2020\" , \"Chapter 89\" , type = \"chapter\" , effective = datetime ( 2020 , 08 , 01 ), url = \"https://www.revisor.mn.gov/laws/2020/0/Session+Law/Chapter/89/\" , ) bill . add_citation ( \"Minnesota Statutes 2018\" , \"Chapter 27, Section 27\" , type = \"final\" , effective = datetime ( 2020 , 08 , 01 ) url = \"https://www.revisor.mn.gov/laws/2020/0/Session+Law/Chapter/89/\" ) bill . add_citation ( \"Minnesota Statutes 2018\" , \"Chapter 27, Section 13\" , type = \"final\" , effective = datetime ( 2020 , 08 , 01 ) url = \"https://www.revisor.mn.gov/laws/2020/0/Session+Law/Chapter/89/\" ) bill . add_citation ( \"Minnesota Statutes 2018\" , \"Chapter 27, Section 17\" , type = \"final\" , effective = datetime ( 2020 , 08 , 01 ) url = \"https://www.revisor.mn.gov/laws/2020/0/Session+Law/Chapter/89/\" ) 23rd Council DC 997 which results in a DC register entry in vol 67. bill . add_citation ( \"DC Register\" , \"Vol 67 and Page 14429\" , type = \"final\" , expires = datetime ( 2021 , 03 , 06 ) ) MO 2020 HJR 104 a (failed) constitutional amendment. bill . add_citation ( \"Constitution of Missouri\" , \"Article X Section 6\" , type = \"proposed\" , )","title":"Examples"},{"location":"enhancement-proposals/002-legal-citations/#rationale","text":"We want to be able to catch this both proactively -- \"Tell me if a bill is introduced that would alter the California corporations code\", and historically -- \"Who sponsored the bill that made (X) a misdemeanor instead of a felony?\". This also allows us to provide interesting session overview data -- \"What laws were changed as a result of the 2019 session?\", \"What legislator introduced the most changes?\".","title":"Rationale"},{"location":"enhancement-proposals/002-legal-citations/#chapter-citations","text":"Many jurisdictions keep a running list of all the bills that have become law in a session, often with effective dates and redlines to aid legal researchers. These are generally known as 'chaptered laws', and may or may not also link to the state codes. Sometimes these are a holding area for laws to folded into state code(s) at a later date for something like an annual printing, but not always. Some states (MN) chapter things like budgets that aren't statutory, so there's a chapter law without a corresponding change to a legal code. Note that the \"Chaptered Laws\" of a given session, and a legal code that might have structural elements called \"chapters\" aren't necessarily related. So 2019/CH24 may not modify chapter 24 of some title of a legal code. Example chapter list from MN","title":"Chapter Citations:"},{"location":"enhancement-proposals/002-legal-citations/#granularity-of-data","text":"A given citation may be provided at multiple levels of granularity; for example: \"Minnesota Statutes 2018, section 31\" vs \"Minnesota Statutes 2018, section 31A.10\", \"Minnesota Statutes 2018, section 31A.14\", \"Minnesota Statutes 2018, section 31B.10-12\" There's often a comparatively simple first-pass approach that will yield the first result, while the second result requires a full-featured legal citation parser. The second approach also generally requires downloading and processing bill texts, which we prefer to keep seperate from the main bill scrapers for operational reasons. Here we leave the granularity up the individual scraper and contributor, allowing for maximum flexibility at the cost of some uniformity. We want to collect the 'low hanging fruit' of references that are on pages we already routinely scrape or could easily collect without exploding the volume of scraper requests. If a future contributor wants to add a more fully featured legal parser to one or more scrapers, we would need to balance code complexity and scrape time against the value of the collected data. This could potentially be integrated via a new module, or a processing pipeline step that lived seperately from the existing scrapers. Further discussion of this issue can be found the initial draft's pull request .","title":"Granularity of data"},{"location":"enhancement-proposals/002-legal-citations/#formatting-as-open-text","text":"In line with the differing formats provided by jurisdictions, consumers have differing opinions on the most granular citations that are appropriate, and how they should be formatted. A structured approach was considered, for example: {'chapter': '12', 'section': '23A', 'subsection': '3', 'paragraph': '1'} One consumer maybe interested in the specific paragraph for redline purposes, while another may want to point to the subsection or even section for providing context around the change. The large number of possible formats, and inconsistent naming ('title' lives at different levels across jursidictions, for example) makes writing parsers and validators for this impractical, particularly if end consumers are likely to re-parse this data anyway. Requiring 'parsed' citations also significantly raises the barrier to entry to adding this data to a scraper, and likely increases scraper failures as new citation formats are encountered. Given these drawbacks, an unstructured string is a more effective format here. See the Formatting section in drawbacks for further details.","title":"Formatting as open-text"},{"location":"enhancement-proposals/002-legal-citations/#drawbacks","text":"Data Availability -- The availability of this data is limited. See Note[1] below for a non-exhaustive survey. Formatting - State legal citation formats are extremely varied. See Note[2] for some examples. Rather than try to standardize them, I propose we just take the states citations as is and leave followup steps to the consumer. Codes/Laws are confusing. States handle rulemaking in a variety of formats -- Constitutions, Statutes, Codes, Administrative Registers, Administrative Rulemaking, etc. Many states also split their lawmaking into multiple publications, e.g. \"criminal code\", \"corporations code\", \"environmental code\", etc. Rather than try to handle that extreme complexity for a few jurisdictions, plaintext citations have us punt it to the end consumer. Proposed legal citations don't seem to be available as structured data in any states right now so this would require regexing bill text which has historically not been a thing we've done in scrapers. I think it's still better to have the ability in the model. Backwards Compatibility Issues: None, except for one-time a mass 'updated at' change in the affected states as the scrapers are updated.","title":"Drawbacks"},{"location":"enhancement-proposals/002-legal-citations/#implementation-plan","text":"Tim adds the model to openstates-core, with some help from James on questions. James updates the poetry requirements and rolls new images, and does database migrations. Scraper writers add the functionality to states as desired. Tim would take the first shot at the known states.","title":"Implementation Plan"},{"location":"enhancement-proposals/002-legal-citations/#notes","text":"States that offer chapters post hoc -- DC, GA, US, WY, MN Projects that try to parse legal citations: law-identifier , citation-regexes","title":"Notes"},{"location":"enhancement-proposals/002-legal-citations/#copyright","text":"This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"Copyright"},{"location":"enhancement-proposals/003-manual-people-data-tools/","text":"OSEP #3: Manual People Data Tools \u00b6 Author(s) James Turk Implementer(s) James Turk Status Accepted Issue https://github.com/openstates/enhancement-proposals/issues/32 Draft PR(s) https://github.com/openstates/enhancement-proposals/pull/14 Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Created 2021-02-23 Updated 2021-03-01 Abstract \u00b6 Introduce a public means by which we can improve and enhance legislator data, one that is available to less technical volunteers. Specification \u00b6 I am proposing we build a public tool to handle common corrections to legislator data. This tool would be used primarily by less technical volunteers that are not comfortable with the clone-edit YAML-PR workflow that currently exists. The tool is aimed at four types of issues in particular: adding details such as phone numbers, social media accounts, etc. to an individual or in bulk retiring legislators that have left office adding new one-off legislators that won special elections or appointments adding sponsor/vote aliases to legislators It's important to note that not all issues will be considered. For example: - an election has occurred and we need a full update - a systemic issue has resulted in us grabbing incorrect addresses, swapping voice/fax, or similar These will continue to be best handled by re-scraping, and will be considered out of scope for this proposal. What I am proposing is a publicly available web app that provides the following: Use Case 1: Bulk Editing & Review \u00b6 This will be the primary view into the new app. It will present the user with a table like: name district chamber [Retire] [Edit] There will be form controls to add editable columns including District & Capitol Voice, Fax, Address, Twitter, etc. Ideally the tool could provide a summary table for a sense of coverage for important fields as well, e.g. (23% of legislators are missing phone number). If a user wishes to perform a bulk edit, they will add the appropriate fields using the controls, and make the appropriate edits like they would in a spreadsheet. At the bottom of this page, there will be a form that will ask them for a short description of their changes, including a link to sources they used. Pressing this will save a ChangeSet to the local database. Each row also has a retire & edit link. Use Case 2: Retirement \u00b6 Another option available, is to retire a given individual. Clicking the retire link will prompt for the necessary details (retirement date, source, death?) and clicking save will create a simple ChangeSet indicating the retirement. Use Case 3: New Legislators (and editing a full legislator) \u00b6 In addition to the legislator table, there will also be individual edit pages. If a [New Legislator] button is pressed, the user will be prompted for the initial non-editable details such as Name, District, Chamber. Pressing [Save] will create a ChangeSet with just this legislator's information. (As noted above, this is only intended to be used for 1-2 legislators that have been appointed, a full session turnover should be processed by scraping.) Use Case 4: Adding Sponsor/Vote Aliases \u00b6 A separate view will be made available for the explicit purpose of showing unmatched sponsor & vote data. This view will allow contributors to select the matching legislator (or designate appropriate unmatched statuses as necessary). Pressing [Save] will create a ChangeSet that adds the sponsor/vote aliases to the appropriate legislators. Saving ChangeSets \u00b6 Each of the above actions saves data representing its delta to the existing legislator data. These will be stored in a local database, and converted to a PR against the openstates/people repository. As the tool's UI allows, it may be possible to combine multiple ChangeSets into a single PR. The PR will: - provide ChangeSet represented in minimally-edited YAML - to avoid dealing with GitHub credentials, the PR will likely be by openstates-bot or similar. - the Open States user's username (planning to use existing OpenStates.org authentication most likely for simplicity) -- might want a feature to @ mention their GitHub account Rationale \u00b6 After a full audit of Open States' legislator data, and reviewing the recently filed legislator issues, it seems like there are 3 types of issues that are common. I think with a bit of work we could come up with a tool that allows less technical volunteers to handle all of these. I think it will be worthwhile as we commit to keeping people data as up-to-date as possible. Other options considered included: - a locally-runnable app that worked directly on the YAML. - (This likely wouldn't really reduce the barrier to entry and would still require users to have basic GitHub skills.) - an AirTable app, as started in the openstates/people#383 PR . - (This still requires someone running the script locally, then handing the airtable over to PRs, also there are things that are likely easier with a simple CRUD app instead of trying to bend AirTable to the schema here.) - a web YAML editor - (This doesn't really allow retirements, bulk editing, etc. so doesn't feel like as much of a win.) Drawbacks \u00b6 There's of course added complexity in building and supporting a new tool. I am hoping that the scope detailed here will be possible to implement a working prototype in ~2 weeks and make continued revisions as the first batch of users work with it. This tool will not be able to do everything, so YAML will still be required for certain tasks. While this could be seen as a drawback, this limitation is by design, since the YAML exists there is no need for the tool to make it possible to cover all the strange edge cases that exist in needing to change the data. One example is that for now the tool will not provide a way to change a legislator's district or party as those changes are rare and a bit more complex in the YAML than others. (It'd be possible to add those features later though if they prove necessary, but the features described above aim at 80% of the edits we want to do.) Implementation Plan \u00b6 I'd plan to work on this myself, possibly with the assistance of one other engineer if resources were available. If prioritized, I think a working version could be produced in the next couple of weeks, leaving room for UI and other functionality improvements down the line. Copyright \u00b6 This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"OSEP #3: Manual People Data Tools"},{"location":"enhancement-proposals/003-manual-people-data-tools/#osep-3-manual-people-data-tools","text":"Author(s) James Turk Implementer(s) James Turk Status Accepted Issue https://github.com/openstates/enhancement-proposals/issues/32 Draft PR(s) https://github.com/openstates/enhancement-proposals/pull/14 Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Created 2021-02-23 Updated 2021-03-01","title":"OSEP #3: Manual People Data Tools"},{"location":"enhancement-proposals/003-manual-people-data-tools/#abstract","text":"Introduce a public means by which we can improve and enhance legislator data, one that is available to less technical volunteers.","title":"Abstract"},{"location":"enhancement-proposals/003-manual-people-data-tools/#specification","text":"I am proposing we build a public tool to handle common corrections to legislator data. This tool would be used primarily by less technical volunteers that are not comfortable with the clone-edit YAML-PR workflow that currently exists. The tool is aimed at four types of issues in particular: adding details such as phone numbers, social media accounts, etc. to an individual or in bulk retiring legislators that have left office adding new one-off legislators that won special elections or appointments adding sponsor/vote aliases to legislators It's important to note that not all issues will be considered. For example: - an election has occurred and we need a full update - a systemic issue has resulted in us grabbing incorrect addresses, swapping voice/fax, or similar These will continue to be best handled by re-scraping, and will be considered out of scope for this proposal. What I am proposing is a publicly available web app that provides the following:","title":"Specification"},{"location":"enhancement-proposals/003-manual-people-data-tools/#use-case-1-bulk-editing-review","text":"This will be the primary view into the new app. It will present the user with a table like: name district chamber [Retire] [Edit] There will be form controls to add editable columns including District & Capitol Voice, Fax, Address, Twitter, etc. Ideally the tool could provide a summary table for a sense of coverage for important fields as well, e.g. (23% of legislators are missing phone number). If a user wishes to perform a bulk edit, they will add the appropriate fields using the controls, and make the appropriate edits like they would in a spreadsheet. At the bottom of this page, there will be a form that will ask them for a short description of their changes, including a link to sources they used. Pressing this will save a ChangeSet to the local database. Each row also has a retire & edit link.","title":"Use Case 1: Bulk Editing & Review"},{"location":"enhancement-proposals/003-manual-people-data-tools/#use-case-2-retirement","text":"Another option available, is to retire a given individual. Clicking the retire link will prompt for the necessary details (retirement date, source, death?) and clicking save will create a simple ChangeSet indicating the retirement.","title":"Use Case 2: Retirement"},{"location":"enhancement-proposals/003-manual-people-data-tools/#use-case-3-new-legislators-and-editing-a-full-legislator","text":"In addition to the legislator table, there will also be individual edit pages. If a [New Legislator] button is pressed, the user will be prompted for the initial non-editable details such as Name, District, Chamber. Pressing [Save] will create a ChangeSet with just this legislator's information. (As noted above, this is only intended to be used for 1-2 legislators that have been appointed, a full session turnover should be processed by scraping.)","title":"Use Case 3: New Legislators (and editing a full legislator)"},{"location":"enhancement-proposals/003-manual-people-data-tools/#use-case-4-adding-sponsorvote-aliases","text":"A separate view will be made available for the explicit purpose of showing unmatched sponsor & vote data. This view will allow contributors to select the matching legislator (or designate appropriate unmatched statuses as necessary). Pressing [Save] will create a ChangeSet that adds the sponsor/vote aliases to the appropriate legislators.","title":"Use Case 4: Adding Sponsor/Vote Aliases"},{"location":"enhancement-proposals/003-manual-people-data-tools/#saving-changesets","text":"Each of the above actions saves data representing its delta to the existing legislator data. These will be stored in a local database, and converted to a PR against the openstates/people repository. As the tool's UI allows, it may be possible to combine multiple ChangeSets into a single PR. The PR will: - provide ChangeSet represented in minimally-edited YAML - to avoid dealing with GitHub credentials, the PR will likely be by openstates-bot or similar. - the Open States user's username (planning to use existing OpenStates.org authentication most likely for simplicity) -- might want a feature to @ mention their GitHub account","title":"Saving ChangeSets"},{"location":"enhancement-proposals/003-manual-people-data-tools/#rationale","text":"After a full audit of Open States' legislator data, and reviewing the recently filed legislator issues, it seems like there are 3 types of issues that are common. I think with a bit of work we could come up with a tool that allows less technical volunteers to handle all of these. I think it will be worthwhile as we commit to keeping people data as up-to-date as possible. Other options considered included: - a locally-runnable app that worked directly on the YAML. - (This likely wouldn't really reduce the barrier to entry and would still require users to have basic GitHub skills.) - an AirTable app, as started in the openstates/people#383 PR . - (This still requires someone running the script locally, then handing the airtable over to PRs, also there are things that are likely easier with a simple CRUD app instead of trying to bend AirTable to the schema here.) - a web YAML editor - (This doesn't really allow retirements, bulk editing, etc. so doesn't feel like as much of a win.)","title":"Rationale"},{"location":"enhancement-proposals/003-manual-people-data-tools/#drawbacks","text":"There's of course added complexity in building and supporting a new tool. I am hoping that the scope detailed here will be possible to implement a working prototype in ~2 weeks and make continued revisions as the first batch of users work with it. This tool will not be able to do everything, so YAML will still be required for certain tasks. While this could be seen as a drawback, this limitation is by design, since the YAML exists there is no need for the tool to make it possible to cover all the strange edge cases that exist in needing to change the data. One example is that for now the tool will not provide a way to change a legislator's district or party as those changes are rare and a bit more complex in the YAML than others. (It'd be possible to add those features later though if they prove necessary, but the features described above aim at 80% of the edits we want to do.)","title":"Drawbacks"},{"location":"enhancement-proposals/003-manual-people-data-tools/#implementation-plan","text":"I'd plan to work on this myself, possibly with the assistance of one other engineer if resources were available. If prioritized, I think a working version could be produced in the next couple of weeks, leaving room for UI and other functionality improvements down the line.","title":"Implementation Plan"},{"location":"enhancement-proposals/003-manual-people-data-tools/#copyright","text":"This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"Copyright"},{"location":"enhancement-proposals/004-committee-data/","text":"OSEP #4: Committee Data \u00b6 Author(s) @jamesturk Implementer(s) @jamesturk Status Final Draft PR(s) https://github.com/openstates/enhancement-proposals/pull/18 Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Created 2021-03-31 Updated 2021-08-03 Abstract \u00b6 Open States provided committee membership data from 2011-2018, but ceased to when it was no longer viable given the project's funding and staffing situations. This proposal would restore committee membership data via a hybrid scraped/manual process with a focus on maintaining up-to-date current membership of committees. Specification \u00b6 Data Model Changes \u00b6 The data model Open States inherited from OCD/Popolo is still a part of the core OS models and will be renewed to support this work. As part of this proposal, several small changes will be made to make working with this data slightly easier: OrganizationLink and OrganizationSource will be migrated to inline JSON fields on the Organization model. Organization will also gain an other_names JSON field. These changes can be implemented in a way that does not impact any of Open States' public APIs. Scraping \u00b6 Right now we have 45 committee scrapers in openstates-scrapers that have not been run in years. A quick check of 10 of them showed that the vast majority do not run as-is. That fact combined with the limitations in the old pupa import path for Organizations points towards us adopting a different method that allows easier augmentation of scraped data with manual. To this end, we will port/rewrite committee scrapers to a new format similar to the direction that new people scrapers are heading. The proposed path looks like this: The committee portions of openstates-people will be restored: each jurisdiction will again have a committees directory lint_yaml.py will be renamed lint_people.py a new lint_committees.py will be created the committee portion of to_database.py will be restored the committee portion of to_csv.py will be restored documentation and CI processes will be updated to include committee data Committee scrapers will be added to openstates-people and be written in a way that yields compatible YAML. Existing committee scrapers will be kept in openstates-scrapers until that state has been ported to openstates-people. new committee scrapers will be run regularly with changes to the YAML submitted as PRs (manually at first, but likely the subject of future automation) Manual Data \u00b6 The existing organization schema will be updated to require either ID or name, if ID is present it will be used for linking, whereas name will serve as a placeholder for unlinked IDs. Initially we'll be focused on manual data that is not impacted by the scrapers, such as alternate names and links. As such the tool that merges scraper output with the YAML will be focused on replacing member names with those found by the most recent scrape. This also makes it possible to add committee data by alternate processes, particularly useful in cases where scraping a given chamber or state's committees is harder than maintaining a manual roster. We will not focus on historical data, intending the committee membership in the file to reflect the membership at a given time. It is possible that in the future we could use git history to try to construct historical records from this. This manual data process also leaves the door open for future iterations where some data could be marked in the YAML as to-be-kept so that the membership of a committee could be partially manually curated, but that will be kept to future enhancements. API v2 \u00b6 API v2 will be updated in a backwards-compatible way, such that the existing organizations node can be used to fetch committee data again. API v3 \u00b6 Committees search & detail endpoints will be added to API v3 in the same manner as existing Jurisdictions/People endpoints. Rationale \u00b6 Restoration of committees has been one of the most requested data sets since their deprecation was deemed necessary. Now that there are resources available to pursue this again it is a high priority addition to our existing data. As for the approach itself, it mainly draws on the experience of the earlier iterations of committee data. Drawing upon the expertise gained in the first iteration, we are not attempting a fully automated process as augmentation & deduplication became quite difficult under that approach. The manual approach on the other hand quickly went stale and was not updated in any meaningful way after the initial release. By introducing a new process that combines both of these, it is believed we will trade some initial complexity for an approach that can keep up with the complexities of dealing with committee data. Drawbacks \u00b6 This approach is more complicated than it might otherwise be if we felt that relying entirely on scraped data (a la bills) was viable for committees. As a result it will require some additional upfront infrastructure work to achieve the hybrid scraped/manual data approach that is desired. There will be ongoing maintenance costs to the new infrastructure and scrapers required for this. As noted above plan does not account for historical committee information. Backwards Compatibility Note: The new committee scraping code will not be backwards compatible with the old. The API and website will be updated in a way that doesn't disturb any public interfaces. Implementation Plan \u00b6 James will lead infrastructure implementation. Several team members, and hopefully some community members, will help to contribute updated committee scrapers. Copyright \u00b6 This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"OSEP #4: Committee Data"},{"location":"enhancement-proposals/004-committee-data/#osep-4-committee-data","text":"Author(s) @jamesturk Implementer(s) @jamesturk Status Final Draft PR(s) https://github.com/openstates/enhancement-proposals/pull/18 Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Created 2021-03-31 Updated 2021-08-03","title":"OSEP #4: Committee Data"},{"location":"enhancement-proposals/004-committee-data/#abstract","text":"Open States provided committee membership data from 2011-2018, but ceased to when it was no longer viable given the project's funding and staffing situations. This proposal would restore committee membership data via a hybrid scraped/manual process with a focus on maintaining up-to-date current membership of committees.","title":"Abstract"},{"location":"enhancement-proposals/004-committee-data/#specification","text":"","title":"Specification"},{"location":"enhancement-proposals/004-committee-data/#data-model-changes","text":"The data model Open States inherited from OCD/Popolo is still a part of the core OS models and will be renewed to support this work. As part of this proposal, several small changes will be made to make working with this data slightly easier: OrganizationLink and OrganizationSource will be migrated to inline JSON fields on the Organization model. Organization will also gain an other_names JSON field. These changes can be implemented in a way that does not impact any of Open States' public APIs.","title":"Data Model Changes"},{"location":"enhancement-proposals/004-committee-data/#scraping","text":"Right now we have 45 committee scrapers in openstates-scrapers that have not been run in years. A quick check of 10 of them showed that the vast majority do not run as-is. That fact combined with the limitations in the old pupa import path for Organizations points towards us adopting a different method that allows easier augmentation of scraped data with manual. To this end, we will port/rewrite committee scrapers to a new format similar to the direction that new people scrapers are heading. The proposed path looks like this: The committee portions of openstates-people will be restored: each jurisdiction will again have a committees directory lint_yaml.py will be renamed lint_people.py a new lint_committees.py will be created the committee portion of to_database.py will be restored the committee portion of to_csv.py will be restored documentation and CI processes will be updated to include committee data Committee scrapers will be added to openstates-people and be written in a way that yields compatible YAML. Existing committee scrapers will be kept in openstates-scrapers until that state has been ported to openstates-people. new committee scrapers will be run regularly with changes to the YAML submitted as PRs (manually at first, but likely the subject of future automation)","title":"Scraping"},{"location":"enhancement-proposals/004-committee-data/#manual-data","text":"The existing organization schema will be updated to require either ID or name, if ID is present it will be used for linking, whereas name will serve as a placeholder for unlinked IDs. Initially we'll be focused on manual data that is not impacted by the scrapers, such as alternate names and links. As such the tool that merges scraper output with the YAML will be focused on replacing member names with those found by the most recent scrape. This also makes it possible to add committee data by alternate processes, particularly useful in cases where scraping a given chamber or state's committees is harder than maintaining a manual roster. We will not focus on historical data, intending the committee membership in the file to reflect the membership at a given time. It is possible that in the future we could use git history to try to construct historical records from this. This manual data process also leaves the door open for future iterations where some data could be marked in the YAML as to-be-kept so that the membership of a committee could be partially manually curated, but that will be kept to future enhancements.","title":"Manual Data"},{"location":"enhancement-proposals/004-committee-data/#api-v2","text":"API v2 will be updated in a backwards-compatible way, such that the existing organizations node can be used to fetch committee data again.","title":"API v2"},{"location":"enhancement-proposals/004-committee-data/#api-v3","text":"Committees search & detail endpoints will be added to API v3 in the same manner as existing Jurisdictions/People endpoints.","title":"API v3"},{"location":"enhancement-proposals/004-committee-data/#rationale","text":"Restoration of committees has been one of the most requested data sets since their deprecation was deemed necessary. Now that there are resources available to pursue this again it is a high priority addition to our existing data. As for the approach itself, it mainly draws on the experience of the earlier iterations of committee data. Drawing upon the expertise gained in the first iteration, we are not attempting a fully automated process as augmentation & deduplication became quite difficult under that approach. The manual approach on the other hand quickly went stale and was not updated in any meaningful way after the initial release. By introducing a new process that combines both of these, it is believed we will trade some initial complexity for an approach that can keep up with the complexities of dealing with committee data.","title":"Rationale"},{"location":"enhancement-proposals/004-committee-data/#drawbacks","text":"This approach is more complicated than it might otherwise be if we felt that relying entirely on scraped data (a la bills) was viable for committees. As a result it will require some additional upfront infrastructure work to achieve the hybrid scraped/manual data approach that is desired. There will be ongoing maintenance costs to the new infrastructure and scrapers required for this. As noted above plan does not account for historical committee information. Backwards Compatibility Note: The new committee scraping code will not be backwards compatible with the old. The API and website will be updated in a way that doesn't disturb any public interfaces.","title":"Drawbacks"},{"location":"enhancement-proposals/004-committee-data/#implementation-plan","text":"James will lead infrastructure implementation. Several team members, and hopefully some community members, will help to contribute updated committee scrapers.","title":"Implementation Plan"},{"location":"enhancement-proposals/004-committee-data/#copyright","text":"This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"Copyright"},{"location":"enhancement-proposals/005-dedupe-key/","text":"OSEP #5: Replacing pupa_id with dedupe_key \u00b6 Author(s) @jamesturk Implementer(s) @jamesturk Status Accepted Draft PR(s) https://github.com/openstates/enhancement-proposals/pull/19 Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Created 2021-04-01 Updated 2021-04-01 Abstract \u00b6 There are cases where the automated import pipeline cannot distinguish between two similar objects and relies upon a hint from the scraper to decide if an object should be treated as unique or not. Right now this is done by setting pupa_id, which is then stored in the pupa_identifier table with a join to the object in question. This logic is not well understood, and the naming was poorly thought out. This is a small OSEP that will fix this, making other deduplication work in the future easier to perform. Specification \u00b6 Scraped VoteEvents will gain a new field 'dedupe_key' which will be optional and function identically to pupa_id on import. Rationale \u00b6 The current name was never great, and as the last real reference to pupa makes it even more confusing to new people that aren't aware of the history. Fixing this code now will make other import-side improvements to come easier. Drawbacks \u00b6 Very few, a little disruptive to alter a bunch of scrapers at once, but the plan allows doing it incrementally if needed. This will be a minor change to the scrape output, which is the main reason this rises to the level of OSEP. This will not have any public facing impact. Implementation Plan \u00b6 This needs to be done in several steps to avoid disruption: addition of VoteEvent.dedupe_key in database write a migration script that copies data from the pupa_identifier table to VoteEvent.dedupe_key and drops the old pupa_identifier table write updated version of openstates-core which supports pupa_id in the scrape output but saves to dedupe_key on the backend when ready, near-simultaneous release of openstates-core and run of migration script update scrapers to not use pupa_id anymore once all scrapers updated, release updated version of openstates-core which no longer supports pupa_id in the scrape output James will handle all of this, once approved it should take 1-2 hours. Copyright \u00b6 This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"OSEP #5: Replacing pupa_id with dedupe_key"},{"location":"enhancement-proposals/005-dedupe-key/#osep-5-replacing-pupa_id-with-dedupe_key","text":"Author(s) @jamesturk Implementer(s) @jamesturk Status Accepted Draft PR(s) https://github.com/openstates/enhancement-proposals/pull/19 Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Created 2021-04-01 Updated 2021-04-01","title":"OSEP #5: Replacing pupa_id with dedupe_key"},{"location":"enhancement-proposals/005-dedupe-key/#abstract","text":"There are cases where the automated import pipeline cannot distinguish between two similar objects and relies upon a hint from the scraper to decide if an object should be treated as unique or not. Right now this is done by setting pupa_id, which is then stored in the pupa_identifier table with a join to the object in question. This logic is not well understood, and the naming was poorly thought out. This is a small OSEP that will fix this, making other deduplication work in the future easier to perform.","title":"Abstract"},{"location":"enhancement-proposals/005-dedupe-key/#specification","text":"Scraped VoteEvents will gain a new field 'dedupe_key' which will be optional and function identically to pupa_id on import.","title":"Specification"},{"location":"enhancement-proposals/005-dedupe-key/#rationale","text":"The current name was never great, and as the last real reference to pupa makes it even more confusing to new people that aren't aware of the history. Fixing this code now will make other import-side improvements to come easier.","title":"Rationale"},{"location":"enhancement-proposals/005-dedupe-key/#drawbacks","text":"Very few, a little disruptive to alter a bunch of scrapers at once, but the plan allows doing it incrementally if needed. This will be a minor change to the scrape output, which is the main reason this rises to the level of OSEP. This will not have any public facing impact.","title":"Drawbacks"},{"location":"enhancement-proposals/005-dedupe-key/#implementation-plan","text":"This needs to be done in several steps to avoid disruption: addition of VoteEvent.dedupe_key in database write a migration script that copies data from the pupa_identifier table to VoteEvent.dedupe_key and drops the old pupa_identifier table write updated version of openstates-core which supports pupa_id in the scrape output but saves to dedupe_key on the backend when ready, near-simultaneous release of openstates-core and run of migration script update scrapers to not use pupa_id anymore once all scrapers updated, release updated version of openstates-core which no longer supports pupa_id in the scrape output James will handle all of this, once approved it should take 1-2 hours.","title":"Implementation Plan"},{"location":"enhancement-proposals/005-dedupe-key/#copyright","text":"This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"Copyright"},{"location":"enhancement-proposals/006-new-people-offices/","text":"OSEP #6: New People Offices Schema \u00b6 Author(s) @jamesturk Implementer(s) @jamesturk Status Final Issue https://github.com/openstates/enhancement-proposals/issues/26 Draft PR(s) https://github.com/openstates/enhancement-proposals/pull/27 Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Created 2021-08-13 Updated 2021-10-13 Abstract \u00b6 Make improvements to how office/contact information is represented to allow better modeling of real world scenarios & reduce user confusion. Specification \u00b6 Currently Open States stores contact information in a highly normalized form: ContactDetail - type (address|email|url|fax|text|voice|video|pager|textphone) [1] - value - note - person_id This is based on the Popolo spec, which heavily influenced Open Civic Data's modeling. One reason for this was to avoid a bunch of null columns for lesser used values, but in practice, we only use voice , fax , and address . ( email was used but then moved to a top level field in early 2020). In practice though, we typically tie addresses together \"e.g. Capitol Office or District Office\". This is done via note , which is now required to be either Capitol Office , District Office , or Primary Office . This proposal suggests moving us towards a model that more accurately represents how most states provide (and users use) the data: Office - classification: capitol | district | primary - address - voice - fax - name (optional) With additional validation rules: - an office must have at least one of address , voice , fax . - At most one address may have the classification capitol or primary . - Any number of district addresses may be provided. - If name is not provided, an office will display as Capitol Office , District Office , or Primary Office as it does now (based on classification ) Rationale \u00b6 As noted above, we currently group address information into Capitol Office & District Office (& occasionally Primary Office). Doing so does not allow us to distinguish between multiple district offices, which are common in certain states (e.g. California). This leads to unclear pairings of data on OpenStates.org as well as an issue in API v3 which attempts to do this grouping for user convenience. Additionally, the code to merge scraped people is quite complex and has several conditions that cause it to bail & request manual resolution. These conditions are direct consequences of the current schema. For more clear examples of the issues presented, assume a person has the following ContactDetail records: - type=voice, note=District, value=555-555-1234 - type=address, note=District, value=123 Main Street - type=voice, note=District, value=555-555-6666 - type=address, note=District, value=1600 Pennsylvania Ave OpenStates.org will show two district offices, but there is no way to know which number corresponds to which address, so they will be grouped based on database retrieval order. API v3 will only show one district office, since the logic there is to attempt to convert ContactDetail records into more usable \"offices\" which works well for most cases, but the pairing information is not currently available in this case. Finally, the people merge code (openstates-core/people) will not be able to merge any changes that affect the District offices since it can not be confident which phone number was added/removed/changed. (Essentially due to missing a unique data path to the changed item, which is not an issue with any other piece of data at present.) Drawbacks \u00b6 We will lose the flexibility of supporting other contact detail types. Given that they haven't been used thus far makes that feel like a reasonable trade-off. This proposal will require consumers of the raw people data to update their code, as the openstates/people repository will need to be updated, with a migration script run to convert existing offices to the new format. Implementation Plan \u00b6 After approval, a script will be written to convert openstates/people to the new data format. All appropriate data scripts ( os-people to-database and merge ) will be updated, initially to import the data to both the old ContactDetail table as well as the new Offices table. Initially this means that upstream impact will be limited, this will give us time to update OpenStates.org, API v2, and v3. OpenStates.org's display logic will be changed to support the new offices data. From this point, the GraphQL API will be updated to support both forms with a transformation step to allow backwards compatibility. The new offices key will be added and queryable via GraphQL, whereas the old contactDetails key can be left intact as-is with a backwards-compatibility shim put in place. API v3 will be updated to query the database's offices key. Given that it already uses the offices key in responses, the new name & classification key will be the only impact there. After all of these changes are implemented, the old ContactDetail table can be dropped from the database & import process. Copyright \u00b6 This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"OSEP #6: New People Offices Schema"},{"location":"enhancement-proposals/006-new-people-offices/#osep-6-new-people-offices-schema","text":"Author(s) @jamesturk Implementer(s) @jamesturk Status Final Issue https://github.com/openstates/enhancement-proposals/issues/26 Draft PR(s) https://github.com/openstates/enhancement-proposals/pull/27 Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Created 2021-08-13 Updated 2021-10-13","title":"OSEP #6: New People Offices Schema"},{"location":"enhancement-proposals/006-new-people-offices/#abstract","text":"Make improvements to how office/contact information is represented to allow better modeling of real world scenarios & reduce user confusion.","title":"Abstract"},{"location":"enhancement-proposals/006-new-people-offices/#specification","text":"Currently Open States stores contact information in a highly normalized form: ContactDetail - type (address|email|url|fax|text|voice|video|pager|textphone) [1] - value - note - person_id This is based on the Popolo spec, which heavily influenced Open Civic Data's modeling. One reason for this was to avoid a bunch of null columns for lesser used values, but in practice, we only use voice , fax , and address . ( email was used but then moved to a top level field in early 2020). In practice though, we typically tie addresses together \"e.g. Capitol Office or District Office\". This is done via note , which is now required to be either Capitol Office , District Office , or Primary Office . This proposal suggests moving us towards a model that more accurately represents how most states provide (and users use) the data: Office - classification: capitol | district | primary - address - voice - fax - name (optional) With additional validation rules: - an office must have at least one of address , voice , fax . - At most one address may have the classification capitol or primary . - Any number of district addresses may be provided. - If name is not provided, an office will display as Capitol Office , District Office , or Primary Office as it does now (based on classification )","title":"Specification"},{"location":"enhancement-proposals/006-new-people-offices/#rationale","text":"As noted above, we currently group address information into Capitol Office & District Office (& occasionally Primary Office). Doing so does not allow us to distinguish between multiple district offices, which are common in certain states (e.g. California). This leads to unclear pairings of data on OpenStates.org as well as an issue in API v3 which attempts to do this grouping for user convenience. Additionally, the code to merge scraped people is quite complex and has several conditions that cause it to bail & request manual resolution. These conditions are direct consequences of the current schema. For more clear examples of the issues presented, assume a person has the following ContactDetail records: - type=voice, note=District, value=555-555-1234 - type=address, note=District, value=123 Main Street - type=voice, note=District, value=555-555-6666 - type=address, note=District, value=1600 Pennsylvania Ave OpenStates.org will show two district offices, but there is no way to know which number corresponds to which address, so they will be grouped based on database retrieval order. API v3 will only show one district office, since the logic there is to attempt to convert ContactDetail records into more usable \"offices\" which works well for most cases, but the pairing information is not currently available in this case. Finally, the people merge code (openstates-core/people) will not be able to merge any changes that affect the District offices since it can not be confident which phone number was added/removed/changed. (Essentially due to missing a unique data path to the changed item, which is not an issue with any other piece of data at present.)","title":"Rationale"},{"location":"enhancement-proposals/006-new-people-offices/#drawbacks","text":"We will lose the flexibility of supporting other contact detail types. Given that they haven't been used thus far makes that feel like a reasonable trade-off. This proposal will require consumers of the raw people data to update their code, as the openstates/people repository will need to be updated, with a migration script run to convert existing offices to the new format.","title":"Drawbacks"},{"location":"enhancement-proposals/006-new-people-offices/#implementation-plan","text":"After approval, a script will be written to convert openstates/people to the new data format. All appropriate data scripts ( os-people to-database and merge ) will be updated, initially to import the data to both the old ContactDetail table as well as the new Offices table. Initially this means that upstream impact will be limited, this will give us time to update OpenStates.org, API v2, and v3. OpenStates.org's display logic will be changed to support the new offices data. From this point, the GraphQL API will be updated to support both forms with a transformation step to allow backwards compatibility. The new offices key will be added and queryable via GraphQL, whereas the old contactDetails key can be left intact as-is with a backwards-compatibility shim put in place. API v3 will be updated to query the database's offices key. Given that it already uses the offices key in responses, the new name & classification key will be the only impact there. After all of these changes are implemented, the old ContactDetail table can be dropped from the database & import process.","title":"Implementation Plan"},{"location":"enhancement-proposals/006-new-people-offices/#copyright","text":"This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"Copyright"},{"location":"enhancement-proposals/007-events/","text":"OSEP #7: Restoring Events Data \u00b6 Author(s) @jamesturk Implementer(s) @jamesturk Status Final Issue https://github.com/openstates/enhancement-proposals/issues/34 Draft PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Created 2021-08-13 Updated 2021-10-13 Abstract \u00b6 From around 2010-2015 Open States scraped \"events\" data, mostly data on upcoming hearings and legislative meetings. That data has not been part of Open States' data offerings for quite a while, despite maintained scrapers in many states. This proposal would restore events data to Open States' public data offerings. Specification \u00b6 The existing events scrapers would regain \"first class\" status, and be run regularly (daily at least). The existing schema would be left mostly intact, and Event importers would be restored. Additional scrape jobs would be configured to run event scrapers regularly again & report failures similarly to how bill scrapers are configured today. API v3 will be updated to include event data. For now, OpenStates.org & API v2 will not be affected. Some changes will be introduced as part of this proposal: Schema Changes \u00b6 Scraped Events will gain the following optional fields: upstream_id : This can be used to record an upstream identifier, such as a database identifier that can be obtained from the source data. If present it will be used to uniquely identify events. (This is distinct from dedupe_key which has no semantic meaning and can be a constructed value or URL.) Soft Deletes on Import \u00b6 To better address downstream user's needs, when future events can not be found in the current scrape nor reconciled via the standard means ( dedupe_key , the new upstream_id , nor a match on the main attributes), the future event will be marked as deleted in the database. Deleted events will not be returned in the API response by default, but may be explicitly requested by clients. Windowing \u00b6 In some states we have to select a date range to scrape, which currently leads to ad hoc code (as described by Tim here: https://github.com/openstates/enhancement-proposals/pull/28#issuecomment-898720989) To bring some sense of standardization to this, we will standardize on the following parameters: today (default: today's date, can be overridden for testing purposes) days_before (default: 30) days_after (default: 90) All of which can be overridden on the command line. A scraper that supports this interface will call a utility function with these variables and obtain a start_date & end_date centered around the today value. The obtained start/end dates should then be used by the scraper to window its request to the source. Helper Methods \u00b6 The openstates.scrape.Event object will gain a couple of helper functions: Event.add_bill which will add an agenda item with configurable placeholder text (if one does not exist already) and associate a bill with that item. This will serve as a workaround to associate bills with events that do not have an appropriate agenda item already. Event.add_media_link will gain a classification parameter to classify the type of media being added. Options will include: 'video', 'audio', 'hosted video', 'hosted audio', 'youtube'. Rationale \u00b6 This is valuable data, and with renewed development resources, it has again become feasible to support maintaining it. Drawbacks \u00b6 The main drawback here is just in terms of resource allocation, supporting this across states will be significant work. There are no backwards compatibility concerns to address. Implementation Plan \u00b6 Tim has already been maintaining the events scrapers within the openstates-scrapers repository. Now they will be maintained by other members of the team as well. The additional work to support the import of events & access via the API will be done by James or others under his supervision. Copyright \u00b6 This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"OSEP #7: Restoring Events Data"},{"location":"enhancement-proposals/007-events/#osep-7-restoring-events-data","text":"Author(s) @jamesturk Implementer(s) @jamesturk Status Final Issue https://github.com/openstates/enhancement-proposals/issues/34 Draft PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Created 2021-08-13 Updated 2021-10-13","title":"OSEP #7: Restoring Events Data"},{"location":"enhancement-proposals/007-events/#abstract","text":"From around 2010-2015 Open States scraped \"events\" data, mostly data on upcoming hearings and legislative meetings. That data has not been part of Open States' data offerings for quite a while, despite maintained scrapers in many states. This proposal would restore events data to Open States' public data offerings.","title":"Abstract"},{"location":"enhancement-proposals/007-events/#specification","text":"The existing events scrapers would regain \"first class\" status, and be run regularly (daily at least). The existing schema would be left mostly intact, and Event importers would be restored. Additional scrape jobs would be configured to run event scrapers regularly again & report failures similarly to how bill scrapers are configured today. API v3 will be updated to include event data. For now, OpenStates.org & API v2 will not be affected. Some changes will be introduced as part of this proposal:","title":"Specification"},{"location":"enhancement-proposals/007-events/#schema-changes","text":"Scraped Events will gain the following optional fields: upstream_id : This can be used to record an upstream identifier, such as a database identifier that can be obtained from the source data. If present it will be used to uniquely identify events. (This is distinct from dedupe_key which has no semantic meaning and can be a constructed value or URL.)","title":"Schema Changes"},{"location":"enhancement-proposals/007-events/#soft-deletes-on-import","text":"To better address downstream user's needs, when future events can not be found in the current scrape nor reconciled via the standard means ( dedupe_key , the new upstream_id , nor a match on the main attributes), the future event will be marked as deleted in the database. Deleted events will not be returned in the API response by default, but may be explicitly requested by clients.","title":"Soft Deletes on Import"},{"location":"enhancement-proposals/007-events/#windowing","text":"In some states we have to select a date range to scrape, which currently leads to ad hoc code (as described by Tim here: https://github.com/openstates/enhancement-proposals/pull/28#issuecomment-898720989) To bring some sense of standardization to this, we will standardize on the following parameters: today (default: today's date, can be overridden for testing purposes) days_before (default: 30) days_after (default: 90) All of which can be overridden on the command line. A scraper that supports this interface will call a utility function with these variables and obtain a start_date & end_date centered around the today value. The obtained start/end dates should then be used by the scraper to window its request to the source.","title":"Windowing"},{"location":"enhancement-proposals/007-events/#helper-methods","text":"The openstates.scrape.Event object will gain a couple of helper functions: Event.add_bill which will add an agenda item with configurable placeholder text (if one does not exist already) and associate a bill with that item. This will serve as a workaround to associate bills with events that do not have an appropriate agenda item already. Event.add_media_link will gain a classification parameter to classify the type of media being added. Options will include: 'video', 'audio', 'hosted video', 'hosted audio', 'youtube'.","title":"Helper Methods"},{"location":"enhancement-proposals/007-events/#rationale","text":"This is valuable data, and with renewed development resources, it has again become feasible to support maintaining it.","title":"Rationale"},{"location":"enhancement-proposals/007-events/#drawbacks","text":"The main drawback here is just in terms of resource allocation, supporting this across states will be significant work. There are no backwards compatibility concerns to address.","title":"Drawbacks"},{"location":"enhancement-proposals/007-events/#implementation-plan","text":"Tim has already been maintaining the events scrapers within the openstates-scrapers repository. Now they will be maintained by other members of the team as well. The additional work to support the import of events & access via the API will be done by James or others under his supervision.","title":"Implementation Plan"},{"location":"enhancement-proposals/007-events/#copyright","text":"This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"Copyright"},{"location":"enhancement-proposals/008-active-sessions/","text":"OSEP #8: Active Sessions \u00b6 Author(s) @jamesturk Implementer(s) @jamesturk Status Final Issue https://github.com/openstates/enhancement-proposals/issues/29 Draft PR(s) https://github.com/openstates/enhancement-proposals/pull/30 Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Created 2021-08-16 Updated 2021-11-08 Abstract \u00b6 This proposal would add a flag to sessions indicating whether or not it is considered active or not. This information would be used for determining which sessions to scrape by default, and could be used within the openstates.org UI as well. Specification \u00b6 The Legislative Session schema will be updated to have a required field named active to indicate if the session is currently receiving updates. Other portions of the code will be updated to use this field as needed, most notably the code that selects which session to run if no session is given on the command line, and the OpenStates.org search logic. This field will also be surfaced in the APIs so that downstream consumers can make decisions based upon it as they wish. Additional care will be taken to add validation rules that either error out or warn loudly if the flag is potentially set incorrectly. Rationale \u00b6 This helps address two major problems that are present right now: - When scrapers are run, only the latest (by order of appearance in the metadata) session is scraped. Due to complex special session rules, that means that the active session information winds up within the task-definitions repository, where we add additional tasks for one-off re-scrapes, special sessions, etc. This also means when running locally, special care must be taken to run with the correct session=XYZ argument. - When doing things like full text search, that should default to the current session- we use the same rule (latest in the metadata) whereas it might be more beneficial to search within multiple sessions or a session that is not the latest but is more likely to contain the bill(s) being searched for. The main alternative considered was the idea to use begin & end dates instead (see https://github.com/openstates/enhancement-proposals/issues/29). It was pointed out that there are a lot of special cases which would render this solution ineffective, for example: - IL governor sometimes signs bills from the previous session even after the new one started, this year I think that happened as late as march 2022 after 2021 regular session close. - VA as of August 2021 is in a state where the Regular session is getting updates (from the executive) despite being adjourned. Special #1 is finished, and Special #2 is actively meeting. - Prefiles will need to be considered active before their start date to be scraped. These cases make using start & end date fraught, as we'd wind up needing to manipulate them to our needs or still falling back to overrides. Drawbacks \u00b6 This will be an extra piece of data to manually maintain across each state. It is somewhat redundant with the begin & end date (but not entirely, as noted above in Rationale). It is not unlikely that we'll run into issues where we forget to update the flag & continue to scrape old sessions. As noted in the specification, special care will be taken to try to add checks for this behavior that run alongside the session check code that runs at scraper start. Implementation Plan \u00b6 James will take the lead on adding this to openstates-core: - schema support for LegislativeSession.active - expose & document new flag in APIs - additional checks at scraper start that the active session list is sane - an automated update across all jurisdictions to set active sessions (defaulting to use current last-is-active logic) - fix os-update command to use new logic - fix OpenStates.org search to use active sessions by default instead of current logic which assesses latest session with any bills Copyright \u00b6 This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"OSEP #8: Active Sessions"},{"location":"enhancement-proposals/008-active-sessions/#osep-8-active-sessions","text":"Author(s) @jamesturk Implementer(s) @jamesturk Status Final Issue https://github.com/openstates/enhancement-proposals/issues/29 Draft PR(s) https://github.com/openstates/enhancement-proposals/pull/30 Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Created 2021-08-16 Updated 2021-11-08","title":"OSEP #8: Active Sessions"},{"location":"enhancement-proposals/008-active-sessions/#abstract","text":"This proposal would add a flag to sessions indicating whether or not it is considered active or not. This information would be used for determining which sessions to scrape by default, and could be used within the openstates.org UI as well.","title":"Abstract"},{"location":"enhancement-proposals/008-active-sessions/#specification","text":"The Legislative Session schema will be updated to have a required field named active to indicate if the session is currently receiving updates. Other portions of the code will be updated to use this field as needed, most notably the code that selects which session to run if no session is given on the command line, and the OpenStates.org search logic. This field will also be surfaced in the APIs so that downstream consumers can make decisions based upon it as they wish. Additional care will be taken to add validation rules that either error out or warn loudly if the flag is potentially set incorrectly.","title":"Specification"},{"location":"enhancement-proposals/008-active-sessions/#rationale","text":"This helps address two major problems that are present right now: - When scrapers are run, only the latest (by order of appearance in the metadata) session is scraped. Due to complex special session rules, that means that the active session information winds up within the task-definitions repository, where we add additional tasks for one-off re-scrapes, special sessions, etc. This also means when running locally, special care must be taken to run with the correct session=XYZ argument. - When doing things like full text search, that should default to the current session- we use the same rule (latest in the metadata) whereas it might be more beneficial to search within multiple sessions or a session that is not the latest but is more likely to contain the bill(s) being searched for. The main alternative considered was the idea to use begin & end dates instead (see https://github.com/openstates/enhancement-proposals/issues/29). It was pointed out that there are a lot of special cases which would render this solution ineffective, for example: - IL governor sometimes signs bills from the previous session even after the new one started, this year I think that happened as late as march 2022 after 2021 regular session close. - VA as of August 2021 is in a state where the Regular session is getting updates (from the executive) despite being adjourned. Special #1 is finished, and Special #2 is actively meeting. - Prefiles will need to be considered active before their start date to be scraped. These cases make using start & end date fraught, as we'd wind up needing to manipulate them to our needs or still falling back to overrides.","title":"Rationale"},{"location":"enhancement-proposals/008-active-sessions/#drawbacks","text":"This will be an extra piece of data to manually maintain across each state. It is somewhat redundant with the begin & end date (but not entirely, as noted above in Rationale). It is not unlikely that we'll run into issues where we forget to update the flag & continue to scrape old sessions. As noted in the specification, special care will be taken to try to add checks for this behavior that run alongside the session check code that runs at scraper start.","title":"Drawbacks"},{"location":"enhancement-proposals/008-active-sessions/#implementation-plan","text":"James will take the lead on adding this to openstates-core: - schema support for LegislativeSession.active - expose & document new flag in APIs - additional checks at scraper start that the active session list is sane - an automated update across all jurisdictions to set active sessions (defaulting to use current last-is-active logic) - fix os-update command to use new logic - fix OpenStates.org search to use active sessions by default instead of current logic which assesses latest session with any bills","title":"Implementation Plan"},{"location":"enhancement-proposals/008-active-sessions/#copyright","text":"This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"Copyright"},{"location":"enhancement-proposals/template/","text":"OSEP #: [TITLE] \u00b6 Author(s) TODO Implementer(s) TODO Status Draft Issue https://github.com/openstates/enhancement-proposals/issues/TBD Draft PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Created TODO Updated TODO Abstract \u00b6 TODO: Write a short summary of what you are hoping to achieve. Specification \u00b6 TODO: Describe how the proposal will work. Rationale \u00b6 TODO: Explain the reason for this in detail. Discuss alternatives considered. Drawbacks \u00b6 TODO: Will this affect backwards compatibility in scrapers, the API, or the website? (It is ok to leave this blank in a draft if you aren't sure.) Implementation Plan \u00b6 TODO: How will this be done? Are you volunteering to do it? Do you want someone else to do it? (It is ok to leave this blank in a draft if you aren't sure.) Copyright \u00b6 This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"OSEP #: [TITLE]"},{"location":"enhancement-proposals/template/#osep-title","text":"Author(s) TODO Implementer(s) TODO Status Draft Issue https://github.com/openstates/enhancement-proposals/issues/TBD Draft PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Approval PR(s) https://github.com/openstates/enhancement-proposals/pull/TBD Created TODO Updated TODO","title":"OSEP #: [TITLE]"},{"location":"enhancement-proposals/template/#abstract","text":"TODO: Write a short summary of what you are hoping to achieve.","title":"Abstract"},{"location":"enhancement-proposals/template/#specification","text":"TODO: Describe how the proposal will work.","title":"Specification"},{"location":"enhancement-proposals/template/#rationale","text":"TODO: Explain the reason for this in detail. Discuss alternatives considered.","title":"Rationale"},{"location":"enhancement-proposals/template/#drawbacks","text":"TODO: Will this affect backwards compatibility in scrapers, the API, or the website? (It is ok to leave this blank in a draft if you aren't sure.)","title":"Drawbacks"},{"location":"enhancement-proposals/template/#implementation-plan","text":"TODO: How will this be done? Are you volunteering to do it? Do you want someone else to do it? (It is ok to leave this blank in a draft if you aren't sure.)","title":"Implementation Plan"},{"location":"enhancement-proposals/template/#copyright","text":"This document has been placed in the public domain per the Creative Commons CC0 1.0 Universal license.","title":"Copyright"},{"location":"openstates.org/scheduled-tasks/","text":"Openstates.org scheduled tasks \u00b6 Openstates.org is a django application that has some simple scheduled tasks that execute using Django admin commands . Jobs and Locations \u00b6 Containerized versions of the actual scheduled jobs are managed in Github . These are not the jobs executed in production. We deploy jobs to the Openstates.org host using ansible . openstates.org runs in AWS. Access credentials/location/etc. can be found in the AWS console. Currently Scheduled Jobs \u00b6 Subscription Processing \u00b6 Defined here Tool that processes search subscriptions for users. Currently (2022-08-02) scheduled to run once a day (12:30 UTC) Aggregate API Usage \u00b6 Defined here Tool that generates some internal stats for user interactions with Openstates. Currently (2022-08-02) scheduled to run every 2 hours (39 */2) System Maintenance Jobs \u00b6 Let's Encrypt certificate collection/rotation Nginx maintenance (tied to certificate rotation)","title":"Openstates.org scheduled tasks"},{"location":"openstates.org/scheduled-tasks/#openstatesorg-scheduled-tasks","text":"Openstates.org is a django application that has some simple scheduled tasks that execute using Django admin commands .","title":"Openstates.org scheduled tasks"},{"location":"openstates.org/scheduled-tasks/#jobs-and-locations","text":"Containerized versions of the actual scheduled jobs are managed in Github . These are not the jobs executed in production. We deploy jobs to the Openstates.org host using ansible . openstates.org runs in AWS. Access credentials/location/etc. can be found in the AWS console.","title":"Jobs and Locations"},{"location":"openstates.org/scheduled-tasks/#currently-scheduled-jobs","text":"","title":"Currently Scheduled Jobs"},{"location":"openstates.org/scheduled-tasks/#subscription-processing","text":"Defined here Tool that processes search subscriptions for users. Currently (2022-08-02) scheduled to run once a day (12:30 UTC)","title":"Subscription Processing"},{"location":"openstates.org/scheduled-tasks/#aggregate-api-usage","text":"Defined here Tool that generates some internal stats for user interactions with Openstates. Currently (2022-08-02) scheduled to run every 2 hours (39 */2)","title":"Aggregate API Usage"},{"location":"openstates.org/scheduled-tasks/#system-maintenance-jobs","text":"Let's Encrypt certificate collection/rotation Nginx maintenance (tied to certificate rotation)","title":"System Maintenance Jobs"}]} \ No newline at end of file +{"config":{"indexing":"full","lang":["en"],"min_search_length":3,"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"Introduction \u00b6 Welcome to the Documentation for Open States. The sections below cover contributing to scrapers & data, and how to use the API. About Open States \u00b6 Open States strives to improve civic engagement at the state level by providing data and tools regarding state legislatures. We aim to serve members of the public, activist groups, journalists, and researchers with better data on what is happening in their state capital, and to provide tools to reduce barriers to participation and increase engagement. The project aggregates legislative information from all 50 states, Washington, D.C., and Puerto Rico. This information is then standardized, cleaned, and published to the public via OpenStates.org, a powerful API, and bulk downloads. All of this work is done in the open , and depends heavily on contributors striving to make this important resource available. Thank you for your interest in our community, whether you are looking to use our bulk data or APIs or interested in contributing data or code, we're glad to have you here. Communication \u00b6 When joining a new community, it can be tough to figure out where to ask questions, provide feedback, or help out. Don't worry! As long as you're respectful and follow our Code of Conduct , we're happy to have you! Here are some guidelines regarding the best way to get in touch or contribute. Do note that Open States is a volunteer-powered project, and all of the core developers have day jobs; we're excited to talk to you, but it may sometimes take a bit of time to get back to you. Recommendations \u00b6 Want to ask a general question, have a conversation, or keep up with the community? The best place is our Discussion Forum . This is the main place for Open States discussion. The core team and many other contributors are present there, and we're usually able to answer questions in a timely fashion. Have a private question, or a security concern? Email contact@openstates.org ; only the administrative team can see these. Have you found an error or issue in the Open States data? Have a technical issue not related to the data itself? File an issue on our bug tracker . And before you do, quickly check whether anyone else there has already reported the same bug. Want to contribute to the project more regularly? We also have a Slack that you can join. It is not the best place for asking questions better handled by filing an issue or opening a conversation on GitHub, but you're welcome to join if you're interested in being a part of the community . Discouraged Methods of Communication \u00b6 Please avoid using these channels to get in touch with us: Personal email addresses of Open States developers Please respect our boundaries & refrain from contacting any of the developers directly, unless we ask you to do so. Twitter (or any other social media) We mainly use the @openstates twitter account to make announcements, and don't have the resources to provide technical support or other feedback on Twitter.","title":"Introduction"},{"location":"#introduction","text":"Welcome to the Documentation for Open States. The sections below cover contributing to scrapers & data, and how to use the API.","title":"Introduction"},{"location":"#about-open-states","text":"Open States strives to improve civic engagement at the state level by providing data and tools regarding state legislatures. We aim to serve members of the public, activist groups, journalists, and researchers with better data on what is happening in their state capital, and to provide tools to reduce barriers to participation and increase engagement. The project aggregates legislative information from all 50 states, Washington, D.C., and Puerto Rico. This information is then standardized, cleaned, and published to the public via OpenStates.org, a powerful API, and bulk downloads. All of this work is done in the open , and depends heavily on contributors striving to make this important resource available. Thank you for your interest in our community, whether you are looking to use our bulk data or APIs or interested in contributing data or code, we're glad to have you here.","title":"About Open States"},{"location":"#communication","text":"When joining a new community, it can be tough to figure out where to ask questions, provide feedback, or help out. Don't worry! As long as you're respectful and follow our Code of Conduct , we're happy to have you! Here are some guidelines regarding the best way to get in touch or contribute. Do note that Open States is a volunteer-powered project, and all of the core developers have day jobs; we're excited to talk to you, but it may sometimes take a bit of time to get back to you.","title":"Communication"},{"location":"#recommendations","text":"Want to ask a general question, have a conversation, or keep up with the community? The best place is our Discussion Forum . This is the main place for Open States discussion. The core team and many other contributors are present there, and we're usually able to answer questions in a timely fashion. Have a private question, or a security concern? Email contact@openstates.org ; only the administrative team can see these. Have you found an error or issue in the Open States data? Have a technical issue not related to the data itself? File an issue on our bug tracker . And before you do, quickly check whether anyone else there has already reported the same bug. Want to contribute to the project more regularly? We also have a Slack that you can join. It is not the best place for asking questions better handled by filing an issue or opening a conversation on GitHub, but you're welcome to join if you're interested in being a part of the community .","title":"Recommendations"},{"location":"#discouraged-methods-of-communication","text":"Please avoid using these channels to get in touch with us: Personal email addresses of Open States developers Please respect our boundaries & refrain from contacting any of the developers directly, unless we ask you to do so. Twitter (or any other social media) We mainly use the @openstates twitter account to make announcements, and don't have the resources to provide technical support or other feedback on Twitter.","title":"Discouraged Methods of Communication"},{"location":"code-of-conduct/","text":"Code of Conduct \u00b6 Our Pledge \u00b6 We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. Our Standards \u00b6 Examples of behavior that contributes to a positive environment for our community include: Demonstrating empathy and kindness toward other people Being respectful of differing opinions, viewpoints, and experiences Giving and gracefully accepting constructive feedback Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behavior include: The use of sexualized language or imagery, and sexual attention or advances of any kind Trolling, insulting or derogatory comments, and personal or political attacks Public or private harassment Publishing others' private information, such as a physical or email address, without their explicit permission Other conduct which could reasonably be considered inappropriate in a professional setting Enforcement Responsibilities \u00b6 Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. Scope \u00b6 This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Enforcement \u00b6 Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at contact@openstates.org . All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter of any incident. Enforcement Guidelines \u00b6 Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: 1. Correction \u00b6 Community Impact : Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. Consequence : A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. 2. Warning \u00b6 Community Impact : A violation through a single incident or series of actions. Consequence : A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. 3. Temporary Ban \u00b6 Community Impact : A serious violation of community standards, including sustained inappropriate behavior. Consequence : A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. 4. Permanent Ban \u00b6 Community Impact : Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. Consequence : A permanent ban from any sort of public interaction within the project community. Attribution \u00b6 This Code of Conduct is adapted from the Contributor Covenant , version 2.0, available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html . Community Impact Guidelines were inspired by Mozilla's code of conduct enforcement ladder . For answers to common questions about this code of conduct, see the FAQ at https://www.contributor-covenant.org/faq . Translations are available at https://www.contributor-covenant.org/translations .","title":"Code of Conduct"},{"location":"code-of-conduct/#code-of-conduct","text":"","title":"Code of Conduct"},{"location":"code-of-conduct/#our-pledge","text":"We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.","title":"Our Pledge"},{"location":"code-of-conduct/#our-standards","text":"Examples of behavior that contributes to a positive environment for our community include: Demonstrating empathy and kindness toward other people Being respectful of differing opinions, viewpoints, and experiences Giving and gracefully accepting constructive feedback Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behavior include: The use of sexualized language or imagery, and sexual attention or advances of any kind Trolling, insulting or derogatory comments, and personal or political attacks Public or private harassment Publishing others' private information, such as a physical or email address, without their explicit permission Other conduct which could reasonably be considered inappropriate in a professional setting","title":"Our Standards"},{"location":"code-of-conduct/#enforcement-responsibilities","text":"Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate.","title":"Enforcement Responsibilities"},{"location":"code-of-conduct/#scope","text":"This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event.","title":"Scope"},{"location":"code-of-conduct/#enforcement","text":"Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at contact@openstates.org . All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter of any incident.","title":"Enforcement"},{"location":"code-of-conduct/#enforcement-guidelines","text":"Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct:","title":"Enforcement Guidelines"},{"location":"code-of-conduct/#1-correction","text":"Community Impact : Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. Consequence : A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested.","title":"1. Correction"},{"location":"code-of-conduct/#2-warning","text":"Community Impact : A violation through a single incident or series of actions. Consequence : A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban.","title":"2. Warning"},{"location":"code-of-conduct/#3-temporary-ban","text":"Community Impact : A serious violation of community standards, including sustained inappropriate behavior. Consequence : A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban.","title":"3. Temporary Ban"},{"location":"code-of-conduct/#4-permanent-ban","text":"Community Impact : Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. Consequence : A permanent ban from any sort of public interaction within the project community.","title":"4. Permanent Ban"},{"location":"code-of-conduct/#attribution","text":"This Code of Conduct is adapted from the Contributor Covenant , version 2.0, available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html . Community Impact Guidelines were inspired by Mozilla's code of conduct enforcement ladder . For answers to common questions about this code of conduct, see the FAQ at https://www.contributor-covenant.org/faq . Translations are available at https://www.contributor-covenant.org/translations .","title":"Attribution"},{"location":"api-v2/","text":"GraphQL API \u00b6 API keys are required. You can register for an API key and once activated, you'll pass your API key via the X-API-KEY header. You can also check out our introductory blog post for more details. Basics \u00b6 This is a GraphQL API, and some of the concepts may seem unfamiliar at first. There is in essence, only one endpoint: https://openstates.org/graphql . This endpoint, when accessed in a browser, will provide an interface that allows you to experiment with queries in the browser, it features autocomplete and a way to browse the full graph (click the \\'Docs\\' link in the upper right corner). A GraphQL query mirrors the structure of the data that you\\'d like to obtain. For example, to obtain a list of legislators you\\'d pass something like: { people { edges { node { name } } } } Note If you are using the API programatically it is recommended you send the data as part of the POST body, e.g.: curl -X POST https://openstates.org/graphql -d \"query={people{edges{node{name}}}}\" Of course, if you try this you'll see it doesn't work since there are some basic limits on how much data you can request at once. We paginate with the first , last , before and after parameters to a root node. So let's try that again: { people(first: 3) { edges { node { name } } } } And you'd get back JSON like: { \"data\": { \"people\": { \"edges\": [ { \"node\": { \"name\": \"Lydia Brasch\" } }, { \"node\": { \"name\": \"Matt Williams\" } }, { \"node\": { \"name\": \"Merv Riepe\" } } ] } } } Ah, much better. Nodes also can take other parameters to filter the returned content. Let's try the \"name\" filter which restricts our search to people named Lydia: { people(first: 3, name: \"Lydia\") { edges { node { name } } } } Results in: { \"data\": { \"people\": { \"edges\": [ { \"node\": { \"name\": \"Lydia Brasch\" } }, { \"node\": { \"name\": \"Lydia Graves Chassaniol\" } }, { \"node\": { \"name\": \"Lydia C. Blume\" } } ] } } } It is also possible to request data from multiple root nodes at once, for example: { people(first: 1) { edges { node { name } } } bills(first: 1) { edges { node { title } } } } Would give back something like: { \"data\": { \"people\": { \"edges\": [ { \"node\": { \"name\": \"Lydia Brasch\" } } ] }, \"bills\": { \"edges\": [ { \"node\": { \"title\": \"Criminal Law - Animal Abuse Emergency Compensation Fund - Establishment\" } } ] } } } You may notice something here, that you get back just the data you need. This is extremely powerful, and lets you do the equivalent of many traditional API calls in a single query. Full-fledged Example \u00b6 Let's take a look at a more useful example: { bill(jurisdiction: \"New York\", session: \"2017-2018\", identifier: \"S 5772\") { title actions { description date } votes { edges { node { counts { value option } votes { voterName voter { id contactDetails { value note type } } option } } } } sources { url } createdAt updatedAt } } There's a lot going on there, let's break it down: bill(jurisdiction: \"New York\", session: \"2017-2018\", identifier: \"S 5772\") { We're hitting the bill root node, which takes 3 parameters. This should get us to a single bill from New York. title This is going to give us the title, just like we saw before. actions { description date } Here we're going into a child node, in this case all of the actions taken on the bill. For each action we're requesting a the date & description. votes { edges { node { Here too we're going into a child node, but note that this time we use that \"edges\" and \"node\" pattern that we see on root level nodes. Certain child nodes in the API have the ability to be paginated or further limited, and votes happen to be one of them. In this case however we're not making use of that so we'll just ignore this. (A full discussion of this pattern is out of scope but check out the Relay pagination specification for more detail for more.) counts { value option } votes { voterName voter { id contactDetails { value note type } } option } } Here we grab a few more fields, including child nodes of each vote on our Bill. First, we get a list of counts (essentially pairs of outcomes + numbers e.g. (yes, 31), (no, 5)) We also get individual legislator votes by name, and we traverse into another object to get the Open States ID and contact details for the voter. (Don't sweat the exact data model here, there will be more on the structure once we get to the actual graph documentation.) sources { url } createdAt updatedAt And back up at the top level, we grab a few more pieces of information about the Bill. And now you've seen a glimpse of the power of this API. We were able to get back theexact fields we wanted on a bill, contact information on the legislators that have voted on the bill, and more. Our result looks like this: { \"data\": { \"bill\": { \"title\": \"Relates to bureaus of administrative adjudication\", \"actions\": [ { \"description\": \"REFERRED TO LOCAL GOVERNMENT\", \"date\": \"2017-04-28\" }, { \"description\": \"COMMITTEE DISCHARGED AND COMMITTED TO RULES\", \"date\": \"2017-06-19\" }, { \"description\": \"ORDERED TO THIRD READING CAL.1896\", \"date\": \"2017-06-19\" }, { \"description\": \"RECOMMITTED TO RULES\", \"date\": \"2017-06-21\" } ], \"votes\": { \"edges\": [ { \"node\": { \"counts\": [ { \"value\": 25, \"option\": \"yes\" }, { \"value\": 0, \"option\": \"no\" }, { \"value\": 0, \"option\": \"other\" } ], \"votes\": [ { \"voterName\": \"John J. Bonacic\", \"voter\": { \"id\": \"ocd-person/da013cd5-dc67-4e65-a310-73aa32ad1f7c\" \"contactDetails\": [ { \"value\": \"bonacic@nysenate.gov\", \"note\": \"Capitol Office\", \"type\": \"email\" }, { \"value\": \"Room 503\\nAlbany, NY 12247\", \"note\": \"District Office\", \"type\": \"address\" }, { \"value\": \"518-455-3181\", \"note\": \"District Office\", \"type\": \"voice\" }, ...etc... ] }, \"option\": \"yes\" }, { \"voterName\": \"Neil D. Breslin\", \"voter\": { \"id\": \"ocd-person/4b710aee-1b99-42e0-90e2-d41338e8c5df\" \"contactDetails\": [ ...etc... ], }, \"option\": \"yes\" }, { \"voterName\": \"David Carlucci\", \"voter\": { \"id\": \"ocd-person/1b0feab9-02a7-4bcc-b089-3ab23286da68\" \"contactDetails\": [ ...etc... ], }, \"option\": \"yes\" }, ] } }, ...etc... ] }, \"sources\": [ { \"url\": \"http://legislation.nysenate.gov/api/3/bills/2017-2018/S5772?summary=&detail=\" }, { \"url\": \"http://www.nysenate.gov/legislation/bills/2017/S5772\" }, { \"url\": \"http://assembly.state.ny.us/leg/?default_fld=&bn=S5772&Summary=Y&Actions=Y&Text=Y\" } ], \"createdAt\": \"2017-07-15 05:08:15.848526+00:00\", \"updatedAt\": \"2017-07-15 05:08:15.848541+00:00\" } } }","title":"GraphQL API"},{"location":"api-v2/#graphql-api","text":"API keys are required. You can register for an API key and once activated, you'll pass your API key via the X-API-KEY header. You can also check out our introductory blog post for more details.","title":"GraphQL API"},{"location":"api-v2/#basics","text":"This is a GraphQL API, and some of the concepts may seem unfamiliar at first. There is in essence, only one endpoint: https://openstates.org/graphql . This endpoint, when accessed in a browser, will provide an interface that allows you to experiment with queries in the browser, it features autocomplete and a way to browse the full graph (click the \\'Docs\\' link in the upper right corner). A GraphQL query mirrors the structure of the data that you\\'d like to obtain. For example, to obtain a list of legislators you\\'d pass something like: { people { edges { node { name } } } } Note If you are using the API programatically it is recommended you send the data as part of the POST body, e.g.: curl -X POST https://openstates.org/graphql -d \"query={people{edges{node{name}}}}\" Of course, if you try this you'll see it doesn't work since there are some basic limits on how much data you can request at once. We paginate with the first , last , before and after parameters to a root node. So let's try that again: { people(first: 3) { edges { node { name } } } } And you'd get back JSON like: { \"data\": { \"people\": { \"edges\": [ { \"node\": { \"name\": \"Lydia Brasch\" } }, { \"node\": { \"name\": \"Matt Williams\" } }, { \"node\": { \"name\": \"Merv Riepe\" } } ] } } } Ah, much better. Nodes also can take other parameters to filter the returned content. Let's try the \"name\" filter which restricts our search to people named Lydia: { people(first: 3, name: \"Lydia\") { edges { node { name } } } } Results in: { \"data\": { \"people\": { \"edges\": [ { \"node\": { \"name\": \"Lydia Brasch\" } }, { \"node\": { \"name\": \"Lydia Graves Chassaniol\" } }, { \"node\": { \"name\": \"Lydia C. Blume\" } } ] } } } It is also possible to request data from multiple root nodes at once, for example: { people(first: 1) { edges { node { name } } } bills(first: 1) { edges { node { title } } } } Would give back something like: { \"data\": { \"people\": { \"edges\": [ { \"node\": { \"name\": \"Lydia Brasch\" } } ] }, \"bills\": { \"edges\": [ { \"node\": { \"title\": \"Criminal Law - Animal Abuse Emergency Compensation Fund - Establishment\" } } ] } } } You may notice something here, that you get back just the data you need. This is extremely powerful, and lets you do the equivalent of many traditional API calls in a single query.","title":"Basics"},{"location":"api-v2/#full-fledged-example","text":"Let's take a look at a more useful example: { bill(jurisdiction: \"New York\", session: \"2017-2018\", identifier: \"S 5772\") { title actions { description date } votes { edges { node { counts { value option } votes { voterName voter { id contactDetails { value note type } } option } } } } sources { url } createdAt updatedAt } } There's a lot going on there, let's break it down: bill(jurisdiction: \"New York\", session: \"2017-2018\", identifier: \"S 5772\") { We're hitting the bill root node, which takes 3 parameters. This should get us to a single bill from New York. title This is going to give us the title, just like we saw before. actions { description date } Here we're going into a child node, in this case all of the actions taken on the bill. For each action we're requesting a the date & description. votes { edges { node { Here too we're going into a child node, but note that this time we use that \"edges\" and \"node\" pattern that we see on root level nodes. Certain child nodes in the API have the ability to be paginated or further limited, and votes happen to be one of them. In this case however we're not making use of that so we'll just ignore this. (A full discussion of this pattern is out of scope but check out the Relay pagination specification for more detail for more.) counts { value option } votes { voterName voter { id contactDetails { value note type } } option } } Here we grab a few more fields, including child nodes of each vote on our Bill. First, we get a list of counts (essentially pairs of outcomes + numbers e.g. (yes, 31), (no, 5)) We also get individual legislator votes by name, and we traverse into another object to get the Open States ID and contact details for the voter. (Don't sweat the exact data model here, there will be more on the structure once we get to the actual graph documentation.) sources { url } createdAt updatedAt And back up at the top level, we grab a few more pieces of information about the Bill. And now you've seen a glimpse of the power of this API. We were able to get back theexact fields we wanted on a bill, contact information on the legislators that have voted on the bill, and more. Our result looks like this: { \"data\": { \"bill\": { \"title\": \"Relates to bureaus of administrative adjudication\", \"actions\": [ { \"description\": \"REFERRED TO LOCAL GOVERNMENT\", \"date\": \"2017-04-28\" }, { \"description\": \"COMMITTEE DISCHARGED AND COMMITTED TO RULES\", \"date\": \"2017-06-19\" }, { \"description\": \"ORDERED TO THIRD READING CAL.1896\", \"date\": \"2017-06-19\" }, { \"description\": \"RECOMMITTED TO RULES\", \"date\": \"2017-06-21\" } ], \"votes\": { \"edges\": [ { \"node\": { \"counts\": [ { \"value\": 25, \"option\": \"yes\" }, { \"value\": 0, \"option\": \"no\" }, { \"value\": 0, \"option\": \"other\" } ], \"votes\": [ { \"voterName\": \"John J. Bonacic\", \"voter\": { \"id\": \"ocd-person/da013cd5-dc67-4e65-a310-73aa32ad1f7c\" \"contactDetails\": [ { \"value\": \"bonacic@nysenate.gov\", \"note\": \"Capitol Office\", \"type\": \"email\" }, { \"value\": \"Room 503\\nAlbany, NY 12247\", \"note\": \"District Office\", \"type\": \"address\" }, { \"value\": \"518-455-3181\", \"note\": \"District Office\", \"type\": \"voice\" }, ...etc... ] }, \"option\": \"yes\" }, { \"voterName\": \"Neil D. Breslin\", \"voter\": { \"id\": \"ocd-person/4b710aee-1b99-42e0-90e2-d41338e8c5df\" \"contactDetails\": [ ...etc... ], }, \"option\": \"yes\" }, { \"voterName\": \"David Carlucci\", \"voter\": { \"id\": \"ocd-person/1b0feab9-02a7-4bcc-b089-3ab23286da68\" \"contactDetails\": [ ...etc... ], }, \"option\": \"yes\" }, ] } }, ...etc... ] }, \"sources\": [ { \"url\": \"http://legislation.nysenate.gov/api/3/bills/2017-2018/S5772?summary=&detail=\" }, { \"url\": \"http://www.nysenate.gov/legislation/bills/2017/S5772\" }, { \"url\": \"http://assembly.state.ny.us/leg/?default_fld=&bn=S5772&Summary=Y&Actions=Y&Text=Y\" } ], \"createdAt\": \"2017-07-15 05:08:15.848526+00:00\", \"updatedAt\": \"2017-07-15 05:08:15.848541+00:00\" } } }","title":"Full-fledged Example"},{"location":"api-v2/changelog/","text":"Changelog \u00b6 Changelog for Open States GraphQL API: v2.6 (March 2021) \u00b6 added preliminary support for federal jurisdiction added Jurisdiction.classification node Jurisdictions are now filterable by classification (municipal, state, country) v2.5 (July 2020) \u00b6 added Jurisdiction.lastScrapedAt, openstates/issues#32 v2.4 (April 2020) \u00b6 removed unused fields from graph (organization.links, organization.other_names) v2.3 (August 2019) \u00b6 add experimental full text search via searchQuery parameter to bills node v2.2 (June 2019) \u00b6 add openstatesUrl to bills query speed improvments v2.1 (Feb 2019) \u00b6 fix lat-lon behavior to limit to active memberships improve handling of retired legislators fix type of maximum_memberships bill version ordering is now consistent v2.0 (January 2019) \u00b6 bugfix for maximum_memberships type bugfix for versions field improve tests Beta Release (November 2018) \u00b6 API Keys are now required consider classification when using current_memberships fix geo filtering add openstatesUrl to Bill node for ease of linkage to OpenStates.org add Person.oldMemberships as analog to currentMemberships add actionSince filter to bills node fix 500 errors/optimization when using GraphQL fragments addition of basic protection for excessive queries add totalCount to assist in pagination add Organization.currentMemberships Preview Release 1 (May 2018) \u00b6 fix for people pagination add updatedSince for people add sponsor argument for bills node allow votes to take pagination parameters allow traversing to votes from person Preview Release 0 (Dec 2017) \u00b6 Initial draft release of the API, no backwards-compatibility guarantee made.","title":"Changelog"},{"location":"api-v2/changelog/#changelog","text":"Changelog for Open States GraphQL API:","title":"Changelog"},{"location":"api-v2/changelog/#v26-march-2021","text":"added preliminary support for federal jurisdiction added Jurisdiction.classification node Jurisdictions are now filterable by classification (municipal, state, country)","title":"v2.6 (March 2021)"},{"location":"api-v2/changelog/#v25-july-2020","text":"added Jurisdiction.lastScrapedAt, openstates/issues#32","title":"v2.5 (July 2020)"},{"location":"api-v2/changelog/#v24-april-2020","text":"removed unused fields from graph (organization.links, organization.other_names)","title":"v2.4 (April 2020)"},{"location":"api-v2/changelog/#v23-august-2019","text":"add experimental full text search via searchQuery parameter to bills node","title":"v2.3 (August 2019)"},{"location":"api-v2/changelog/#v22-june-2019","text":"add openstatesUrl to bills query speed improvments","title":"v2.2 (June 2019)"},{"location":"api-v2/changelog/#v21-feb-2019","text":"fix lat-lon behavior to limit to active memberships improve handling of retired legislators fix type of maximum_memberships bill version ordering is now consistent","title":"v2.1 (Feb 2019)"},{"location":"api-v2/changelog/#v20-january-2019","text":"bugfix for maximum_memberships type bugfix for versions field improve tests","title":"v2.0 (January 2019)"},{"location":"api-v2/changelog/#beta-release-november-2018","text":"API Keys are now required consider classification when using current_memberships fix geo filtering add openstatesUrl to Bill node for ease of linkage to OpenStates.org add Person.oldMemberships as analog to currentMemberships add actionSince filter to bills node fix 500 errors/optimization when using GraphQL fragments addition of basic protection for excessive queries add totalCount to assist in pagination add Organization.currentMemberships","title":"Beta Release (November 2018)"},{"location":"api-v2/changelog/#preview-release-1-may-2018","text":"fix for people pagination add updatedSince for people add sponsor argument for bills node allow votes to take pagination parameters allow traversing to votes from person","title":"Preview Release 1 (May 2018)"},{"location":"api-v2/changelog/#preview-release-0-dec-2017","text":"Initial draft release of the API, no backwards-compatibility guarantee made.","title":"Preview Release 0 (Dec 2017)"},{"location":"api-v2/examples/","text":"Examples \u00b6 Get basic information for all legislatures \u00b6 See in GraphiQL { jurisdictions { edges { node { name legislativeSessions { edges { node { name } } } legislature: organizations(classification: \"legislature\", first: 1) { edges { node { name classification children(first: 5) { edges { node { name classification } } } } } } } } } } Get overview of a legislature\\'s structure \u00b6 See in GraphiQL { jurisdiction(name: \"North Dakota\") { name url legislativeSessions { edges { node { name identifier } } } organizations(classification: \"legislature\", first: 1) { edges { node { id name children(first: 20) { edges { node { name } } } } } } } } Search for bills that match a given condition \u00b6 See in GraphiQL { search_1: bills(first: 5, jurisdiction: \"New York\", session: \"2017-2018\", chamber: \"lower\", classification: \"resolution\", updatedSince: \"2017-01-15\") { edges { node { id identifier title classification updatedAt createdAt legislativeSession { identifier jurisdiction { name } } actions { date description classification } documents { date note links { url } } versions { date note links { url } } sources { url note } } } } } Get all information on a particular bill \u00b6 See in GraphiQL { b1: bill(jurisdiction: \"Hawaii\", session: \"2017 Regular Session\", identifier: \"HB 475\") { id identifier title classification updatedAt createdAt legislativeSession { identifier jurisdiction { name } } actions { date description classification } documents { date note links { url } } versions { date note links { url } } sources { url note } } b2: bill(id: \"ocd-bill/9c24aaa2-6acc-43ad-883b-ae9f677062e9\") { id identifier title classification updatedAt createdAt legislativeSession { identifier jurisdiction { name } } actions { date description classification } documents { date note links { url } } versions { date note links { url } } sources { url note } } } Get information about a specific legislator \u00b6 See in GraphiQL { person(id:\"ocd-person/dd05bd23-fe49-4e65-bfff-62db997e56e0\"){ name contactDetails { note type value } otherNames { name } sources { url } currentMemberships { organization { name } } } } Get legislators for a given state/chamber \u00b6 See in GraphiQL ocd-organization/ddf820b5-5246-46b3-a807-99b5914ad39f is the id of the Alabama Senate chamber. { people(memberOf:\"ocd-organization/ddf820b5-5246-46b3-a807-99b5914ad39f\", first: 100) { edges { node { name party: currentMemberships(classification:\"party\") { organization { name } } links { url } sources { url } chamber: currentMemberships(classification:[\"upper\", \"lower\"]) { post { label } organization { name classification parent { name } } } } } } } Search for legislators that represent a given area \u00b6 See in GraphQL { people(latitude: 40.7460022, longitude: -73.9584642, first: 100) { edges { node { name chamber: currentMemberships(classification:[\"upper\", \"lower\"]) { post { label } organization { name classification parent { name } } } } } } }","title":"Examples"},{"location":"api-v2/examples/#examples","text":"","title":"Examples"},{"location":"api-v2/examples/#get-basic-information-for-all-legislatures","text":"See in GraphiQL { jurisdictions { edges { node { name legislativeSessions { edges { node { name } } } legislature: organizations(classification: \"legislature\", first: 1) { edges { node { name classification children(first: 5) { edges { node { name classification } } } } } } } } } }","title":"Get basic information for all legislatures"},{"location":"api-v2/examples/#get-overview-of-a-legislatures-structure","text":"See in GraphiQL { jurisdiction(name: \"North Dakota\") { name url legislativeSessions { edges { node { name identifier } } } organizations(classification: \"legislature\", first: 1) { edges { node { id name children(first: 20) { edges { node { name } } } } } } } }","title":"Get overview of a legislature\\'s structure"},{"location":"api-v2/examples/#search-for-bills-that-match-a-given-condition","text":"See in GraphiQL { search_1: bills(first: 5, jurisdiction: \"New York\", session: \"2017-2018\", chamber: \"lower\", classification: \"resolution\", updatedSince: \"2017-01-15\") { edges { node { id identifier title classification updatedAt createdAt legislativeSession { identifier jurisdiction { name } } actions { date description classification } documents { date note links { url } } versions { date note links { url } } sources { url note } } } } }","title":"Search for bills that match a given condition"},{"location":"api-v2/examples/#get-all-information-on-a-particular-bill","text":"See in GraphiQL { b1: bill(jurisdiction: \"Hawaii\", session: \"2017 Regular Session\", identifier: \"HB 475\") { id identifier title classification updatedAt createdAt legislativeSession { identifier jurisdiction { name } } actions { date description classification } documents { date note links { url } } versions { date note links { url } } sources { url note } } b2: bill(id: \"ocd-bill/9c24aaa2-6acc-43ad-883b-ae9f677062e9\") { id identifier title classification updatedAt createdAt legislativeSession { identifier jurisdiction { name } } actions { date description classification } documents { date note links { url } } versions { date note links { url } } sources { url note } } }","title":"Get all information on a particular bill"},{"location":"api-v2/examples/#get-information-about-a-specific-legislator","text":"See in GraphiQL { person(id:\"ocd-person/dd05bd23-fe49-4e65-bfff-62db997e56e0\"){ name contactDetails { note type value } otherNames { name } sources { url } currentMemberships { organization { name } } } }","title":"Get information about a specific legislator"},{"location":"api-v2/examples/#get-legislators-for-a-given-statechamber","text":"See in GraphiQL ocd-organization/ddf820b5-5246-46b3-a807-99b5914ad39f is the id of the Alabama Senate chamber. { people(memberOf:\"ocd-organization/ddf820b5-5246-46b3-a807-99b5914ad39f\", first: 100) { edges { node { name party: currentMemberships(classification:\"party\") { organization { name } } links { url } sources { url } chamber: currentMemberships(classification:[\"upper\", \"lower\"]) { post { label } organization { name classification parent { name } } } } } } }","title":"Get legislators for a given state/chamber"},{"location":"api-v2/examples/#search-for-legislators-that-represent-a-given-area","text":"See in GraphQL { people(latitude: 40.7460022, longitude: -73.9584642, first: 100) { edges { node { name chamber: currentMemberships(classification:[\"upper\", \"lower\"]) { post { label } organization { name classification parent { name } } } } } } }","title":"Search for legislators that represent a given area"},{"location":"api-v2/other/","text":"Other Notes \u00b6 There are a few other things to be aware of while using the API: Explore the Graph \u00b6 GraphQL is still quite new, so we figured it might be good to provide some helpful tips on how to think about the data and how you'll use the API. First, it is probably well worth your time to play around in GraphiQL to explore the API and data. It was heavily used when developing the API and writing tests, and is a very powerful tool, particularly when you make use of the self-documenting nature of the graph. When you're thinking about how to query don't necessarily try to replicate your old API calls exactly. For example, perhaps you were grabbing all bills that met a given criteria and then grabbing all sponsors contact details. This can now be done in one call by traversing from the bills-root root node into the BillSponsorshipNode and then up to the PersonNode and finally to the ContactDetailNode This may sound complex at first, but once you get the hang of it, it really does unlock a ton of power and will make your apps more powerful and efficient. Pagination \u00b6 In several places (such as the bills-root and BillNode 's votes ) we mention that nodes are paginated. What this means in practice is that instead of getting back the underlying node type, say BillNode , directly, you'll get back BillConnectionNode or similar. (In practice there are connection node types for each paginated type, but all work the same way in our case.) Arguments \u00b6 Each paginated endpoint accepts any of four parameters: first - given an integer N, only return the first N nodes last - given an integer N, only return the last N nodes after - combined with first , will return first N nodes after a given \"cursor\" before - combined with last , will return last N nodes before a given \"cursor\" So typically you'd paginate using first , obtaining a cursor, and then calling the API again with a combination of first and after . The same process could be carried out with last and before to paginate in reverse. Responses \u00b6 Let's take a look at everything that pagination makes available: { bills(first:20) { edges { node { title } cursor } pageInfo { hasNextPage hasPreviousPage endCursor startCursor } totalCount } } You'll see that the connection node has three nodes: edges , pageInfo , and totalCount edges - a list of objects that each have a node and cursor attribute: node - the underlying node type, in our case BillNode cursor - an opaque cursor for this particular item, it can be used with the before and after parameters each paginated node accepts as arguments. pageInfo - a list of helpful pieces of information about this page: hasNextPage - boolean that is true if there is another page after this hasPreviousPage - boolean that is true if there is a page before this endCursor - last cursor in the set of edges, can be used with after to paginate forward startCursor - first cursor in the set of edges, can be used with before to paginate backwards totalCount - total number of objects available from this connection In Practice \u00b6 Let's say you want to get all of the people matching a given criteria: You'd start with a query for all people matching your criteria, ensuring to set the page size to no greater than the maximum: { people(memberOf: \"Some Organization\", first: 100) { edges { node { name } } pageInfo { hasNextPage endCursor } } } Let's say we got back a list of 100 edges and our pageInfo object looked like: { \"hasNextPage\": true, \"endCursor\": \"ZXJyYXlxb20uZWN0aW9uOjA=\" } So you'd make another call: { people(memberOf: \"Some Organization\", first: 100, after:\"ZXJyYXlxb20uZWN0aW9uOjA=\" ) { edges { node { name } } pageInfo { hasNextPage endCursor } } } And let's say in this case you got back only 75 edges, and our pageInfo object looks like: { \"hasNextPage\": false, \"endCursor\": \"AXjYylxX2bu1wxa9uunnb=\" } We'd stop iteration at this point, of course, if hasNextPage had been true, we'd continue on until it wasn't. Renaming fields \u00b6 A really useful trick that is often overlooked is that you can rename fields when retrieving them, for example: { republicans: people(memberOf: \"Republican\", first: 5) { edges { node { full_name: name } } } } Would give back: { \"data\": { \"republicans\": { \"edges\": [ { \"node\": { \"full_name\": \"Michelle Udall\" } }, { \"node\": { \"full_name\": \"Kimberly Yee\" } }, { \"node\": { \"full_name\": \"Regina E. Cobb\" } }, { \"node\": { \"full_name\": \"Michelle B. Ugenti-Rita\" } }, { \"node\": { \"full_name\": \"David Livingston\" } } ] } } } Note that we're both renaming a top-level node here as well as a piece of data within the query. You can also use this to query the same root node twice (doing so without renaming isn't allowed since it results in a name conflict). For example: { republicans: people(memberOf: \"Republican\", first: 5) { edges { node { full_name: name } } } democrats: people(memberOf: \"Democratic\", first: 5) { edges { node { full_name: name } } } } Fuzzy Date Format {#date-format} \u00b6 Unless otherwise noted (most notably createdAt and updatedAt all date objects are \"fuzzy\". Instead of being expressed as an exact date, it is possible a given date takes any of the following formats: YYYY YYYY-MM YYYY-MM-DD YYYY-MM-DD HH:MM:SS (if times are allowed) Action/Vote times are all assumed to be in the state capitol's time zone. Times related to our updates such as updatedAt and createdAt are in UTC. Name Matching \u00b6 In several places such as bill sponsorships and votes you'll notice that we have a raw string representing a person or organization as well as a place for a link to the appropriate OrganizationNode or PersonNode . Because of the way we collect the data from states, we always collect the raw data and later make an attempt to (via a mix of automated matching and manual fixes) connect the reference with data we've already collected. In many cases these linkages will not be provided, but with some upcoming new tools to help us improve this matching we'll be able to dramatically improve the number of matched entities in the near future.","title":"Other Notes"},{"location":"api-v2/other/#other-notes","text":"There are a few other things to be aware of while using the API:","title":"Other Notes"},{"location":"api-v2/other/#explore-the-graph","text":"GraphQL is still quite new, so we figured it might be good to provide some helpful tips on how to think about the data and how you'll use the API. First, it is probably well worth your time to play around in GraphiQL to explore the API and data. It was heavily used when developing the API and writing tests, and is a very powerful tool, particularly when you make use of the self-documenting nature of the graph. When you're thinking about how to query don't necessarily try to replicate your old API calls exactly. For example, perhaps you were grabbing all bills that met a given criteria and then grabbing all sponsors contact details. This can now be done in one call by traversing from the bills-root root node into the BillSponsorshipNode and then up to the PersonNode and finally to the ContactDetailNode This may sound complex at first, but once you get the hang of it, it really does unlock a ton of power and will make your apps more powerful and efficient.","title":"Explore the Graph"},{"location":"api-v2/other/#pagination","text":"In several places (such as the bills-root and BillNode 's votes ) we mention that nodes are paginated. What this means in practice is that instead of getting back the underlying node type, say BillNode , directly, you'll get back BillConnectionNode or similar. (In practice there are connection node types for each paginated type, but all work the same way in our case.)","title":"Pagination"},{"location":"api-v2/other/#arguments","text":"Each paginated endpoint accepts any of four parameters: first - given an integer N, only return the first N nodes last - given an integer N, only return the last N nodes after - combined with first , will return first N nodes after a given \"cursor\" before - combined with last , will return last N nodes before a given \"cursor\" So typically you'd paginate using first , obtaining a cursor, and then calling the API again with a combination of first and after . The same process could be carried out with last and before to paginate in reverse.","title":"Arguments"},{"location":"api-v2/other/#responses","text":"Let's take a look at everything that pagination makes available: { bills(first:20) { edges { node { title } cursor } pageInfo { hasNextPage hasPreviousPage endCursor startCursor } totalCount } } You'll see that the connection node has three nodes: edges , pageInfo , and totalCount edges - a list of objects that each have a node and cursor attribute: node - the underlying node type, in our case BillNode cursor - an opaque cursor for this particular item, it can be used with the before and after parameters each paginated node accepts as arguments. pageInfo - a list of helpful pieces of information about this page: hasNextPage - boolean that is true if there is another page after this hasPreviousPage - boolean that is true if there is a page before this endCursor - last cursor in the set of edges, can be used with after to paginate forward startCursor - first cursor in the set of edges, can be used with before to paginate backwards totalCount - total number of objects available from this connection","title":"Responses"},{"location":"api-v2/other/#in-practice","text":"Let's say you want to get all of the people matching a given criteria: You'd start with a query for all people matching your criteria, ensuring to set the page size to no greater than the maximum: { people(memberOf: \"Some Organization\", first: 100) { edges { node { name } } pageInfo { hasNextPage endCursor } } } Let's say we got back a list of 100 edges and our pageInfo object looked like: { \"hasNextPage\": true, \"endCursor\": \"ZXJyYXlxb20uZWN0aW9uOjA=\" } So you'd make another call: { people(memberOf: \"Some Organization\", first: 100, after:\"ZXJyYXlxb20uZWN0aW9uOjA=\" ) { edges { node { name } } pageInfo { hasNextPage endCursor } } } And let's say in this case you got back only 75 edges, and our pageInfo object looks like: { \"hasNextPage\": false, \"endCursor\": \"AXjYylxX2bu1wxa9uunnb=\" } We'd stop iteration at this point, of course, if hasNextPage had been true, we'd continue on until it wasn't.","title":"In Practice"},{"location":"api-v2/other/#renaming-fields","text":"A really useful trick that is often overlooked is that you can rename fields when retrieving them, for example: { republicans: people(memberOf: \"Republican\", first: 5) { edges { node { full_name: name } } } } Would give back: { \"data\": { \"republicans\": { \"edges\": [ { \"node\": { \"full_name\": \"Michelle Udall\" } }, { \"node\": { \"full_name\": \"Kimberly Yee\" } }, { \"node\": { \"full_name\": \"Regina E. Cobb\" } }, { \"node\": { \"full_name\": \"Michelle B. Ugenti-Rita\" } }, { \"node\": { \"full_name\": \"David Livingston\" } } ] } } } Note that we're both renaming a top-level node here as well as a piece of data within the query. You can also use this to query the same root node twice (doing so without renaming isn't allowed since it results in a name conflict). For example: { republicans: people(memberOf: \"Republican\", first: 5) { edges { node { full_name: name } } } democrats: people(memberOf: \"Democratic\", first: 5) { edges { node { full_name: name } } } }","title":"Renaming fields"},{"location":"api-v2/other/#fuzzy-date-format-date-format","text":"Unless otherwise noted (most notably createdAt and updatedAt all date objects are \"fuzzy\". Instead of being expressed as an exact date, it is possible a given date takes any of the following formats: YYYY YYYY-MM YYYY-MM-DD YYYY-MM-DD HH:MM:SS (if times are allowed) Action/Vote times are all assumed to be in the state capitol's time zone. Times related to our updates such as updatedAt and createdAt are in UTC.","title":"Fuzzy Date Format {#date-format}"},{"location":"api-v2/other/#name-matching","text":"In several places such as bill sponsorships and votes you'll notice that we have a raw string representing a person or organization as well as a place for a link to the appropriate OrganizationNode or PersonNode . Because of the way we collect the data from states, we always collect the raw data and later make an attempt to (via a mix of automated matching and manual fixes) connect the reference with data we've already collected. In many cases these linkages will not be provided, but with some upcoming new tools to help us improve this matching we'll be able to dramatically improve the number of matched entities in the near future.","title":"Name Matching"},{"location":"api-v2/root-nodes/","text":"Root Nodes \u00b6 As seen in the introduction, when constructing a query you will start your query at one (or more) root nodes. The following root nodes are available: jurisdictions \u00b6 Get a list of all jurisdictions. This will return a list of JurisdictionNode objects, one for each state (plus Puerto Rico and DC). Pagination : This endpoint accepts the usual pagination parameters, but pagination is not required. people \u00b6 Get a list of all people matching certain criteria. This will return a list of PersonNode objects, one for each person matching your query. Pagination : This endpoint accepts the usual pagination parameters, and you must limit your results to no more than 100 using either the \"first\" or \"last\" parameter. Parameters \u00b6 name Limit response to people who's name contains the provided string. Includes partial matches & case-insensitive matches. memberOf Limit response to people that have a currently active membership record for an organization. The value passed to memberOf can be an ocd-organization ID or a name (e.g. 'Republican' or 'Nebraska Legislature'). everMemberOf Limit response to people that have any recorded membership record for an organization. Operates as a superset of memberOf. Specifying memberOf and everMemberOf in the same query is invalid. district When specifying either memberOf or everMemberOf, limits to people who's membership represented the district with a given label. (e.g. memberOf: \"Nebraska Legislature\", district: \"7\") Specifying district without memberOf or everMemberOf is invalid. latitude and longitude Limit to people that are currently representing the district(s) containing the point specified by the provided coordinates. Must be specified together. bills \u00b6 Get a list of all bills matching certain criteria. This will return a list of BillNode objects, one for each person matching your query. Pagination : This endpoint accepts the usual pagination parameters, and you must limit your results to no more than 100 using either the \"first\" or \"last\" parameter. Parameters \u00b6 jurisdiction Limit to bills associated with given jurisdiction, parameter can either be a human-readable jurisdiction name or an ocd-jurisdiction ID. chamber Limit to bills originating in a given chamber. (e.g. upper, lower, legislature) session Limit to bills originating in a given legislative session. This parameter should be the desired session's identifier . (See LegislativeSessionNode ). classification Limit to bills with a given classification (e.g. \"bill\" or \"resolution\") subject Limit to bills with a given subject (e.g. \"Agriculture\") searchQuery Limit to bills that contain a given term. (Experimental until 2020!) updatedSince Limit to bills that have had data updated since a given time (UTC). Time should be in the format YYYY-MM-DD[THH:MM:SS]. actionsSince Limit to bills that have had actions since a given time (UTC). Time should be in the format YYYY-MM-DD. jurisdiction \u00b6 Look up a single jurisdiction by name or ID. This will return a single JurisdictionNode object with the provided name or ID parameter. Parameters \u00b6 name The human-readable name of the jurisdiction, such as 'New Hampshire'. id The ocd-jurisdiction ID of the desired jurisdiction, such as 'ocd-jurisdiction/country:us/state:nh'. You are required to provide one of the two available parameters. person \u00b6 Look up a single person by ocd-person ID. This will return a single PersonNode by ID. Parameters \u00b6 id ocd-person ID for the desired individual. organization \u00b6 Look up a single organization by ocd-organization ID. This will return a single OrganizationNode by ID. Parameters \u00b6 id ocd-organization ID for the desired individual. bill \u00b6 Look up a single bill by ID, URL, or (jurisdiction, session, identifier) combo. This will return a single BillNode object with the specified bill. Parameters \u00b6 id The ocd-bill ID of the desired bill, such as 'ocd-jurisdiction/country:us/state:nh'. openstatesUrl The URL of the desired bill, such as ' https://openstates.org/nc/bills/2019/HB760/ '. jurisdiction , session , identifier Must be specified together to fully identify a bill. As is true elsewhere, jurisdiction may be specified by name (New Hampshire) or ocd-jurisdiction ID (ocd-jurisdiction/country:us/state:nh). Session is specified by legislative session identifier (e.g. 2018 or 49). Identifier is the exact identifier of the desired bill, such as \"HB 327\". You are required to provide one either id or the other parameters to fully specify a bill. Use bills if you are looking for something more broad.","title":"Root Nodes"},{"location":"api-v2/root-nodes/#root-nodes","text":"As seen in the introduction, when constructing a query you will start your query at one (or more) root nodes. The following root nodes are available:","title":"Root Nodes"},{"location":"api-v2/root-nodes/#jurisdictions","text":"Get a list of all jurisdictions. This will return a list of JurisdictionNode objects, one for each state (plus Puerto Rico and DC). Pagination : This endpoint accepts the usual pagination parameters, but pagination is not required.","title":"jurisdictions"},{"location":"api-v2/root-nodes/#people","text":"Get a list of all people matching certain criteria. This will return a list of PersonNode objects, one for each person matching your query. Pagination : This endpoint accepts the usual pagination parameters, and you must limit your results to no more than 100 using either the \"first\" or \"last\" parameter.","title":"people"},{"location":"api-v2/root-nodes/#parameters","text":"name Limit response to people who's name contains the provided string. Includes partial matches & case-insensitive matches. memberOf Limit response to people that have a currently active membership record for an organization. The value passed to memberOf can be an ocd-organization ID or a name (e.g. 'Republican' or 'Nebraska Legislature'). everMemberOf Limit response to people that have any recorded membership record for an organization. Operates as a superset of memberOf. Specifying memberOf and everMemberOf in the same query is invalid. district When specifying either memberOf or everMemberOf, limits to people who's membership represented the district with a given label. (e.g. memberOf: \"Nebraska Legislature\", district: \"7\") Specifying district without memberOf or everMemberOf is invalid. latitude and longitude Limit to people that are currently representing the district(s) containing the point specified by the provided coordinates. Must be specified together.","title":"Parameters"},{"location":"api-v2/root-nodes/#bills","text":"Get a list of all bills matching certain criteria. This will return a list of BillNode objects, one for each person matching your query. Pagination : This endpoint accepts the usual pagination parameters, and you must limit your results to no more than 100 using either the \"first\" or \"last\" parameter.","title":"bills"},{"location":"api-v2/root-nodes/#parameters_1","text":"jurisdiction Limit to bills associated with given jurisdiction, parameter can either be a human-readable jurisdiction name or an ocd-jurisdiction ID. chamber Limit to bills originating in a given chamber. (e.g. upper, lower, legislature) session Limit to bills originating in a given legislative session. This parameter should be the desired session's identifier . (See LegislativeSessionNode ). classification Limit to bills with a given classification (e.g. \"bill\" or \"resolution\") subject Limit to bills with a given subject (e.g. \"Agriculture\") searchQuery Limit to bills that contain a given term. (Experimental until 2020!) updatedSince Limit to bills that have had data updated since a given time (UTC). Time should be in the format YYYY-MM-DD[THH:MM:SS]. actionsSince Limit to bills that have had actions since a given time (UTC). Time should be in the format YYYY-MM-DD.","title":"Parameters"},{"location":"api-v2/root-nodes/#jurisdiction","text":"Look up a single jurisdiction by name or ID. This will return a single JurisdictionNode object with the provided name or ID parameter.","title":"jurisdiction"},{"location":"api-v2/root-nodes/#parameters_2","text":"name The human-readable name of the jurisdiction, such as 'New Hampshire'. id The ocd-jurisdiction ID of the desired jurisdiction, such as 'ocd-jurisdiction/country:us/state:nh'. You are required to provide one of the two available parameters.","title":"Parameters"},{"location":"api-v2/root-nodes/#person","text":"Look up a single person by ocd-person ID. This will return a single PersonNode by ID.","title":"person"},{"location":"api-v2/root-nodes/#parameters_3","text":"id ocd-person ID for the desired individual.","title":"Parameters"},{"location":"api-v2/root-nodes/#organization","text":"Look up a single organization by ocd-organization ID. This will return a single OrganizationNode by ID.","title":"organization"},{"location":"api-v2/root-nodes/#parameters_4","text":"id ocd-organization ID for the desired individual.","title":"Parameters"},{"location":"api-v2/root-nodes/#bill","text":"Look up a single bill by ID, URL, or (jurisdiction, session, identifier) combo. This will return a single BillNode object with the specified bill.","title":"bill"},{"location":"api-v2/root-nodes/#parameters_5","text":"id The ocd-bill ID of the desired bill, such as 'ocd-jurisdiction/country:us/state:nh'. openstatesUrl The URL of the desired bill, such as ' https://openstates.org/nc/bills/2019/HB760/ '. jurisdiction , session , identifier Must be specified together to fully identify a bill. As is true elsewhere, jurisdiction may be specified by name (New Hampshire) or ocd-jurisdiction ID (ocd-jurisdiction/country:us/state:nh). Session is specified by legislative session identifier (e.g. 2018 or 49). Identifier is the exact identifier of the desired bill, such as \"HB 327\". You are required to provide one either id or the other parameters to fully specify a bill. Use bills if you are looking for something more broad.","title":"Parameters"},{"location":"api-v2/types/","text":"Data Types \u00b6 Starting at the base nodes, data in the API is represented as interconnected nodes of various types. This page provides an overview of the nodes. Another good way to get acquainted with the layout is to use the GraphiQL browser (click Docs in the upper right corner). Jurisdictions & Sessions \u00b6 JurisdictionNode \u00b6 A Jurisdiction is the Open Civic Data term for the top level divisions of the US. Open States is comprised of 52 jurisdictions, one for each state, and two more for D.C. and Puerto Rico. Each JurisdictionNode has the following attributes & nodes available: id - ocd-jurisdiction identifier, these are permanent identifiers assigned to each Jurisdiction name - human-readable name for the jurisdiction (e.g. Kansas) url - URL of official website for jurisdiction featureFlags - reserved for future use legislativeSessions - Paginated list (see pagination ) of LegislativeSessionNode belonging to this jurisdiction's legislature. organizations - Paginated list of OrganizationNode belonging to this jurisdiction. it is also possible to filter the list of children using the classification parameter lastScrapedAt - Time when last scrape finished. See also: Open Civic Data Jurisdiction reference LegislativeSessionNode \u00b6 A legislative session is a convening of the legislature, either a primary or special session. Each LegislativeSessionNode has the following attributes and nodes available: jurisdiction - JurisdictionNode which this session belongs to. identifier - short identifier by which this session is referred to (e.g. 2017s1 or 121) name - formal name of session (e.g. \"2017 Special Session #1\" or \"121st Session\" classification - \"primary\" or \"special\" startDate - start date of session if known endDate - end date of session if known DivisionNode \u00b6 Divisions represent particular geopolitical boundaries. Divisions exist for states as well as their component districts and are tied closely to political geographies. id - Open Civic Data Division ID name - human-readable name for the division redirect - link to another DivisionNode, only present if division has been replaced country - country code (will be \"us\") for all Open States divisions createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object People & Organizations \u00b6 PersonNode \u00b6 People, typically legislators and their associated metadata. Note that most fields are optional beyond name as often we don't have a reliable given/family name or birthDate for instance. id - Open Civic Data Person ID name - primary name for the person sortName - alternate name to sort by (if known) familyName - hereditary name, essentially a \"last name\" (if known) givenName - essentially a \"first name\" (if known) image - full URL to official image of legislator birthDate - see date-format deathDate - see date-format identifiers - list of other known identifiers, IdentifierNode otherNames - list of other known names, NameNode links - official URLs relating to this person, LinkNode contactDetails - ways to contact this person (via email, phone, etc.), contactdetailnode currentMemberships - currently active memberships MembershipNode can be filtered with the classification parameter to only get memberships to certain types of OrganizationNode oldMemberships - inactive memberships MembershipNode can be filtered with the classification parameter to only get memberships to certain types of OrganizationNode sources - URLs which were used in compiling Open States' information on this subject, [LinkNode] createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Popolo's person Open Civic Data OCDEP 5 OrganizationNode \u00b6 Organizations that comprise the state legislatures and their associated metdata. A typical bicameral legislature is comprised of a top-level organization (classification=legislature), two chambers (classification=upper & lower), and any number of committees (classification=committee). Each Organization is comprised of the following attributes and nodes: id - Open Civic Data Organization ID name - primary name for the person image - full URL to official image for organization classification - the type of organization as described above foundingDate - see date-format dissolutionDate - see date-format parent - parent OrganizationNode if one exists children - paginated list of child OrganizationNode objects it is also possible to filter the list of children using the classification parameter currentMemberships - list of all current members of this Organization identifiers - list of other known identifiers for this organization, IdentifierNode otherNames - list of other known names for this organization, NameNode links - official URLs relating to this person, LinkNode sources - URLs which were used in compiling Open States' information on this subject, [LinkNode] createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Popolo's organization Open Civic Data OCDEP 5 MembershipNode \u00b6 A MembershipNode represents a connection between a personnode and a organizationnode . A membership may optionally also reference a particular postnode , such as a particular seat within a given chamber. Each membership has the following attributes and nodes: id - Open Civic Data Membership ID personName the raw name of the person that the membership describes (see name-matching person - personnode organization - organizationnode post - postnode label - label assigned to this membership role - role fulfilled by this membership startDate - start date of membership if known endDate - end date of membership if known createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Popolo's membership Open Civic Data OCDEP 5 PostNode \u00b6 A PostNode represents a given position within an organization. The most common example would be a seat such as Maryland's 4th House Seat. It is worth noting that some seats can have multiple active memberships at once, as noted in maximumMemberships . Each post has the following attributes and nodes: id - Open Civic Data Post ID label - label assigned to this post (e.g. 3) role - role fulfilled by this membership (e.g. 'member') division - related divisionnode if this role has a relevant division startDate - start date of membership if known endDate - end date of membership if known maximumMemberships - typically 1, but set higher in the case of multi-member districts createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Popolo's post Open Civic Data OCDEP 5 Bills & Votes \u00b6 BillNode \u00b6 A BillNode represents any legislative instrument such as a bill or resolution. Each node has the following attributes and nodes available: id - Internal ocd-bill identifier for this bill. legislativeSession - link to LegislativeSessionNode this bill is from identifier - primary identifier for this bill (e.g. HB 264) title - primary title for this bill fromOrganization - organization (typically upper or lower chamber) primarily associated with this bill classification - list of one or more bill types such as \"bill\" or \"resolution\" subject - list of zero or more subjects assigned by the state abstracts - list of abstracts provided by the state, BillAbstractNode otherTitles - list of other titles provided by the state, BillTitleNode otherIdentifiers - list of other identifiers provided by the state, BillIdentifierNode actions - list of actions (such as introduction, amendment, passage, etc.) that have been taken on the bill, BillActionNode sponsorships - list of bill sponsors, BillSponsorshipNode relatedBills - list of related bills as provided by the state, RelatedBillNode versions - list of bill versions as provided by the state, BillDocumentNode documents - list of related documents (e.g. legal analysis, fiscal notes, etc.) as provided by the state, BillDocumentNode votes - paginated list of VoteEventNode related to the bill sources - URLs which were used in compiling Open States' information on this subject, linknode openstatesUrl - URL to bill page on OpenStates.org createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object BillAbstractNode \u00b6 Represents an official abstract for a bill, each BillAbstractNode has the following attributes: abstract - the abstract itself note - optional note about origin/purpose of abstract date - optional date associated with abstract BillTitleNode \u00b6 Represents an alternate title for a bill, each BillTitleNode has the following attributes: title - the alternate title note - optional note about origin/purpose of this title BillIdentifierNode \u00b6 Represents an alternate identifier for a bill, each BillIdentifierNode has the following attributes: identifier - the alternate identifier scheme - a name for the identifier scheme note - optional note about origin/purpose of this identifier BillActionNode \u00b6 Represents an action taken on a bill, each BillActionNode has the following attributes and nodes: organization - OrganizationNode where this action originated, will typically be either upper or lower chamber, or perhaps legislature as a whole. description - text describing the action as provided by the jurisdiction. date - date action took place (see date-format ) classification - list of zero or more normalized action types (see action-categorization ) order - integer by which actions can be sorted, not intended for display purposes extras - JSON string providing extra information about this action vote - if there is a known associated vote, pointer to the relevant VoteEventNode relatedEntities - a list of RelatedEntityNode with known entities referenced in this action RelatedEntityNode \u00b6 Represents an entity that is related to a BillActionNode . name - raw (source-provided) name of entity entityType - either organization or person organization - if entityType is 'organization', the resolved OrganizationNode person - if entityType is 'person', the resolved PersonNode See name-matching for details on how name relates to organiation and person . BillSponsorshipNode \u00b6 Represents a sponsor of a bill. name - raw (source-provided) name of sponsoring person or organization entityType - either organization or person organization - if entityType is 'organization', the resolved OrganizationNode person - if entityType is 'person', the resolved PersonNode primary - boolean, true if sponsorship is considered by the jurisdiction to be \"primary\" (note: in many states multiple primary sponsors may exist) classification - jurisdiction-provided type of sponsorship, such as \"author\" or \"cosponsor\". These meanings typically vary across states, which is why we provide primary as a sort of indicator of the degree of sponsorship indicated. See name-matching for details on how name relates to organiation and person . RelatedBillNode \u00b6 Represents relationships between bills. identifier - identifier of related bill (e.g. SB 401) legislativeSession - identifier of related session (in same jurisdiction) relationType - type of relationship such as \"companion\", \"prior-session\", \"replaced-by\", or \"replaces\" relatedBill - if the related bill is found to exist in our data, link to the BillNode BillDocumentNode \u00b6 Representation of documents and versions on bills. A given document can have multiple links representing different manifestations (e.g. HTML, PDF, DOC) of the same content. note - note describing the purpose of the document or version (e.g. Final Printing) date - optional date associated with the document links - list of one or more MimetypeLinkNode with actual URLs to bills. MimetypeLinkNode \u00b6 Represents a single manifestation of a particular document. mediaType - media type (aka MIME type) such as application/pdf or text/html url - URL to official copy of the bill text - text describing this particular manifestation (e.g. PDF) VoteEventNode \u00b6 Represents a vote taken on a bill. id - Internal ocd-vote identifier for this bill. identifier - Identifier used by jurisdiction to uniquely identify the vote. motionText - Text of the motion being voted upon, such as \"motion to pass the bill as amended.\" motionClassification - List with zero or more classifications for this motion, such as \"passage\" or \"veto-override\" startDate - Date on which the vote took place. (see date-format result - Outcome of the vote, 'pass' or 'fail'. organization - Related OrganizationNode where vote took place. billAction - Optional linked BillActionNode . votes - List of PersonVoteNode for each individual's recorded vote. (May not be present depending on jurisdiction.) counts - List of VoteCountNode with sums of each outcome (e.g. yea/nay/abstain). sources - URLs which were used in compiling Open States' information on this subject, [LinkNode] createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Open Civic Data vote format . PersonVoteNode \u00b6 Represents an individual person's vote (e.g. yea or nay) on a given bill. option - Option chosen by this individual. (yea, nay, abstain, other, etc.) voterName - Raw name of voter as provided by jurisdiction. voter - Resolved PersonNode representing voter. (See name-matching note - Note attached to this vote, sometimes used for explaining an \"other\" vote. VoteCountNode \u00b6 Represents the sum of votes for a given option . option - Option in question. (yea, nay, abstain, other, etc.) value - Number of individuals voting this way. Other Nodes \u00b6 IdentifierNode \u00b6 Represents an alternate identifier, each with the following attributes: identifier - the alternate identifier scheme - a name for the identifier scheme NameNode \u00b6 Represents an alterante name, each with the following attributes: name - the alternate name note - note about usage/origin of this alternate name startDate - date at which this name began being valid (blank if unknown) endDate - date at which this name stopped being valid (blank if unknown or still active) LinkNode \u00b6 Represents a single link associated with a person or used as a source. url - URL text - text describing the use of this particular URL ContactDetailNode \u00b6 Used to represent a contact method for a given person. type - type of contact detail (e.g. voice, email, address, etc.) value - actual phone number, email address, etc. note - used to group contact data by location (e.g. Home Address, Office Address) label - human-readable label for this contact detail","title":"Data Types"},{"location":"api-v2/types/#data-types","text":"Starting at the base nodes, data in the API is represented as interconnected nodes of various types. This page provides an overview of the nodes. Another good way to get acquainted with the layout is to use the GraphiQL browser (click Docs in the upper right corner).","title":"Data Types"},{"location":"api-v2/types/#jurisdictions-sessions","text":"","title":"Jurisdictions & Sessions"},{"location":"api-v2/types/#jurisdictionnode","text":"A Jurisdiction is the Open Civic Data term for the top level divisions of the US. Open States is comprised of 52 jurisdictions, one for each state, and two more for D.C. and Puerto Rico. Each JurisdictionNode has the following attributes & nodes available: id - ocd-jurisdiction identifier, these are permanent identifiers assigned to each Jurisdiction name - human-readable name for the jurisdiction (e.g. Kansas) url - URL of official website for jurisdiction featureFlags - reserved for future use legislativeSessions - Paginated list (see pagination ) of LegislativeSessionNode belonging to this jurisdiction's legislature. organizations - Paginated list of OrganizationNode belonging to this jurisdiction. it is also possible to filter the list of children using the classification parameter lastScrapedAt - Time when last scrape finished. See also: Open Civic Data Jurisdiction reference","title":"JurisdictionNode"},{"location":"api-v2/types/#legislativesessionnode","text":"A legislative session is a convening of the legislature, either a primary or special session. Each LegislativeSessionNode has the following attributes and nodes available: jurisdiction - JurisdictionNode which this session belongs to. identifier - short identifier by which this session is referred to (e.g. 2017s1 or 121) name - formal name of session (e.g. \"2017 Special Session #1\" or \"121st Session\" classification - \"primary\" or \"special\" startDate - start date of session if known endDate - end date of session if known","title":"LegislativeSessionNode"},{"location":"api-v2/types/#divisionnode","text":"Divisions represent particular geopolitical boundaries. Divisions exist for states as well as their component districts and are tied closely to political geographies. id - Open Civic Data Division ID name - human-readable name for the division redirect - link to another DivisionNode, only present if division has been replaced country - country code (will be \"us\") for all Open States divisions createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object","title":"DivisionNode"},{"location":"api-v2/types/#people-organizations","text":"","title":"People & Organizations"},{"location":"api-v2/types/#personnode","text":"People, typically legislators and their associated metadata. Note that most fields are optional beyond name as often we don't have a reliable given/family name or birthDate for instance. id - Open Civic Data Person ID name - primary name for the person sortName - alternate name to sort by (if known) familyName - hereditary name, essentially a \"last name\" (if known) givenName - essentially a \"first name\" (if known) image - full URL to official image of legislator birthDate - see date-format deathDate - see date-format identifiers - list of other known identifiers, IdentifierNode otherNames - list of other known names, NameNode links - official URLs relating to this person, LinkNode contactDetails - ways to contact this person (via email, phone, etc.), contactdetailnode currentMemberships - currently active memberships MembershipNode can be filtered with the classification parameter to only get memberships to certain types of OrganizationNode oldMemberships - inactive memberships MembershipNode can be filtered with the classification parameter to only get memberships to certain types of OrganizationNode sources - URLs which were used in compiling Open States' information on this subject, [LinkNode] createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Popolo's person Open Civic Data OCDEP 5","title":"PersonNode"},{"location":"api-v2/types/#organizationnode","text":"Organizations that comprise the state legislatures and their associated metdata. A typical bicameral legislature is comprised of a top-level organization (classification=legislature), two chambers (classification=upper & lower), and any number of committees (classification=committee). Each Organization is comprised of the following attributes and nodes: id - Open Civic Data Organization ID name - primary name for the person image - full URL to official image for organization classification - the type of organization as described above foundingDate - see date-format dissolutionDate - see date-format parent - parent OrganizationNode if one exists children - paginated list of child OrganizationNode objects it is also possible to filter the list of children using the classification parameter currentMemberships - list of all current members of this Organization identifiers - list of other known identifiers for this organization, IdentifierNode otherNames - list of other known names for this organization, NameNode links - official URLs relating to this person, LinkNode sources - URLs which were used in compiling Open States' information on this subject, [LinkNode] createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Popolo's organization Open Civic Data OCDEP 5","title":"OrganizationNode"},{"location":"api-v2/types/#membershipnode","text":"A MembershipNode represents a connection between a personnode and a organizationnode . A membership may optionally also reference a particular postnode , such as a particular seat within a given chamber. Each membership has the following attributes and nodes: id - Open Civic Data Membership ID personName the raw name of the person that the membership describes (see name-matching person - personnode organization - organizationnode post - postnode label - label assigned to this membership role - role fulfilled by this membership startDate - start date of membership if known endDate - end date of membership if known createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Popolo's membership Open Civic Data OCDEP 5","title":"MembershipNode"},{"location":"api-v2/types/#postnode","text":"A PostNode represents a given position within an organization. The most common example would be a seat such as Maryland's 4th House Seat. It is worth noting that some seats can have multiple active memberships at once, as noted in maximumMemberships . Each post has the following attributes and nodes: id - Open Civic Data Post ID label - label assigned to this post (e.g. 3) role - role fulfilled by this membership (e.g. 'member') division - related divisionnode if this role has a relevant division startDate - start date of membership if known endDate - end date of membership if known maximumMemberships - typically 1, but set higher in the case of multi-member districts createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Popolo's post Open Civic Data OCDEP 5","title":"PostNode"},{"location":"api-v2/types/#bills-votes","text":"","title":"Bills & Votes"},{"location":"api-v2/types/#billnode","text":"A BillNode represents any legislative instrument such as a bill or resolution. Each node has the following attributes and nodes available: id - Internal ocd-bill identifier for this bill. legislativeSession - link to LegislativeSessionNode this bill is from identifier - primary identifier for this bill (e.g. HB 264) title - primary title for this bill fromOrganization - organization (typically upper or lower chamber) primarily associated with this bill classification - list of one or more bill types such as \"bill\" or \"resolution\" subject - list of zero or more subjects assigned by the state abstracts - list of abstracts provided by the state, BillAbstractNode otherTitles - list of other titles provided by the state, BillTitleNode otherIdentifiers - list of other identifiers provided by the state, BillIdentifierNode actions - list of actions (such as introduction, amendment, passage, etc.) that have been taken on the bill, BillActionNode sponsorships - list of bill sponsors, BillSponsorshipNode relatedBills - list of related bills as provided by the state, RelatedBillNode versions - list of bill versions as provided by the state, BillDocumentNode documents - list of related documents (e.g. legal analysis, fiscal notes, etc.) as provided by the state, BillDocumentNode votes - paginated list of VoteEventNode related to the bill sources - URLs which were used in compiling Open States' information on this subject, linknode openstatesUrl - URL to bill page on OpenStates.org createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object","title":"BillNode"},{"location":"api-v2/types/#billabstractnode","text":"Represents an official abstract for a bill, each BillAbstractNode has the following attributes: abstract - the abstract itself note - optional note about origin/purpose of abstract date - optional date associated with abstract","title":"BillAbstractNode"},{"location":"api-v2/types/#billtitlenode","text":"Represents an alternate title for a bill, each BillTitleNode has the following attributes: title - the alternate title note - optional note about origin/purpose of this title","title":"BillTitleNode"},{"location":"api-v2/types/#billidentifiernode","text":"Represents an alternate identifier for a bill, each BillIdentifierNode has the following attributes: identifier - the alternate identifier scheme - a name for the identifier scheme note - optional note about origin/purpose of this identifier","title":"BillIdentifierNode"},{"location":"api-v2/types/#billactionnode","text":"Represents an action taken on a bill, each BillActionNode has the following attributes and nodes: organization - OrganizationNode where this action originated, will typically be either upper or lower chamber, or perhaps legislature as a whole. description - text describing the action as provided by the jurisdiction. date - date action took place (see date-format ) classification - list of zero or more normalized action types (see action-categorization ) order - integer by which actions can be sorted, not intended for display purposes extras - JSON string providing extra information about this action vote - if there is a known associated vote, pointer to the relevant VoteEventNode relatedEntities - a list of RelatedEntityNode with known entities referenced in this action","title":"BillActionNode"},{"location":"api-v2/types/#relatedentitynode","text":"Represents an entity that is related to a BillActionNode . name - raw (source-provided) name of entity entityType - either organization or person organization - if entityType is 'organization', the resolved OrganizationNode person - if entityType is 'person', the resolved PersonNode See name-matching for details on how name relates to organiation and person .","title":"RelatedEntityNode"},{"location":"api-v2/types/#billsponsorshipnode","text":"Represents a sponsor of a bill. name - raw (source-provided) name of sponsoring person or organization entityType - either organization or person organization - if entityType is 'organization', the resolved OrganizationNode person - if entityType is 'person', the resolved PersonNode primary - boolean, true if sponsorship is considered by the jurisdiction to be \"primary\" (note: in many states multiple primary sponsors may exist) classification - jurisdiction-provided type of sponsorship, such as \"author\" or \"cosponsor\". These meanings typically vary across states, which is why we provide primary as a sort of indicator of the degree of sponsorship indicated. See name-matching for details on how name relates to organiation and person .","title":"BillSponsorshipNode"},{"location":"api-v2/types/#relatedbillnode","text":"Represents relationships between bills. identifier - identifier of related bill (e.g. SB 401) legislativeSession - identifier of related session (in same jurisdiction) relationType - type of relationship such as \"companion\", \"prior-session\", \"replaced-by\", or \"replaces\" relatedBill - if the related bill is found to exist in our data, link to the BillNode","title":"RelatedBillNode"},{"location":"api-v2/types/#billdocumentnode","text":"Representation of documents and versions on bills. A given document can have multiple links representing different manifestations (e.g. HTML, PDF, DOC) of the same content. note - note describing the purpose of the document or version (e.g. Final Printing) date - optional date associated with the document links - list of one or more MimetypeLinkNode with actual URLs to bills.","title":"BillDocumentNode"},{"location":"api-v2/types/#mimetypelinknode","text":"Represents a single manifestation of a particular document. mediaType - media type (aka MIME type) such as application/pdf or text/html url - URL to official copy of the bill text - text describing this particular manifestation (e.g. PDF)","title":"MimetypeLinkNode"},{"location":"api-v2/types/#voteeventnode","text":"Represents a vote taken on a bill. id - Internal ocd-vote identifier for this bill. identifier - Identifier used by jurisdiction to uniquely identify the vote. motionText - Text of the motion being voted upon, such as \"motion to pass the bill as amended.\" motionClassification - List with zero or more classifications for this motion, such as \"passage\" or \"veto-override\" startDate - Date on which the vote took place. (see date-format result - Outcome of the vote, 'pass' or 'fail'. organization - Related OrganizationNode where vote took place. billAction - Optional linked BillActionNode . votes - List of PersonVoteNode for each individual's recorded vote. (May not be present depending on jurisdiction.) counts - List of VoteCountNode with sums of each outcome (e.g. yea/nay/abstain). sources - URLs which were used in compiling Open States' information on this subject, [LinkNode] createdAt - date at which this object was created in our system updatedAt - date at which this object was last updated in our system extras - JSON string with optional additional information about the object See also: Open Civic Data vote format .","title":"VoteEventNode"},{"location":"api-v2/types/#personvotenode","text":"Represents an individual person's vote (e.g. yea or nay) on a given bill. option - Option chosen by this individual. (yea, nay, abstain, other, etc.) voterName - Raw name of voter as provided by jurisdiction. voter - Resolved PersonNode representing voter. (See name-matching note - Note attached to this vote, sometimes used for explaining an \"other\" vote.","title":"PersonVoteNode"},{"location":"api-v2/types/#votecountnode","text":"Represents the sum of votes for a given option . option - Option in question. (yea, nay, abstain, other, etc.) value - Number of individuals voting this way.","title":"VoteCountNode"},{"location":"api-v2/types/#other-nodes","text":"","title":"Other Nodes"},{"location":"api-v2/types/#identifiernode","text":"Represents an alternate identifier, each with the following attributes: identifier - the alternate identifier scheme - a name for the identifier scheme","title":"IdentifierNode"},{"location":"api-v2/types/#namenode","text":"Represents an alterante name, each with the following attributes: name - the alternate name note - note about usage/origin of this alternate name startDate - date at which this name began being valid (blank if unknown) endDate - date at which this name stopped being valid (blank if unknown or still active)","title":"NameNode"},{"location":"api-v2/types/#linknode","text":"Represents a single link associated with a person or used as a source. url - URL text - text describing the use of this particular URL","title":"LinkNode"},{"location":"api-v2/types/#contactdetailnode","text":"Used to represent a contact method for a given person. type - type of contact detail (e.g. voice, email, address, etc.) value - actual phone number, email address, etc. note - used to group contact data by location (e.g. Home Address, Office Address) label - human-readable label for this contact detail","title":"ContactDetailNode"},{"location":"api-v3/","text":"API v3 Overview \u00b6 Open States provides a JSON API that can be used to programatically access state legislative information. API Basics \u00b6 The root URL for the API is https://v3.openstates.org/ . API keys are required. You can register for an API key and once activated, you\\'ll pass your API key via the X-API-KEY header or ?apikey query parameter. Auto-generated interactive documentation is available at either: https://v3.openstates.org/docs/ https://v3.openstates.org/redoc/ (whichever you prefer) Issues should be filed at our issue tracker . You can also check out our introductory blog post for more details. Methods \u00b6 Method Description Interactive Docs /jurisdictions Get list of available jurisdictions. try it! /jurisdictions/{jurisdiction_id} Get detailed metadata for a particular jurisdiction. try it! /people List or search people (legislators, governors, etc.) try it! /people.geo Get legislators for a given location. try it! /bills Search bills by various criteria. try it! /bills/ocd-bill/{uuid} Get bill by internal ID. try it! /bills/{jurisdiction}/{session}/{id} Get bill by jurisdiction, session, and ID. try it! /committees Get list of committees by jurisdiction. try it! /committees/{committee_id} Get details on committee by internal ID. try it! /events Get list of events by jurisdiction. try it! /events/{event_id} Get details on event by internal ID. try it! Concepts \u00b6 Jurisdiction The fundamental unit by which data is partitioned is the \\'jurisdiction.\\' If you are just interested in states you can consider the words synonymous for the most part. Jurisdictions include states, DC & Puerto Rico, and municipal governments for which we have limited support. Person A legislator, governor, mayor, etc. Each person possibly has a number of roles, at most one of which is considered \\'current.\\' Bill A proposed piece of legislation, encompasses bills, resolutions, constitutional amendments, etc. A given bill may have any number of votes, sponsorships, actions, etc.","title":"API v3 Overview"},{"location":"api-v3/#api-v3-overview","text":"Open States provides a JSON API that can be used to programatically access state legislative information.","title":"API v3 Overview"},{"location":"api-v3/#api-basics","text":"The root URL for the API is https://v3.openstates.org/ . API keys are required. You can register for an API key and once activated, you\\'ll pass your API key via the X-API-KEY header or ?apikey query parameter. Auto-generated interactive documentation is available at either: https://v3.openstates.org/docs/ https://v3.openstates.org/redoc/ (whichever you prefer) Issues should be filed at our issue tracker . You can also check out our introductory blog post for more details.","title":"API Basics"},{"location":"api-v3/#methods","text":"Method Description Interactive Docs /jurisdictions Get list of available jurisdictions. try it! /jurisdictions/{jurisdiction_id} Get detailed metadata for a particular jurisdiction. try it! /people List or search people (legislators, governors, etc.) try it! /people.geo Get legislators for a given location. try it! /bills Search bills by various criteria. try it! /bills/ocd-bill/{uuid} Get bill by internal ID. try it! /bills/{jurisdiction}/{session}/{id} Get bill by jurisdiction, session, and ID. try it! /committees Get list of committees by jurisdiction. try it! /committees/{committee_id} Get details on committee by internal ID. try it! /events Get list of events by jurisdiction. try it! /events/{event_id} Get details on event by internal ID. try it!","title":"Methods"},{"location":"api-v3/#concepts","text":"Jurisdiction The fundamental unit by which data is partitioned is the \\'jurisdiction.\\' If you are just interested in states you can consider the words synonymous for the most part. Jurisdictions include states, DC & Puerto Rico, and municipal governments for which we have limited support. Person A legislator, governor, mayor, etc. Each person possibly has a number of roles, at most one of which is considered \\'current.\\' Bill A proposed piece of legislation, encompasses bills, resolutions, constitutional amendments, etc. A given bill may have any number of votes, sponsorships, actions, etc.","title":"Concepts"},{"location":"api-v3/changelog/","text":"Changelog \u00b6 2021.11.12 \u00b6 add LegislativeSession.downloads 2021.10.27 \u00b6 add Bill.related_bills include 2021.10.12 \u00b6 added classification field on Person.offices Person.offices now always have all fields present, even if empty 2021.09.24 \u00b6 added experimental event endpoints 2021.08.30 \u00b6 added identifiers parameter to bill search 2021.08.03 \u00b6 added latest_bill_update & latest_people_update to API v3 2021.08.02 \u00b6 addded experimental committee endpoints 2021.04.19 \u00b6 experimental support for Jurisdiction.latest_runs include parameter 2021.03.24 \u00b6 Jurisdiction classification can now be \\'country\\' experimental US federal support 2021.02.23 \u00b6 people district= parameter added 2020.12.21 \u00b6 people name= parameter is now fuzzy-searched, matching API v1 and v2 2020.10.30 \u00b6 move email to a top-level field on Person responses add consistent post ordering 2020.10.21 \u00b6 fix datetime handling for updated_since and created_since filters add list of divisions when including Jurisdiction.organizations 2020.10.13 \u00b6 add updated_asc sort option add rate limiting bugfix for New York jurisdiction lookup (openstates/issues#136) 2020.09.28 \u00b6 set permissive CORS settings bills endpoint updates: added created_since filter, thanks to Donald Wasserman! added sponsor and sponsor_classification filters added sort parameter added useful error message when searching /bills by session without jurisdiction restored missing Bill.from_organization field introduced new fields: Person.openstates_url, Bill.openstates_url 2020.09.14 \u00b6 removed some unused fields from responses removed deprecated government classification from Jurisdiction 2020.09.10 \u00b6 added Jurisdiction.legislative_sessions corrected initial pagination limits for release 2020.09.09 \u00b6 Initial beta release.","title":"Changelog"},{"location":"api-v3/changelog/#changelog","text":"","title":"Changelog"},{"location":"api-v3/changelog/#20211112","text":"add LegislativeSession.downloads","title":"2021.11.12"},{"location":"api-v3/changelog/#20211027","text":"add Bill.related_bills include","title":"2021.10.27"},{"location":"api-v3/changelog/#20211012","text":"added classification field on Person.offices Person.offices now always have all fields present, even if empty","title":"2021.10.12"},{"location":"api-v3/changelog/#20210924","text":"added experimental event endpoints","title":"2021.09.24"},{"location":"api-v3/changelog/#20210830","text":"added identifiers parameter to bill search","title":"2021.08.30"},{"location":"api-v3/changelog/#20210803","text":"added latest_bill_update & latest_people_update to API v3","title":"2021.08.03"},{"location":"api-v3/changelog/#20210802","text":"addded experimental committee endpoints","title":"2021.08.02"},{"location":"api-v3/changelog/#20210419","text":"experimental support for Jurisdiction.latest_runs include parameter","title":"2021.04.19"},{"location":"api-v3/changelog/#20210324","text":"Jurisdiction classification can now be \\'country\\' experimental US federal support","title":"2021.03.24"},{"location":"api-v3/changelog/#20210223","text":"people district= parameter added","title":"2021.02.23"},{"location":"api-v3/changelog/#20201221","text":"people name= parameter is now fuzzy-searched, matching API v1 and v2","title":"2020.12.21"},{"location":"api-v3/changelog/#20201030","text":"move email to a top-level field on Person responses add consistent post ordering","title":"2020.10.30"},{"location":"api-v3/changelog/#20201021","text":"fix datetime handling for updated_since and created_since filters add list of divisions when including Jurisdiction.organizations","title":"2020.10.21"},{"location":"api-v3/changelog/#20201013","text":"add updated_asc sort option add rate limiting bugfix for New York jurisdiction lookup (openstates/issues#136)","title":"2020.10.13"},{"location":"api-v3/changelog/#20200928","text":"set permissive CORS settings bills endpoint updates: added created_since filter, thanks to Donald Wasserman! added sponsor and sponsor_classification filters added sort parameter added useful error message when searching /bills by session without jurisdiction restored missing Bill.from_organization field introduced new fields: Person.openstates_url, Bill.openstates_url","title":"2020.09.28"},{"location":"api-v3/changelog/#20200914","text":"removed some unused fields from responses removed deprecated government classification from Jurisdiction","title":"2020.09.14"},{"location":"api-v3/changelog/#20200910","text":"added Jurisdiction.legislative_sessions corrected initial pagination limits for release","title":"2020.09.10"},{"location":"api-v3/changelog/#20200909","text":"Initial beta release.","title":"2020.09.09"},{"location":"contributing/","text":"Getting Started \u00b6 We're glad to have you joining us, taking a few minutes to read the following pages will help you be a better member of our community: Our Code of Conduct is important to us, and helps us maintain a healthy community. We also have a guide to help you learn where to get help that you should look over. If you are new to open source, or unfamiliar to contributing to open source projects, it might be beneficial to read this guide . We welcome and value all contributions to Open States, including (but not limited to!) code contributions. This guide assumes a basic familiarity with using the command line , git , and Python. If you are unfamiliar with something that is mentioned in this guide, we encourage you to read the linked resources. No matter how experienced you are, it is a good idea to read through this section before diving into Open States' code. No worries if you aren't an expert though, we'll walk you through the steps. And as for Python, if you've written other languages like Javascript or Ruby you'll probably be just fine. Here's a great guide on getting started installing and managing Python versions. Don't be afraid to ask for help either! Project Overview \u00b6 Open States is a fairly large & somewhat complex project comprised of many moving parts with a long history. As you look to contribute, it may be beneficial to understand a little bit about the various components. These repositories make up the core of the project, if you're looking to contribute there's a 95% chance one of these is what you want. openstates-scrapers - Open States' scrapers. All code related to getting information from a website and storing it in the Open States database lives here. What is a scraper? people - Open States people & committee data, maintained as editable YAML files. openstates-core - Open States data model & scraper backend. openstates.org - Powers OpenStates.org website & GraphQL API. api-v3 - Powers API v3 . documentation - you're reading it now . Installing Prerequisites \u00b6 poetry \u00b6 If you're working on the people repo, api-v3 , or want to work on scrapers without Docker , you'll need poetry to build your Python virtual environment . Note If you haven't used poetry before, it is similar to pipenv , pip , and conda in that it manages a Python virtualenv on your behalf. Installing Poetry The official poetry docs recommend installing with: curl -sSL https://install.python-poetry.org | python3 - Then within each repo you check out, be sure to run: poetry install Which will fetch the correct version of dependencies. docker & docker-compose \u00b6 When working on scrapers or openstates.org, you have the option to use Docker. The first thing you will need to do is get a working docker environment on your local machine. We'll do this using Docker. No worries if you aren't familiar with Docker, you'll barely have to touch it beyond what this guide explains. Install Docker and docker-compose (if not already installed on your local system): (a) Installing Docker: On OSX: Docker for Mac On Windows: Docker for Windows On Linux: Use your package manager of choice or follow Docker's instructions . ( Docker Compose is probably already installed by step 1(a) if not, proceed to step 1(b) ) (b) Installing docker-compose: For easy installation on macOS, Windows, and 64-bit Linux. Ensure that Docker and docker-compose are installed locally: $ docker --version Docker version 19.03.4, build 9013bf5 $ docker-compose --version docker-compose version 1.24.1, build 4667896b Of course, your versions will differ, but ensure they are relatively recent to avoid strange issues. pre-commit \u00b6 To help keep the code as managable as possible we strongly recommend you use pre-commit to make sure all commits adhere to our preferred style. See pre-commit's installation instructions Within each repo you check out, run pre-commit install after checking out. It should look something like: $ pre-commit install pre-commit installed at .git/hooks/pre-commit Note If you're running flake8 and black yourself via your editor or similar this isn't strictly necessary, but we find it helps ensure commits don't fail linting. We require all PRs to pass linting! What is linting? Recent Major Work \u00b6 To give a sense of recent priorities, here are major milestones from the past few years: Federal Data & Committee Data - 2021 API v3 - Q3 2020 Legislation Tracking - Q1 2020 Restoration of Historical Legislator Data - Q4 2019 Full Text Search - Q4 2019 2019 Legislative Session Updates - Q1 2019 OpenStates.org 2019 rewrite - Q1 2019 OpenStates GraphQL API - Q4 2018 Scraper Overhaul - Throughout much of 2017 we reworked our scrapers to be more resilient and to use an updated tech stack, replacing the one that powered the site from 2011-2016.","title":"Getting Started"},{"location":"contributing/#getting-started","text":"We're glad to have you joining us, taking a few minutes to read the following pages will help you be a better member of our community: Our Code of Conduct is important to us, and helps us maintain a healthy community. We also have a guide to help you learn where to get help that you should look over. If you are new to open source, or unfamiliar to contributing to open source projects, it might be beneficial to read this guide . We welcome and value all contributions to Open States, including (but not limited to!) code contributions. This guide assumes a basic familiarity with using the command line , git , and Python. If you are unfamiliar with something that is mentioned in this guide, we encourage you to read the linked resources. No matter how experienced you are, it is a good idea to read through this section before diving into Open States' code. No worries if you aren't an expert though, we'll walk you through the steps. And as for Python, if you've written other languages like Javascript or Ruby you'll probably be just fine. Here's a great guide on getting started installing and managing Python versions. Don't be afraid to ask for help either!","title":"Getting Started"},{"location":"contributing/#project-overview","text":"Open States is a fairly large & somewhat complex project comprised of many moving parts with a long history. As you look to contribute, it may be beneficial to understand a little bit about the various components. These repositories make up the core of the project, if you're looking to contribute there's a 95% chance one of these is what you want. openstates-scrapers - Open States' scrapers. All code related to getting information from a website and storing it in the Open States database lives here. What is a scraper? people - Open States people & committee data, maintained as editable YAML files. openstates-core - Open States data model & scraper backend. openstates.org - Powers OpenStates.org website & GraphQL API. api-v3 - Powers API v3 . documentation - you're reading it now .","title":"Project Overview"},{"location":"contributing/#installing-prerequisites","text":"","title":"Installing Prerequisites"},{"location":"contributing/#poetry","text":"If you're working on the people repo, api-v3 , or want to work on scrapers without Docker , you'll need poetry to build your Python virtual environment . Note If you haven't used poetry before, it is similar to pipenv , pip , and conda in that it manages a Python virtualenv on your behalf. Installing Poetry The official poetry docs recommend installing with: curl -sSL https://install.python-poetry.org | python3 - Then within each repo you check out, be sure to run: poetry install Which will fetch the correct version of dependencies.","title":"poetry"},{"location":"contributing/#docker-docker-compose","text":"When working on scrapers or openstates.org, you have the option to use Docker. The first thing you will need to do is get a working docker environment on your local machine. We'll do this using Docker. No worries if you aren't familiar with Docker, you'll barely have to touch it beyond what this guide explains. Install Docker and docker-compose (if not already installed on your local system): (a) Installing Docker: On OSX: Docker for Mac On Windows: Docker for Windows On Linux: Use your package manager of choice or follow Docker's instructions . ( Docker Compose is probably already installed by step 1(a) if not, proceed to step 1(b) ) (b) Installing docker-compose: For easy installation on macOS, Windows, and 64-bit Linux. Ensure that Docker and docker-compose are installed locally: $ docker --version Docker version 19.03.4, build 9013bf5 $ docker-compose --version docker-compose version 1.24.1, build 4667896b Of course, your versions will differ, but ensure they are relatively recent to avoid strange issues.","title":"docker & docker-compose"},{"location":"contributing/#pre-commit","text":"To help keep the code as managable as possible we strongly recommend you use pre-commit to make sure all commits adhere to our preferred style. See pre-commit's installation instructions Within each repo you check out, run pre-commit install after checking out. It should look something like: $ pre-commit install pre-commit installed at .git/hooks/pre-commit Note If you're running flake8 and black yourself via your editor or similar this isn't strictly necessary, but we find it helps ensure commits don't fail linting. We require all PRs to pass linting! What is linting?","title":"pre-commit"},{"location":"contributing/#recent-major-work","text":"To give a sense of recent priorities, here are major milestones from the past few years: Federal Data & Committee Data - 2021 API v3 - Q3 2020 Legislation Tracking - Q1 2020 Restoration of Historical Legislator Data - Q4 2019 Full Text Search - Q4 2019 2019 Legislative Session Updates - Q1 2019 OpenStates.org 2019 rewrite - Q1 2019 OpenStates GraphQL API - Q4 2018 Scraper Overhaul - Throughout much of 2017 we reworked our scrapers to be more resilient and to use an updated tech stack, replacing the one that powered the site from 2011-2016.","title":"Recent Major Work"},{"location":"contributing/documentation/","text":"Documentation \u00b6 If you notice any issues on these docs, or just want to help improve them the source is in the openstates/documentation repository. Checking out \u00b6 Fork and clone the documentation repository: Visit https://github.com/openstates/documentation and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/documentation.git Cloning into 'documentation'.. Building Docs Locally \u00b6 Step 1) Install poetry if you haven't already ( https://python-poetry.org/docs/ ) Step 2) Run poetry install to build virtualenv. Step 3) Run poetry run mkdocs serve to preview changes in browser.","title":"Documentation"},{"location":"contributing/documentation/#documentation","text":"If you notice any issues on these docs, or just want to help improve them the source is in the openstates/documentation repository.","title":"Documentation"},{"location":"contributing/documentation/#checking-out","text":"Fork and clone the documentation repository: Visit https://github.com/openstates/documentation and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/documentation.git Cloning into 'documentation'..","title":"Checking out"},{"location":"contributing/documentation/#building-docs-locally","text":"Step 1) Install poetry if you haven't already ( https://python-poetry.org/docs/ ) Step 2) Run poetry install to build virtualenv. Step 3) Run poetry run mkdocs serve to preview changes in browser.","title":"Building Docs Locally"},{"location":"contributing/local-database/","text":"Running a Local Database \u00b6 If you want to ensure your scraped data imports or work on OpenStates.org or API v3, you'll need a local database. This can be a bit cumbersome, since running Postgres locally varies a lot platform-to-platform, and you'll need to populate it as well. If you're comfortable with Postgres, most of these steps can be easily modified to use your own Postgres instance, but for the remainder of this guide we'll be using a dockerized postgres image. Prerequisites \u00b6 Be sure you've already installed docker and docker-compose , as noted in Installing Prerequisites . You'll need openstates-scrapers checked out, even if you aren't working on scrapers. This repository has the docker-compose.yml config and initialization scripts for the database. If you want to initialize the database for openstates.org work you'll need that project checked out as well. Initialize Database For Scraping \u00b6 Run init-db.sh from within the openstates-scrapers directory: Warning If you've already run this before, running scripts/init-db.sh will reset your database to scratch! openstates-scrapers/$ ./scripts/init-db.sh + unset DATABASE_URL + docker-compose down Removing scrapers_db_1 ... done Removing network openstates-network + docker volume rm openstates-postgres openstates-postgres + docker-compose up -d db Creating network \"openstates-network\" with the default driver Creating volume \"openstates-postgres\" with default driver Creating scrapers_db_1 ... done + sleep 3 + DATABASE_URL=postgis://openstates:openstates@db/openstatesorg + docker-compose run --rm --entrypoint 'poetry run os-initdb' scrape Creating scrapers_scrape_run ... done Operations to perform: Apply all migrations: contenttypes, data Running migrations: Applying contenttypes.0001_initial... OK Applying contenttypes.0002_remove_content_type_name... OK Applying data.0001_initial... OK Applying data.0002_auto_20200422_0028... OK Applying data.0003_auto_20200422_0031... OK ...TRUNCATED... loading WY loading DC loading PR loading US This will populate your database with the tables needed for scraping, as well as some basic static data such as the jurisdiction metadata. If all you want to do is run scrapers and import data into a database for inspection, you're good to go! Initialize Database for OpenStates.org \u00b6 Note You must run openstates-scrapers ' init-db.sh as shown above first! From within the openstates.org directory run docker/init-db.sh : openstates.org/$ ./docker/init-db.sh + unset DATABASE_URL + docker-compose run --rm -e PYTHONPATH=docker/ --entrypoint 'poetry run ./manage.py migrate' django Creating openstatesorg_django_run ... done Operations to perform: Apply all migrations: account, admin, auth, bulk, bundles, contenttypes, dashboards, data, people_admin, profiles, sessions, sites, socialaccount Running migrations: Applying auth.0001_initial... OK ...TRUNCATED... docker-compose run --rm -e PYTHONPATH=docker/ --entrypoint 'poetry run ./manage.py shell -c \"import testdata\"' django This creates the Django-specific tables, and also creates a local API key testkey that can be used for local development. Working with the Local Database \u00b6 The database will persist to disk, so for the most part once you run these steps you're good to go. Starting the Database \u00b6 You'll need to make sure that database is running whenever you're working on scrapers or OpenStates.org locally. You can do that by running docker-compose up -d db from the openstates-scrapers directory. openstates-scrapers$ docker-compose up -d db Starting scrapers_db_1 ... done If it is already running output will look like: openstates-scrapers$ docker-compose up -d db scrapers_db_1 is up-to-date Stopping the Database \u00b6 openstates-scrapers$ docker-compose stop db Stopping scrapers_db_1 ... done Resetting the Database \u00b6 You can always run scripts/init-db.sh to reset your database. This can be good if you have some bad data, or just whenever you'd like a fresh start: openstates-scrapers/$ ./scripts/init-db.sh","title":"Running a Local Database"},{"location":"contributing/local-database/#running-a-local-database","text":"If you want to ensure your scraped data imports or work on OpenStates.org or API v3, you'll need a local database. This can be a bit cumbersome, since running Postgres locally varies a lot platform-to-platform, and you'll need to populate it as well. If you're comfortable with Postgres, most of these steps can be easily modified to use your own Postgres instance, but for the remainder of this guide we'll be using a dockerized postgres image.","title":"Running a Local Database"},{"location":"contributing/local-database/#prerequisites","text":"Be sure you've already installed docker and docker-compose , as noted in Installing Prerequisites . You'll need openstates-scrapers checked out, even if you aren't working on scrapers. This repository has the docker-compose.yml config and initialization scripts for the database. If you want to initialize the database for openstates.org work you'll need that project checked out as well.","title":"Prerequisites"},{"location":"contributing/local-database/#initialize-database-for-scraping","text":"Run init-db.sh from within the openstates-scrapers directory: Warning If you've already run this before, running scripts/init-db.sh will reset your database to scratch! openstates-scrapers/$ ./scripts/init-db.sh + unset DATABASE_URL + docker-compose down Removing scrapers_db_1 ... done Removing network openstates-network + docker volume rm openstates-postgres openstates-postgres + docker-compose up -d db Creating network \"openstates-network\" with the default driver Creating volume \"openstates-postgres\" with default driver Creating scrapers_db_1 ... done + sleep 3 + DATABASE_URL=postgis://openstates:openstates@db/openstatesorg + docker-compose run --rm --entrypoint 'poetry run os-initdb' scrape Creating scrapers_scrape_run ... done Operations to perform: Apply all migrations: contenttypes, data Running migrations: Applying contenttypes.0001_initial... OK Applying contenttypes.0002_remove_content_type_name... OK Applying data.0001_initial... OK Applying data.0002_auto_20200422_0028... OK Applying data.0003_auto_20200422_0031... OK ...TRUNCATED... loading WY loading DC loading PR loading US This will populate your database with the tables needed for scraping, as well as some basic static data such as the jurisdiction metadata. If all you want to do is run scrapers and import data into a database for inspection, you're good to go!","title":"Initialize Database For Scraping"},{"location":"contributing/local-database/#initialize-database-for-openstatesorg","text":"Note You must run openstates-scrapers ' init-db.sh as shown above first! From within the openstates.org directory run docker/init-db.sh : openstates.org/$ ./docker/init-db.sh + unset DATABASE_URL + docker-compose run --rm -e PYTHONPATH=docker/ --entrypoint 'poetry run ./manage.py migrate' django Creating openstatesorg_django_run ... done Operations to perform: Apply all migrations: account, admin, auth, bulk, bundles, contenttypes, dashboards, data, people_admin, profiles, sessions, sites, socialaccount Running migrations: Applying auth.0001_initial... OK ...TRUNCATED... docker-compose run --rm -e PYTHONPATH=docker/ --entrypoint 'poetry run ./manage.py shell -c \"import testdata\"' django This creates the Django-specific tables, and also creates a local API key testkey that can be used for local development.","title":"Initialize Database for OpenStates.org"},{"location":"contributing/local-database/#working-with-the-local-database","text":"The database will persist to disk, so for the most part once you run these steps you're good to go.","title":"Working with the Local Database"},{"location":"contributing/local-database/#starting-the-database","text":"You'll need to make sure that database is running whenever you're working on scrapers or OpenStates.org locally. You can do that by running docker-compose up -d db from the openstates-scrapers directory. openstates-scrapers$ docker-compose up -d db Starting scrapers_db_1 ... done If it is already running output will look like: openstates-scrapers$ docker-compose up -d db scrapers_db_1 is up-to-date","title":"Starting the Database"},{"location":"contributing/local-database/#stopping-the-database","text":"openstates-scrapers$ docker-compose stop db Stopping scrapers_db_1 ... done","title":"Stopping the Database"},{"location":"contributing/local-database/#resetting-the-database","text":"You can always run scripts/init-db.sh to reset your database. This can be good if you have some bad data, or just whenever you'd like a fresh start: openstates-scrapers/$ ./scripts/init-db.sh","title":"Resetting the Database"},{"location":"contributing/openstates-org/","text":"Working On openstates.org \u00b6 openstates.org is the public-facing result of all the work we do. The site is built in Django and includes the web frontend and API. Checking out \u00b6 Fork and clone the openstates.org repository: Visit https://github.com/openstates/openstates.org and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/openstates.org.git Cloning into 'openstates.org'... Be sure to run poetry install to fetch the correct version of dependencies. And remember to install pre-commit : $ pre-commit install pre-commit installed at .git/hooks/pre-commit Getting a working database \u00b6 See Running a Local Database to get your database ready for OpenStates.org. Running Tests \u00b6 You can run the tests for the project via: ./docker/run-tests.sh You can also append standard pytest arguments such as -x to bail on first failure. Example of running just the graphapi tests, bailing on error: ./docker/run-tests.sh graphapi -x Repository overview \u00b6 The project is rather large, with quite a few django apps, here's a quick guide: Django Apps: bulk/ - handles bulk downloads on the website dashboards/ - dashboards for viewing various statistics geo/ - geography services for legislator lookup graphapi/ - powers GraphQL API profiles/ - user and subscription management public/ - public-facing pages (bulk of the site) utils/ - utilities shared by the other applications Other Stuff: ansible/ - the files used to deploy OpenStates.org are here docker/ - special scripts for running tests, etc. within docker openstates/ - core Django settings files static/ - various static assets, including frontend code templates/ - Django templates Running openstates.org \u00b6 Simply running docker-compose up should start django & the database, then browse to http://localhost:8000 and you'll be looking at your own local copy of openstates.org. In a separate terminal window, run npm run build and npm run start to see the site's react and style components. Note If you're running into issues with models not being found or an incorrectly configured virtual environment, running docker-compose build should help to fix it. If you have issues getting your instance up and running, please document the errors you're seeing and reach out . Running outside of Docker \u00b6 It might be desirable to test outside of docker sometimes to bypass caching or other issues that make development within the docker environment difficult. If so, you can install goreman (or any foreman clone) and run goreman start .","title":"Working On openstates.org"},{"location":"contributing/openstates-org/#working-on-openstatesorg","text":"openstates.org is the public-facing result of all the work we do. The site is built in Django and includes the web frontend and API.","title":"Working On openstates.org"},{"location":"contributing/openstates-org/#checking-out","text":"Fork and clone the openstates.org repository: Visit https://github.com/openstates/openstates.org and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/openstates.org.git Cloning into 'openstates.org'... Be sure to run poetry install to fetch the correct version of dependencies. And remember to install pre-commit : $ pre-commit install pre-commit installed at .git/hooks/pre-commit","title":"Checking out"},{"location":"contributing/openstates-org/#getting-a-working-database","text":"See Running a Local Database to get your database ready for OpenStates.org.","title":"Getting a working database"},{"location":"contributing/openstates-org/#running-tests","text":"You can run the tests for the project via: ./docker/run-tests.sh You can also append standard pytest arguments such as -x to bail on first failure. Example of running just the graphapi tests, bailing on error: ./docker/run-tests.sh graphapi -x","title":"Running Tests"},{"location":"contributing/openstates-org/#repository-overview","text":"The project is rather large, with quite a few django apps, here's a quick guide: Django Apps: bulk/ - handles bulk downloads on the website dashboards/ - dashboards for viewing various statistics geo/ - geography services for legislator lookup graphapi/ - powers GraphQL API profiles/ - user and subscription management public/ - public-facing pages (bulk of the site) utils/ - utilities shared by the other applications Other Stuff: ansible/ - the files used to deploy OpenStates.org are here docker/ - special scripts for running tests, etc. within docker openstates/ - core Django settings files static/ - various static assets, including frontend code templates/ - Django templates","title":"Repository overview"},{"location":"contributing/openstates-org/#running-openstatesorg","text":"Simply running docker-compose up should start django & the database, then browse to http://localhost:8000 and you'll be looking at your own local copy of openstates.org. In a separate terminal window, run npm run build and npm run start to see the site's react and style components. Note If you're running into issues with models not being found or an incorrectly configured virtual environment, running docker-compose build should help to fix it. If you have issues getting your instance up and running, please document the errors you're seeing and reach out .","title":"Running openstates.org"},{"location":"contributing/openstates-org/#running-outside-of-docker","text":"It might be desirable to test outside of docker sometimes to bypass caching or other issues that make development within the docker environment difficult. If so, you can install goreman (or any foreman clone) and run goreman start .","title":"Running outside of Docker"},{"location":"contributing/people/","text":"Contributing People Data \u00b6 Person data is maintained in the openstates/people repository. This repository contains YAML files with all the information on given individuals and committees. Info Please note that this portion of the project is in the public domain in the United States with all copyright waived via a CC0 dedication. By contributing you agree to waive all copyright claims. Checking out \u00b6 Fork and clone the people repository: Visit https://github.com/openstates/people and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/people.git Cloning into 'people'.. Build the environment with poetry : $ poetry install Installing dependencies from lock file ... And remember to install pre-commit hooks: $ pre-commit install pre-commit installed at .git/hooks/pre-commit Repository overview \u00b6 The repository consists of a few key components: settings.yml Settings for state legislatures, including the number of seats, and current vacancies. data/ Data files in YAML format on legislators, organized by state & status. You can use the os-people and os-committees commands to manage the data: poetry run os-people --help or poetry run os-committees --help Common tasks \u00b6 Updating legislator data by hand \u00b6 Let's say you call a legislator and find out that they have a new phone number, contribute back! See schema.md for details on the acceptable fields. If you're looking to add a lot of data but unsure where it fits feel free to ask via an issue and we can either amend the schema or make a recommendation. Start a new branch for this work Make the edits you need in the appropriate YAML file. Please keep edits to a minimum (e.g. don't re-order fields) Submit a PR, please describe how you came across this information to expedite review. Retiring a legislator \u00b6 Start a new branch for this work Add an end_date to their most recent role within the appropriate legislator's YAML file Run poetry run os-people retire with the appropriate legislator file(s) from the root directory Review the automatically edited files & submit a PR. Updating an entire state's legislators via a scrape \u00b6 Let's say a North Carolina has had an election & it makes sense to re-scrape everything for that state. Start a new branch for this work (e.g. nc-2021-people-update ) Scrape data using Open States' Scrapers Run poetry run os-people merge nc scrapes/2021-01-01/001 against the generated JSON data from the scrape Manually reconcile remaining changes, will often require some retirements as well. Check that data looks clean with poetry run os-people lint nc --summary commit your changes and prepare a PR. Example of the process: (In this example, we assume the people repo is stored at ~/gitroot/people and the openstates-scrapers repo is stored at ~/gitroot/openstates-scrapers ) :#~/gitroot/openstates-scrapers$ poetry run spatula scrape scrapers_next.de.people.House INFO:scrapers_next.de.people.House:fetching https://legis.delaware.gov/json/House/GetRepresentatives INFO:scrapers_next.de.people.LegDetail:fetching https://legis.delaware.gov/LegislatorDetail?personId = 13589 INFO:scrapers_next.de.people.LegDetail:fetching https://legis.delaware.gov/LegislatorDetail?personId = 332 ... success: wrote 41 objects to _scrapes/2022-06-17/001 :#~/gitroot/openstates-scrapers$ OS_PEOPLE_DIRECTORY = ~/gitroot/people poetry run os-people merge de _scrapes/2022-06-17/001/ analyzing 120 existing people and 41 scraped perfect match perfect match perfect match other_names: append { 'start_date' : '' , 'end_date' : '' , 'name' : 'Gerald L. Brady' } name: Gerald L. Brady = > Charles \"Bud\" M. Freel email: Gerald.Brady@delaware.gov = > Bud.Freel@delaware.gov offices changed from: Capitol Office address = 411 Legislative Ave. Dover, DE 19901 voice = 302 -744-4351 to Capitol Office address = 411 Legislative Avenue Dover, DE 19901 voice = 302 -744-4351 links: append { 'url' : 'https://legis.delaware.gov/LegislatorDetail?personId=24197' , 'note' : 'homepage' } sources: append { 'url' : 'https://legis.delaware.gov/LegislatorDetail?personId=24197' , 'note' : '' } ( m ) erge? ( r ) etire Gerald L. Brady? ( s ) kip? ( a ) bort? Aborted! This example stops at step #4 because the final steps all require manual work. We can use --retirement to automatically retire members during this process, but this requires knowing that all potentially retired members happened at the same time.","title":"Contributing People Data"},{"location":"contributing/people/#contributing-people-data","text":"Person data is maintained in the openstates/people repository. This repository contains YAML files with all the information on given individuals and committees. Info Please note that this portion of the project is in the public domain in the United States with all copyright waived via a CC0 dedication. By contributing you agree to waive all copyright claims.","title":"Contributing People Data"},{"location":"contributing/people/#checking-out","text":"Fork and clone the people repository: Visit https://github.com/openstates/people and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/people.git Cloning into 'people'.. Build the environment with poetry : $ poetry install Installing dependencies from lock file ... And remember to install pre-commit hooks: $ pre-commit install pre-commit installed at .git/hooks/pre-commit","title":"Checking out"},{"location":"contributing/people/#repository-overview","text":"The repository consists of a few key components: settings.yml Settings for state legislatures, including the number of seats, and current vacancies. data/ Data files in YAML format on legislators, organized by state & status. You can use the os-people and os-committees commands to manage the data: poetry run os-people --help or poetry run os-committees --help","title":"Repository overview"},{"location":"contributing/people/#common-tasks","text":"","title":"Common tasks"},{"location":"contributing/people/#updating-legislator-data-by-hand","text":"Let's say you call a legislator and find out that they have a new phone number, contribute back! See schema.md for details on the acceptable fields. If you're looking to add a lot of data but unsure where it fits feel free to ask via an issue and we can either amend the schema or make a recommendation. Start a new branch for this work Make the edits you need in the appropriate YAML file. Please keep edits to a minimum (e.g. don't re-order fields) Submit a PR, please describe how you came across this information to expedite review.","title":"Updating legislator data by hand"},{"location":"contributing/people/#retiring-a-legislator","text":"Start a new branch for this work Add an end_date to their most recent role within the appropriate legislator's YAML file Run poetry run os-people retire with the appropriate legislator file(s) from the root directory Review the automatically edited files & submit a PR.","title":"Retiring a legislator"},{"location":"contributing/people/#updating-an-entire-states-legislators-via-a-scrape","text":"Let's say a North Carolina has had an election & it makes sense to re-scrape everything for that state. Start a new branch for this work (e.g. nc-2021-people-update ) Scrape data using Open States' Scrapers Run poetry run os-people merge nc scrapes/2021-01-01/001 against the generated JSON data from the scrape Manually reconcile remaining changes, will often require some retirements as well. Check that data looks clean with poetry run os-people lint nc --summary commit your changes and prepare a PR. Example of the process: (In this example, we assume the people repo is stored at ~/gitroot/people and the openstates-scrapers repo is stored at ~/gitroot/openstates-scrapers ) :#~/gitroot/openstates-scrapers$ poetry run spatula scrape scrapers_next.de.people.House INFO:scrapers_next.de.people.House:fetching https://legis.delaware.gov/json/House/GetRepresentatives INFO:scrapers_next.de.people.LegDetail:fetching https://legis.delaware.gov/LegislatorDetail?personId = 13589 INFO:scrapers_next.de.people.LegDetail:fetching https://legis.delaware.gov/LegislatorDetail?personId = 332 ... success: wrote 41 objects to _scrapes/2022-06-17/001 :#~/gitroot/openstates-scrapers$ OS_PEOPLE_DIRECTORY = ~/gitroot/people poetry run os-people merge de _scrapes/2022-06-17/001/ analyzing 120 existing people and 41 scraped perfect match perfect match perfect match other_names: append { 'start_date' : '' , 'end_date' : '' , 'name' : 'Gerald L. Brady' } name: Gerald L. Brady = > Charles \"Bud\" M. Freel email: Gerald.Brady@delaware.gov = > Bud.Freel@delaware.gov offices changed from: Capitol Office address = 411 Legislative Ave. Dover, DE 19901 voice = 302 -744-4351 to Capitol Office address = 411 Legislative Avenue Dover, DE 19901 voice = 302 -744-4351 links: append { 'url' : 'https://legis.delaware.gov/LegislatorDetail?personId=24197' , 'note' : 'homepage' } sources: append { 'url' : 'https://legis.delaware.gov/LegislatorDetail?personId=24197' , 'note' : '' } ( m ) erge? ( r ) etire Gerald L. Brady? ( s ) kip? ( a ) bort? Aborted! This example stops at step #4 because the final steps all require manual work. We can use --retirement to automatically retire members during this process, but this requires knowing that all potentially retired members happened at the same time.","title":"Updating an entire state's legislators via a scrape"},{"location":"contributing/scrapers/","text":"Contributing to Scrapers \u00b6 Scrapers are at the core of what Open States does, each state requires several custom scrapers designed to extract bills, legislators, committees, and votes from the state website. All together there are around 200 scrapers, each one essentially independent, which means that there is always more work to do, but fortunately plenty of prior work to learn from. Checking Out \u00b6 Fork and clone the main scraper repository: Visit https://github.com/openstates/openstates-scrapers and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/openstates-scrapers.git Cloning into 'openstates-scrapers'... And remember to install pre-commit hooks: $ pre-commit install pre-commit installed at .git/hooks/pre-commit Be sure to run poetry install to fetch the correct version of dependencies. Warning Before cloning on a Windows computer, you will need to disable line-ending conversion. git config --global core.autocrlf false After cloning and entering the repo, you'll likely want to set global line-ending conversion back to true, and set local conversion to false. Repository Overview \u00b6 At this point you'll have a local openstates-scrapers directory. Within it, you'll find a directory called scrapers , lets take a look at it: $ ls scrapers __init__.py dc in mn nj pr va ak de ks mo nm ri vi al fl ky ms nv sc vt ar ga la mt ny sd wa az hi ma nc oh tn wi ca ia md nd ok tx wv co id me ne or ut wy ct il mi nh pa utils This directory has 50+ python modules, one for each state. Let's look inside one: $ ls scrapers/nc __init__.py bills.py votes.py Some states' directories will differ a bit, but all will have __init__.py and bills.py . The __init__.py file for each state has basic metadata on the state including a list of sessions. Other files contain the scrapers, typically named bills , votes , etc. At the root, you'll also find a directory called scrapers_next . This directory also has python modules for each state. Inside a state, you'll find people and potentially committee scrapers written using spatula . The plan is to port all scrapers to this framework and have scrapers_next replace the scraper directory. Running Your First Scraper \u00b6 Let's run your state's bills scraper (substitute your state for 'nc' below) : $ docker-compose run --rm scrape nc bills --fastmode --scrape The parameters you pass after docker-compose run --rm scrape are passed to os-update . Here we're saying that we're running NC's scrapers, and that we want to do it in \"fast mode\". By default, os-update imports results into a postgres database; the --scrape flag skips that step. The following arguments are optional: To bring up a list of the optional arguments in the CLI use -h , --help \\ -h , --help show this help message and exit\\ --debug open debugger on error\\ --loglevel {LOGLEVEL} set log level. options are: DEBUG|INFO|WARNING|ERROR|CRITICAL (default is INFO) \\ --scrape only run scrape post-scrape step\\ --import only run import post-scrape step\\ --nonstrict skip validation on save\\ --fastmode use cache and turn off throttling\\ --datadir {SCRAPED_DATA_DIR} data directory\\ --cachedir {CACHE_DIR} cache directory\\ -r {SCRAPELIB_RPM} scraper rpm\\ --rpm {SCRAPELIB_RPM} scraper rpm\\ --timeout {SCRAPELIB_TIMEOUT} scraper timeout\\ --no-verify skip tls verification\\ --retries {SCRAPELIB_RETRIES} scraper retries\\ --retry_wait {SCRAPELIB_RETRY_WAIT_SECONDS} scraper retry wait\\ --realtime loads bills in realtime to database, this requires configuring an AWS S3 bucket and using the lambda function: openstates-realtime You'll see the run plan , which is what the update aims to capture; in this case we're scraping the state website's data into JSON files: nc (scrape) bills: {} Then legislative posts and organizations get created, which is mostly boilerplate: 08:46:35 INFO openstates: save jurisdiction North Carolina as jurisdiction_ocd-jurisdiction-country:us-state:nc-government.json 08:46:35 INFO openstates: save organization North Carolina General Assembly as organization_01d6327c-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save organization Executive Office of the Governor as organization_01d63560-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save organization Senate as organization_01d636e6-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 1 as post_01d63a06-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 2 as post_01d63b96-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 3 as post_01d63cea-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 4 as post_01d63e34-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 5 as post_01d63f74-72d2-11e7-8df8-0242ac130003.json And then the actual data scraping begins, defaulting to the most recent legislative session: 08:46:36 INFO openstates: no session specified, using 2017 08:46:36 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/SimpleBillInquiry/displaybills.pl?Session=2017&tab=Chamber&Chamber=Senate 08:46:38 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S1 08:46:39 INFO openstates: save bill SR 1 in 2017 as bill_03c7edb4-72d2-11e7-8df8-0242ac130003.json 08:46:39 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S2 08:46:39 INFO openstates: save bill SJR 2 in 2017 as bill_044a5fc4-72d2-11e7-8df8-0242ac130003.json 08:46:39 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S3 08:46:40 INFO openstates: save bill SB 3 in 2017 as bill_04e8c66e-72d2-11e7-8df8-0242ac130003.json 08:46:40 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S4 08:46:41 INFO openstates: save bill SB 4 in 2017 as bill_05781f08-72d2-11e7-8df8-0242ac130003.json 08:46:41 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S5 Depending on the scraper you run, this part takes a while. Some scrapers can take hours to run depending on the number of bills and speed of the state's website. Note It is often desirable to bail out of running the whole scrape (Ctrl-C) after it has gotten a bit of data, instead of letting it run the entire scrape. To review the data you just fetched, you can browse the _data/nc/ directory and inspect the JSON files. If you're trying to make a small fix this is often sufficient, you can confirm that the scraped data looks correct and move on. Note It is of course possible that the scrape fails. If so, there's a good chance that isn't your fault, especially if it starts to run and then errors out. Scrapers do break, and there's no guarantee North Carolina didn't change their legislator page yesterday, breaking our tutorial here. If that's the case and you think the issue is with the scraper, feel free to get in touch with us or file an issue . At this point you're ready to run scrapers and contribute fixes. Hop onto our GitHub ticket queue , pick an issue to solve, and then submit a Pull Request! Importing Data \u00b6 Optionally, if you'd like to see how your scraped data imports into the database, perhaps to diagnose an issue that is happening after the scrape, pop over to getting a working database to see how to get a local database that you can import data into. Once that's done, make sure that the db image from openstates.org is running: $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 27fe691ad7c5 mdillon/postgis:11-alpine \"docker-entrypoint.s\u2026\" 3 hours ago Up 3 hours 0.0.0.0:5405->5432/tcp openstatesorg_db_1 Your output will vary, but if you don't see something named openstatesorg_db running you should run this command (from the openstates.org directory, not your scraper directory): $ docker-compose up -d db Now, when you want to run imports, you can drop the --scrape portion of the command you've been running. Or if you just want to test the import of already scraped data you can replace it with --import . An import looks something like this: $ docker-compose run --rm scrape fl bills --fast ... (truncated) ... 23:03:34 ERROR openstates: cannot resolve pseudo id to Person: ~{\u201cname\u201d: \u201cGrant, M.\u201c} 23:03:36 ERROR openstates: cannot resolve pseudo id to Person: ~{\u201cname\u201d: \u201cRodrigues, R.\u201c} fl (import) bills: {} import: bill: 0 new 0 updated 2620 noop jurisdiction: 0 new 0 updated 1 noop vote_event: 21 new 12 updated 533 noop The errors about unresolved psuedo-ids can safely be ignored, as long as you see the final run report the data you scraped is available in your database. The number of objects of each type that were created & updated are available for spot checking, as well as the total number of items that were seen that already exactly matched what was in the database. These can be useful stats as you try to see if your local changes to a scraper have the impact you expect. Running Spatula Scrapers \u00b6 Let's run a people scraper: $ poetry run spatula scrape scrapers_next.nc.people.SenList The command to run these scrapers is structured differently, as the parameters are set by giving the exact location of the function you want to run: directory.state.file.function . Note Function names do vary and scrapes for legislators are commonly split by chamber, so make sure to check you're passing the right function in your command. The actual data scraping should look something like: INFO:scrapers_next.nc.people.SenList:fetching https://www.ncleg.gov/Members/MemberTable/S INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/430 INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/431 INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/432 INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/433 INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/434 To review the data you scraped, you can inspect the JSON files in the dated directory within _scrapes/ . Each time you run a scrape, a new numbered folder will be within the dated directory, so you can compare older data to new easily. Note If a scrape fails, it's likely an issue with the scraper. Feel free to get in touch with us or file an issue . Spatula is incredibly powerful with lots of flexibility and useful CLI commands that are worth checking out as well. At this point you're ready to run spatula scrapers and contribute fixes. Hop onto our GitHub ticket queue , pick an issue to solve, and then submit a Pull Request!","title":"Contributing to Scrapers"},{"location":"contributing/scrapers/#contributing-to-scrapers","text":"Scrapers are at the core of what Open States does, each state requires several custom scrapers designed to extract bills, legislators, committees, and votes from the state website. All together there are around 200 scrapers, each one essentially independent, which means that there is always more work to do, but fortunately plenty of prior work to learn from.","title":"Contributing to Scrapers"},{"location":"contributing/scrapers/#checking-out","text":"Fork and clone the main scraper repository: Visit https://github.com/openstates/openstates-scrapers and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/openstates-scrapers.git Cloning into 'openstates-scrapers'... And remember to install pre-commit hooks: $ pre-commit install pre-commit installed at .git/hooks/pre-commit Be sure to run poetry install to fetch the correct version of dependencies. Warning Before cloning on a Windows computer, you will need to disable line-ending conversion. git config --global core.autocrlf false After cloning and entering the repo, you'll likely want to set global line-ending conversion back to true, and set local conversion to false.","title":"Checking Out"},{"location":"contributing/scrapers/#repository-overview","text":"At this point you'll have a local openstates-scrapers directory. Within it, you'll find a directory called scrapers , lets take a look at it: $ ls scrapers __init__.py dc in mn nj pr va ak de ks mo nm ri vi al fl ky ms nv sc vt ar ga la mt ny sd wa az hi ma nc oh tn wi ca ia md nd ok tx wv co id me ne or ut wy ct il mi nh pa utils This directory has 50+ python modules, one for each state. Let's look inside one: $ ls scrapers/nc __init__.py bills.py votes.py Some states' directories will differ a bit, but all will have __init__.py and bills.py . The __init__.py file for each state has basic metadata on the state including a list of sessions. Other files contain the scrapers, typically named bills , votes , etc. At the root, you'll also find a directory called scrapers_next . This directory also has python modules for each state. Inside a state, you'll find people and potentially committee scrapers written using spatula . The plan is to port all scrapers to this framework and have scrapers_next replace the scraper directory.","title":"Repository Overview"},{"location":"contributing/scrapers/#running-your-first-scraper","text":"Let's run your state's bills scraper (substitute your state for 'nc' below) : $ docker-compose run --rm scrape nc bills --fastmode --scrape The parameters you pass after docker-compose run --rm scrape are passed to os-update . Here we're saying that we're running NC's scrapers, and that we want to do it in \"fast mode\". By default, os-update imports results into a postgres database; the --scrape flag skips that step. The following arguments are optional: To bring up a list of the optional arguments in the CLI use -h , --help \\ -h , --help show this help message and exit\\ --debug open debugger on error\\ --loglevel {LOGLEVEL} set log level. options are: DEBUG|INFO|WARNING|ERROR|CRITICAL (default is INFO) \\ --scrape only run scrape post-scrape step\\ --import only run import post-scrape step\\ --nonstrict skip validation on save\\ --fastmode use cache and turn off throttling\\ --datadir {SCRAPED_DATA_DIR} data directory\\ --cachedir {CACHE_DIR} cache directory\\ -r {SCRAPELIB_RPM} scraper rpm\\ --rpm {SCRAPELIB_RPM} scraper rpm\\ --timeout {SCRAPELIB_TIMEOUT} scraper timeout\\ --no-verify skip tls verification\\ --retries {SCRAPELIB_RETRIES} scraper retries\\ --retry_wait {SCRAPELIB_RETRY_WAIT_SECONDS} scraper retry wait\\ --realtime loads bills in realtime to database, this requires configuring an AWS S3 bucket and using the lambda function: openstates-realtime You'll see the run plan , which is what the update aims to capture; in this case we're scraping the state website's data into JSON files: nc (scrape) bills: {} Then legislative posts and organizations get created, which is mostly boilerplate: 08:46:35 INFO openstates: save jurisdiction North Carolina as jurisdiction_ocd-jurisdiction-country:us-state:nc-government.json 08:46:35 INFO openstates: save organization North Carolina General Assembly as organization_01d6327c-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save organization Executive Office of the Governor as organization_01d63560-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save organization Senate as organization_01d636e6-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 1 as post_01d63a06-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 2 as post_01d63b96-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 3 as post_01d63cea-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 4 as post_01d63e34-72d2-11e7-8df8-0242ac130003.json 08:46:35 INFO openstates: save post 5 as post_01d63f74-72d2-11e7-8df8-0242ac130003.json And then the actual data scraping begins, defaulting to the most recent legislative session: 08:46:36 INFO openstates: no session specified, using 2017 08:46:36 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/SimpleBillInquiry/displaybills.pl?Session=2017&tab=Chamber&Chamber=Senate 08:46:38 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S1 08:46:39 INFO openstates: save bill SR 1 in 2017 as bill_03c7edb4-72d2-11e7-8df8-0242ac130003.json 08:46:39 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S2 08:46:39 INFO openstates: save bill SJR 2 in 2017 as bill_044a5fc4-72d2-11e7-8df8-0242ac130003.json 08:46:39 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S3 08:46:40 INFO openstates: save bill SB 3 in 2017 as bill_04e8c66e-72d2-11e7-8df8-0242ac130003.json 08:46:40 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S4 08:46:41 INFO openstates: save bill SB 4 in 2017 as bill_05781f08-72d2-11e7-8df8-0242ac130003.json 08:46:41 INFO scrapelib: GET - http://www.ncga.state.nc.us/gascripts/BillLookUp/BillLookUp.pl?Session=2017&BillID=S5 Depending on the scraper you run, this part takes a while. Some scrapers can take hours to run depending on the number of bills and speed of the state's website. Note It is often desirable to bail out of running the whole scrape (Ctrl-C) after it has gotten a bit of data, instead of letting it run the entire scrape. To review the data you just fetched, you can browse the _data/nc/ directory and inspect the JSON files. If you're trying to make a small fix this is often sufficient, you can confirm that the scraped data looks correct and move on. Note It is of course possible that the scrape fails. If so, there's a good chance that isn't your fault, especially if it starts to run and then errors out. Scrapers do break, and there's no guarantee North Carolina didn't change their legislator page yesterday, breaking our tutorial here. If that's the case and you think the issue is with the scraper, feel free to get in touch with us or file an issue . At this point you're ready to run scrapers and contribute fixes. Hop onto our GitHub ticket queue , pick an issue to solve, and then submit a Pull Request!","title":"Running Your First Scraper"},{"location":"contributing/scrapers/#importing-data","text":"Optionally, if you'd like to see how your scraped data imports into the database, perhaps to diagnose an issue that is happening after the scrape, pop over to getting a working database to see how to get a local database that you can import data into. Once that's done, make sure that the db image from openstates.org is running: $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 27fe691ad7c5 mdillon/postgis:11-alpine \"docker-entrypoint.s\u2026\" 3 hours ago Up 3 hours 0.0.0.0:5405->5432/tcp openstatesorg_db_1 Your output will vary, but if you don't see something named openstatesorg_db running you should run this command (from the openstates.org directory, not your scraper directory): $ docker-compose up -d db Now, when you want to run imports, you can drop the --scrape portion of the command you've been running. Or if you just want to test the import of already scraped data you can replace it with --import . An import looks something like this: $ docker-compose run --rm scrape fl bills --fast ... (truncated) ... 23:03:34 ERROR openstates: cannot resolve pseudo id to Person: ~{\u201cname\u201d: \u201cGrant, M.\u201c} 23:03:36 ERROR openstates: cannot resolve pseudo id to Person: ~{\u201cname\u201d: \u201cRodrigues, R.\u201c} fl (import) bills: {} import: bill: 0 new 0 updated 2620 noop jurisdiction: 0 new 0 updated 1 noop vote_event: 21 new 12 updated 533 noop The errors about unresolved psuedo-ids can safely be ignored, as long as you see the final run report the data you scraped is available in your database. The number of objects of each type that were created & updated are available for spot checking, as well as the total number of items that were seen that already exactly matched what was in the database. These can be useful stats as you try to see if your local changes to a scraper have the impact you expect.","title":"Importing Data"},{"location":"contributing/scrapers/#running-spatula-scrapers","text":"Let's run a people scraper: $ poetry run spatula scrape scrapers_next.nc.people.SenList The command to run these scrapers is structured differently, as the parameters are set by giving the exact location of the function you want to run: directory.state.file.function . Note Function names do vary and scrapes for legislators are commonly split by chamber, so make sure to check you're passing the right function in your command. The actual data scraping should look something like: INFO:scrapers_next.nc.people.SenList:fetching https://www.ncleg.gov/Members/MemberTable/S INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/430 INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/431 INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/432 INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/433 INFO:scrapers_next.nc.people.LegDetail:fetching https://www.ncleg.gov/Members/Biography/S/434 To review the data you scraped, you can inspect the JSON files in the dated directory within _scrapes/ . Each time you run a scrape, a new numbered folder will be within the dated directory, so you can compare older data to new easily. Note If a scrape fails, it's likely an issue with the scraper. Feel free to get in touch with us or file an issue . Spatula is incredibly powerful with lots of flexibility and useful CLI commands that are worth checking out as well. At this point you're ready to run spatula scrapers and contribute fixes. Hop onto our GitHub ticket queue , pick an issue to solve, and then submit a Pull Request!","title":"Running Spatula Scrapers"},{"location":"contributing/state-specific/","text":"State-Specific Scraper Info \u00b6 California MySQL \u00b6 California is a unique state that takes a couple of extra steps to get working locally. California provides MySQL dumps of their data, and in order to use those we start up a local MySQL instance and read from that. To download the data for the current session: docker-compose run --rm ca-download (You can append --year YYYY to instead select data for a given year.) This will start a local MySQL image as well, that image will need to stay up for the next step, which is running a scrape like normal: docker-compose run --rm ca-scrape ca bills --fast State API Keys \u00b6 Unfortunately, some states find it necessary to require API Keys (or other credentials) to access their best data. Despite the difficulties this creates for contributors, in the interest of ensuring we have the best possible data we've made the decision that we will use this data where possible. Our policy: We will maintain (when possible) two copies of credentials, one for development and one for production. (Thus minimizing the chance that a mistake made w/ a development key will jeopardize our ability to scrape.) We encourage developers to get an API key of their own, but if necessary we can share our testing key in limited circumstances. Currently only a few states require API keys: New York \u00b6 http://legislation.nysenate.gov/static/docs/html/index.html Request Form: http://legislation.nysenate.gov/ Set in environment prior to running scrape: NEW_YORK_API_KEY Indiana \u00b6 http://docs.api.iga.in.gov/api.html API Key Request Process: Email Bob Amos ( bob.amos@iga.in.gov or bamos@iga.in.gov ), and include your name, address, phone, email address and company. Also indicate that you have read the terms of service at the link above. Set in environment prior to running scrape: INDIANA_API_KEY As a side note, Indiana also requires a user-agent string , so set that in your environment as well, prior to running scrape: USER_AGENT Virginia \u00b6 https://lis.virginia.gov/SiteInformation/ftp.html API Credentials Request Process: To acquire access, please contact the Virginia Legislative Information System help desk at (804) 786-9631 for a user id. Set in environment prior to running scrape: VIRGINIA_FTP_USER , VIRGINIA_FTP_PASSWORD District of Columbia \u00b6 https://lims.dccouncil.us/api/help/index.html API Key Request Form: https://lims.dccouncil.us/developerRegistration Set in environment prior to running scrape: DC_API_KEY","title":"State-Specific Scraper Info"},{"location":"contributing/state-specific/#state-specific-scraper-info","text":"","title":"State-Specific Scraper Info"},{"location":"contributing/state-specific/#california-mysql","text":"California is a unique state that takes a couple of extra steps to get working locally. California provides MySQL dumps of their data, and in order to use those we start up a local MySQL instance and read from that. To download the data for the current session: docker-compose run --rm ca-download (You can append --year YYYY to instead select data for a given year.) This will start a local MySQL image as well, that image will need to stay up for the next step, which is running a scrape like normal: docker-compose run --rm ca-scrape ca bills --fast","title":"California MySQL"},{"location":"contributing/state-specific/#state-api-keys","text":"Unfortunately, some states find it necessary to require API Keys (or other credentials) to access their best data. Despite the difficulties this creates for contributors, in the interest of ensuring we have the best possible data we've made the decision that we will use this data where possible. Our policy: We will maintain (when possible) two copies of credentials, one for development and one for production. (Thus minimizing the chance that a mistake made w/ a development key will jeopardize our ability to scrape.) We encourage developers to get an API key of their own, but if necessary we can share our testing key in limited circumstances. Currently only a few states require API keys:","title":"State API Keys"},{"location":"contributing/state-specific/#new-york","text":"http://legislation.nysenate.gov/static/docs/html/index.html Request Form: http://legislation.nysenate.gov/ Set in environment prior to running scrape: NEW_YORK_API_KEY","title":"New York"},{"location":"contributing/state-specific/#indiana","text":"http://docs.api.iga.in.gov/api.html API Key Request Process: Email Bob Amos ( bob.amos@iga.in.gov or bamos@iga.in.gov ), and include your name, address, phone, email address and company. Also indicate that you have read the terms of service at the link above. Set in environment prior to running scrape: INDIANA_API_KEY As a side note, Indiana also requires a user-agent string , so set that in your environment as well, prior to running scrape: USER_AGENT","title":"Indiana"},{"location":"contributing/state-specific/#virginia","text":"https://lis.virginia.gov/SiteInformation/ftp.html API Credentials Request Process: To acquire access, please contact the Virginia Legislative Information System help desk at (804) 786-9631 for a user id. Set in environment prior to running scrape: VIRGINIA_FTP_USER , VIRGINIA_FTP_PASSWORD","title":"Virginia"},{"location":"contributing/state-specific/#district-of-columbia","text":"https://lims.dccouncil.us/api/help/index.html API Key Request Form: https://lims.dccouncil.us/developerRegistration Set in environment prior to running scrape: DC_API_KEY","title":"District of Columbia"},{"location":"contributing/testing-scrapers/","text":"Testing Scrapers \u00b6 One of the first things people new to the project tend to notice is that there aren't a lot of tests in the scrapers. Over the years we've evolved a de facto policy of somewhat discouraging tests, which is definitely an unusual stance to take and warrants explanation. Intentionally Fragile Scrapers \u00b6 When it comes to scrapers, there are two major types of breakage: 1) the scraper collects bad information and inserts it into the database 2) the scraper encounters an error and quits without importing data Given a choice, the second is greatly preferable. Once bad data makes it into the database, it can be difficult to detect and remove. On the other hand, the second can be triggered to alert us immediately and someone can evaluate the proper fix. The best way to favor the second over first is to write \"intentionally fragile\" scrapers. That is, scrapers that raise an exception when they see unexpected input. While it is possible to try to write a resilient scraper that recovers, by nature these scrapers are more likely to produce the first kind of error, and so we encourage scraper writers to be conservative in what errors are suppressed. Here's an example of an overly permissive scraper: party_abbr = doc.xpath('//span[@class=\"partyabbr\"]) if party_abbr == 'D': party = 'Democratic' elif party_abbr == 'R': party = 'Republican' else: # haven't seen this yet, but let's just keep things moving party = party_abbr The following would be preferred: party_abbr = doc.xpath('//span[@class=\"partyabbr\"]) party = {'D': 'Democratic', 'R': 'Republican'}[party_abbr] This code would raise a KeyError the first time a new party is found. This forces someone to take a look, fix the scraper with an entry for the new party, and then the scraper will be able to run again with correct data. Testing Scrapers Is Hard \u00b6 On most software projects a failing test means that something is broken, and passing tests should mean that things are working just fine. In our experience however, the majority of the \"breaks\" that occur in scrapers are due to upstream site changes. In the past the fragile nature of scrapers has led to people writing a lot of bad tests, which is where our stance of somewhat discouraging tests has come from. An example of a bad test: def extract_name(doc): return doc.xpath('//h2[@class=\"legislatorName\"]').text_content().strip() def test_extract_name(): # probably a snapshot of the page at some point in time EXAMPLE_LEGISLATOR_HTML = '...' doc = lxml.html.fromstring(EXAMPLE_LEGISLATOR_HTML) assert extract_name(doc) == 'Erica Example' With a test like this: As soon as the HTML changes, the scraper will start failing, but the tests will still pass. The scraper will then be updated, breaking the test. The test HTML will be updated, fixing the test. But since the initial scraper breakage isn't predicted by a failing test, this type of test really doesn't serve us any purpose and just results in extra code to maintain every time the scraper needs a slight change. Other Strategies \u00b6 Of course this isn't to say that we just abandon the idea of testing, altogether. If you're more comfortable writing tests, say you're parsing a particularly nasty PDF and want to run it against some test data: a test might make sense there as a way to be confident in your own code, by all means, write a test. We also have some other strategies to help ensure data quality: Validate Scraper Output \u00b6 Scraper output is verified against JSON schemas that protect against common regressions (missing sources, invalid formatted districts, etc.) - most of these tests can be written effectively against scraper output across the board, and in doing so also applies universally across all 50 states. We also aim for our underlying libraries like openstates-core to be as well tested as possible. (To be 100% clear, our lax testing philosophy only applies to site-specific scraper code, not these support libraries.) Run Scrapers Regularly \u00b6 In a sense, the scrapers are tested every night by being run. This is why the intentionally fragile approach is so important; those failures are in essence the same as integration test failures. Of course, this doesn't tell us if the scraper is picking up bad data, etc., but combined with validation we can be fairly confident in our data. Test Utilities \u00b6 One area we can definitely improve upon is our use of (and then thorough testing of) common functions. Right now (largely because of the great variety of authors, etc.) many scrapers do similar things like conversion of party abbreviations and whitespace normalization in slightly different ways. We should be making a push to use common utility functions and thoroughly test those.","title":"Testing Scrapers"},{"location":"contributing/testing-scrapers/#testing-scrapers","text":"One of the first things people new to the project tend to notice is that there aren't a lot of tests in the scrapers. Over the years we've evolved a de facto policy of somewhat discouraging tests, which is definitely an unusual stance to take and warrants explanation.","title":"Testing Scrapers"},{"location":"contributing/testing-scrapers/#intentionally-fragile-scrapers","text":"When it comes to scrapers, there are two major types of breakage: 1) the scraper collects bad information and inserts it into the database 2) the scraper encounters an error and quits without importing data Given a choice, the second is greatly preferable. Once bad data makes it into the database, it can be difficult to detect and remove. On the other hand, the second can be triggered to alert us immediately and someone can evaluate the proper fix. The best way to favor the second over first is to write \"intentionally fragile\" scrapers. That is, scrapers that raise an exception when they see unexpected input. While it is possible to try to write a resilient scraper that recovers, by nature these scrapers are more likely to produce the first kind of error, and so we encourage scraper writers to be conservative in what errors are suppressed. Here's an example of an overly permissive scraper: party_abbr = doc.xpath('//span[@class=\"partyabbr\"]) if party_abbr == 'D': party = 'Democratic' elif party_abbr == 'R': party = 'Republican' else: # haven't seen this yet, but let's just keep things moving party = party_abbr The following would be preferred: party_abbr = doc.xpath('//span[@class=\"partyabbr\"]) party = {'D': 'Democratic', 'R': 'Republican'}[party_abbr] This code would raise a KeyError the first time a new party is found. This forces someone to take a look, fix the scraper with an entry for the new party, and then the scraper will be able to run again with correct data.","title":"Intentionally Fragile Scrapers"},{"location":"contributing/testing-scrapers/#testing-scrapers-is-hard","text":"On most software projects a failing test means that something is broken, and passing tests should mean that things are working just fine. In our experience however, the majority of the \"breaks\" that occur in scrapers are due to upstream site changes. In the past the fragile nature of scrapers has led to people writing a lot of bad tests, which is where our stance of somewhat discouraging tests has come from. An example of a bad test: def extract_name(doc): return doc.xpath('//h2[@class=\"legislatorName\"]').text_content().strip() def test_extract_name(): # probably a snapshot of the page at some point in time EXAMPLE_LEGISLATOR_HTML = '...' doc = lxml.html.fromstring(EXAMPLE_LEGISLATOR_HTML) assert extract_name(doc) == 'Erica Example' With a test like this: As soon as the HTML changes, the scraper will start failing, but the tests will still pass. The scraper will then be updated, breaking the test. The test HTML will be updated, fixing the test. But since the initial scraper breakage isn't predicted by a failing test, this type of test really doesn't serve us any purpose and just results in extra code to maintain every time the scraper needs a slight change.","title":"Testing Scrapers Is Hard"},{"location":"contributing/testing-scrapers/#other-strategies","text":"Of course this isn't to say that we just abandon the idea of testing, altogether. If you're more comfortable writing tests, say you're parsing a particularly nasty PDF and want to run it against some test data: a test might make sense there as a way to be confident in your own code, by all means, write a test. We also have some other strategies to help ensure data quality:","title":"Other Strategies"},{"location":"contributing/testing-scrapers/#validate-scraper-output","text":"Scraper output is verified against JSON schemas that protect against common regressions (missing sources, invalid formatted districts, etc.) - most of these tests can be written effectively against scraper output across the board, and in doing so also applies universally across all 50 states. We also aim for our underlying libraries like openstates-core to be as well tested as possible. (To be 100% clear, our lax testing philosophy only applies to site-specific scraper code, not these support libraries.)","title":"Validate Scraper Output"},{"location":"contributing/testing-scrapers/#run-scrapers-regularly","text":"In a sense, the scrapers are tested every night by being run. This is why the intentionally fragile approach is so important; those failures are in essence the same as integration test failures. Of course, this doesn't tell us if the scraper is picking up bad data, etc., but combined with validation we can be fairly confident in our data.","title":"Run Scrapers Regularly"},{"location":"contributing/testing-scrapers/#test-utilities","text":"One area we can definitely improve upon is our use of (and then thorough testing of) common functions. Right now (largely because of the great variety of authors, etc.) many scrapers do similar things like conversion of party abbreviations and whitespace normalization in slightly different ways. We should be making a push to use common utility functions and thoroughly test those.","title":"Test Utilities"},{"location":"contributing/text-extraction/","text":"Text Extraction \u00b6 The bill scrapers scrape the web and pull down metadata, including links to various versions of the bills. As a later step, we extract the actual text of the bill so that it can be indexed for search and other uses. Checking out \u00b6 Fork and clone the text-extraction repository: Visit https://github.com/openstates/text-extraction and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/text-extraction.git Cloning into 'text-extraction'... And remember to install pre-commit : $ pre-commit install pre-commit installed at .git/hooks/pre-commit Repository overview \u00b6 The text extraction code itself is written as a standalone Python script text_extract.py that uses configuration and utility functions from within extract/ . You'll also notice a directory called raw/ -- this contains a sampling of bills for each state that we can use to test text-extraction. Typically if you're making changes in the repository you'll be editing files within extract/ , we'll come back to that later. Running text_extract \u00b6 Just like in other repositories, we'll use docker-compose to run the code. In this case docker-compose is running text_extract.py , an all-in-one tool that has a few useful subcommands: Usage: text_extract.py [OPTIONS] COMMAND [ARGS]... Options: --help Show this message and exit. Commands: reindex-state rebuild the search index objects for a given state sample obtain a sample of bills to extract text from status print a status table showing the current condition of... test run sample on all states, used for CI update update the saved bill text in the database For the purposes of development, sample and update are the only two commands that you'll need to look at. Let's go ahead and run sample against NC: $ docker-compose run --rm text-extract sample nc raw/nc/2017-HR 924-Edition 1.pdf => text/nc/2017-HR 924-Edition 1.pdf.txt (1507 bytes) raw/nc/2017-HB 1034-Edition 1.pdf => text/nc/2017-HB 1034-Edition 1.pdf.txt (3096 bytes) raw/nc/2019-SB 421-Edition 1.pdf => text/nc/2019-SB 421-Edition 1.pdf.txt (961 bytes) raw/nc/2019-HB 430-Edition 1.pdf => text/nc/2019-HB 430-Edition 1.pdf.txt (4831 bytes) raw/nc/2017-SB 753-Edition 1.pdf => text/nc/2017-SB 753-Edition 1.pdf.txt (719 bytes) raw/nc/2019-HB 788-Edition 1.pdf => text/nc/2019-HB 788-Edition 1.pdf.txt (2674 bytes) raw/nc/2017-SB 373-Filed.pdf => text/nc/2017-SB 373-Filed.pdf.txt (18538 bytes) raw/nc/2019-SB 574-Filed.pdf => text/nc/2019-SB 574-Filed.pdf.txt (1712 bytes) raw/nc/2017-SJR 686-Resolution 2017-12.pdf => text/nc/2017-SJR 686-Resolution 2017-12.pdf.txt (15928 bytes) raw/nc/2017-HB 1007-Filed.pdf => text/nc/2017-HB 1007-Filed.pdf.txt (6248 bytes) nc: processed 10, 0 skipped, 0 missing, 0 empty The exact output and number of bills will vary across states, but should be pretty similar. This command just did a lot: Read in the file raw/nc.csv to get a list of bills to sample. Downloaded those files (assuming this was the first run) to raw/nc/ so future runs will be faster. Used the extraction function(s) defined in extract/__init__.py for NC to extract text from the given documents. Wrote that output to text/nc/ so you can compare. You'll also notice that it helpfully prints the number of bytes of text extracted, this is useful as a first check. Let's go ahead and look at the shortest one, text/nc/2017-SB 753-Edition 1.pdf.txt . (Your run may differ, pick whichever you prefer.) : $ cat \"text/nc/2017-SB 753 Edition 1.pdf.txt\" A BILL TO BE ENTITLED AN ACT PROVIDING THAT THE DEPOSIT OF CURRENCY AND COINS INTO A CASH VAULT THAT PHYSICALLY SECURES THE CASH AND ELECTRONICALLY RECORDS THE DEPOSIT DAILY IN AN OFFICIAL DEPOSITORY BANK QUALIFIES AS A DAILY DEPOSIT UNDER THE LOCAL GOVERNMENT BUDGET AND FISCAL CONTROL ACT FOR FRANKLIN AND WAKE COUNTIES AND THE MUNICIPALITIES IN THOSE COUNTIES. The General Assembly of North Carolina enacts: SECTION 1. Section 2 of S.L. 2011-89 reads as rewritten: \"SECTION 2. This act applies only to the City of Winston-Salem only.Winston-Salem, Franklin County and the municipalities in Franklin County, and Wake County and the municipalities in Wake County.\" SECTION 2. This act is effective when it becomes law. This looks complete, but to check, go ahead and open the equivalent source file, in this case raw/nc/2017-SB 753-Edition 1.pdf and confirm visually that all the text was extracted. Don't worry about formatting, or the preamble, as we'll often exclude that and just aim for the interesting bits of the text. Making changes \u00b6 Let's say that we discover that a state has started publishing their bills in a new format. Perhaps Alabama switches from PDF to HTML. It'd first be good to add some of these new bills to the sample csv, which you can do manually or by invoking sample with the --resample flag.: docker-compose run --rm text-extract sample --resample al Running would result in some warnings being printed and some zero byte files. To actually handle the HTML documents we'd open up extract/__init__.py and find the CONVERSION_FUNCTIONS dictionary, you'll see a line like: CONVERSION_FUNCTIONS = { \"al\": {\"application/pdf\": extract_line_numbered_pdf}, ... The way extraction works is by matching a document found in a scrape to an appropriate function, in this case PDFs will be sent through the extract_line_numbered_pdf function. If the new HTML was wrapped in a given element, perhaps with
    we could just update that line to look like: CONVERSION_FUNCTIONS = { \"al\": { \"application/pdf\": extract_line_numbered_pdf, \"text/html\": extractor_for_element_by_id(\"billtext\"), }, ... And we'd be good to go. Tips & Tricks \u00b6 Functions already exist for common configurations of PDF, HTML, Word Doc, and even OCR. Rarely will you need to write a custom function, always look at the options first. When dealing with PDFs, most are either handled by extract_line_numbered_pdf or extract_sometimes_numbered_pdf , the difference is that \"sometimes numbered PDF\" accounts for cases where 90% or so of bills are numbered, but a few (often resolutions) are not numbered. Formatting Guidelines \u00b6 How far do we go? Should we strip punctuation? Newlines? Whitespace? Section headings? Try not to be too aggressive with punctuation stripping, search indices/etc. can easily strip it later, but it can be handy if someone decides they want to search for things like \"\u00a7 143C-4-8.b\" Ideally leave newlines as-is since it makes looking at changes a lot nicer for humans and stripping newlines out for final products (search/text comparison/etc.) is trivial. Collapsing spaces/etc. is recommended, but not required. Removal of section headers/etc. is fine, but if the only reason you're writing a new function instead of using a common one is to do this, reconsider. When in doubt, ask , you may have encountered something we haven't considered yet and we can discuss the best practice and add it here. Should we include bill digests? There isn't a need to, but it doesn't hurt if separating the two is difficult.","title":"Text Extraction"},{"location":"contributing/text-extraction/#text-extraction","text":"The bill scrapers scrape the web and pull down metadata, including links to various versions of the bills. As a later step, we extract the actual text of the bill so that it can be indexed for search and other uses.","title":"Text Extraction"},{"location":"contributing/text-extraction/#checking-out","text":"Fork and clone the text-extraction repository: Visit https://github.com/openstates/text-extraction and click the 'Fork' button. Clone your fork using your tool of choice or the command line: $ git clone git@github.com:yourname/text-extraction.git Cloning into 'text-extraction'... And remember to install pre-commit : $ pre-commit install pre-commit installed at .git/hooks/pre-commit","title":"Checking out"},{"location":"contributing/text-extraction/#repository-overview","text":"The text extraction code itself is written as a standalone Python script text_extract.py that uses configuration and utility functions from within extract/ . You'll also notice a directory called raw/ -- this contains a sampling of bills for each state that we can use to test text-extraction. Typically if you're making changes in the repository you'll be editing files within extract/ , we'll come back to that later.","title":"Repository overview"},{"location":"contributing/text-extraction/#running-text_extract","text":"Just like in other repositories, we'll use docker-compose to run the code. In this case docker-compose is running text_extract.py , an all-in-one tool that has a few useful subcommands: Usage: text_extract.py [OPTIONS] COMMAND [ARGS]... Options: --help Show this message and exit. Commands: reindex-state rebuild the search index objects for a given state sample obtain a sample of bills to extract text from status print a status table showing the current condition of... test run sample on all states, used for CI update update the saved bill text in the database For the purposes of development, sample and update are the only two commands that you'll need to look at. Let's go ahead and run sample against NC: $ docker-compose run --rm text-extract sample nc raw/nc/2017-HR 924-Edition 1.pdf => text/nc/2017-HR 924-Edition 1.pdf.txt (1507 bytes) raw/nc/2017-HB 1034-Edition 1.pdf => text/nc/2017-HB 1034-Edition 1.pdf.txt (3096 bytes) raw/nc/2019-SB 421-Edition 1.pdf => text/nc/2019-SB 421-Edition 1.pdf.txt (961 bytes) raw/nc/2019-HB 430-Edition 1.pdf => text/nc/2019-HB 430-Edition 1.pdf.txt (4831 bytes) raw/nc/2017-SB 753-Edition 1.pdf => text/nc/2017-SB 753-Edition 1.pdf.txt (719 bytes) raw/nc/2019-HB 788-Edition 1.pdf => text/nc/2019-HB 788-Edition 1.pdf.txt (2674 bytes) raw/nc/2017-SB 373-Filed.pdf => text/nc/2017-SB 373-Filed.pdf.txt (18538 bytes) raw/nc/2019-SB 574-Filed.pdf => text/nc/2019-SB 574-Filed.pdf.txt (1712 bytes) raw/nc/2017-SJR 686-Resolution 2017-12.pdf => text/nc/2017-SJR 686-Resolution 2017-12.pdf.txt (15928 bytes) raw/nc/2017-HB 1007-Filed.pdf => text/nc/2017-HB 1007-Filed.pdf.txt (6248 bytes) nc: processed 10, 0 skipped, 0 missing, 0 empty The exact output and number of bills will vary across states, but should be pretty similar. This command just did a lot: Read in the file raw/nc.csv to get a list of bills to sample. Downloaded those files (assuming this was the first run) to raw/nc/ so future runs will be faster. Used the extraction function(s) defined in extract/__init__.py for NC to extract text from the given documents. Wrote that output to text/nc/ so you can compare. You'll also notice that it helpfully prints the number of bytes of text extracted, this is useful as a first check. Let's go ahead and look at the shortest one, text/nc/2017-SB 753-Edition 1.pdf.txt . (Your run may differ, pick whichever you prefer.) : $ cat \"text/nc/2017-SB 753 Edition 1.pdf.txt\" A BILL TO BE ENTITLED AN ACT PROVIDING THAT THE DEPOSIT OF CURRENCY AND COINS INTO A CASH VAULT THAT PHYSICALLY SECURES THE CASH AND ELECTRONICALLY RECORDS THE DEPOSIT DAILY IN AN OFFICIAL DEPOSITORY BANK QUALIFIES AS A DAILY DEPOSIT UNDER THE LOCAL GOVERNMENT BUDGET AND FISCAL CONTROL ACT FOR FRANKLIN AND WAKE COUNTIES AND THE MUNICIPALITIES IN THOSE COUNTIES. The General Assembly of North Carolina enacts: SECTION 1. Section 2 of S.L. 2011-89 reads as rewritten: \"SECTION 2. This act applies only to the City of Winston-Salem only.Winston-Salem, Franklin County and the municipalities in Franklin County, and Wake County and the municipalities in Wake County.\" SECTION 2. This act is effective when it becomes law. This looks complete, but to check, go ahead and open the equivalent source file, in this case raw/nc/2017-SB 753-Edition 1.pdf and confirm visually that all the text was extracted. Don't worry about formatting, or the preamble, as we'll often exclude that and just aim for the interesting bits of the text.","title":"Running text_extract"},{"location":"contributing/text-extraction/#making-changes","text":"Let's say that we discover that a state has started publishing their bills in a new format. Perhaps Alabama switches from PDF to HTML. It'd first be good to add some of these new bills to the sample csv, which you can do manually or by invoking sample with the --resample flag.: docker-compose run --rm text-extract sample --resample al Running would result in some warnings being printed and some zero byte files. To actually handle the HTML documents we'd open up extract/__init__.py and find the CONVERSION_FUNCTIONS dictionary, you'll see a line like: CONVERSION_FUNCTIONS = { \"al\": {\"application/pdf\": extract_line_numbered_pdf}, ... The way extraction works is by matching a document found in a scrape to an appropriate function, in this case PDFs will be sent through the extract_line_numbered_pdf function. If the new HTML was wrapped in a given element, perhaps with
    we could just update that line to look like: CONVERSION_FUNCTIONS = { \"al\": { \"application/pdf\": extract_line_numbered_pdf, \"text/html\": extractor_for_element_by_id(\"billtext\"), }, ... And we'd be good to go.","title":"Making changes"},{"location":"contributing/text-extraction/#tips-tricks","text":"Functions already exist for common configurations of PDF, HTML, Word Doc, and even OCR. Rarely will you need to write a custom function, always look at the options first. When dealing with PDFs, most are either handled by extract_line_numbered_pdf or extract_sometimes_numbered_pdf , the difference is that \"sometimes numbered PDF\" accounts for cases where 90% or so of bills are numbered, but a few (often resolutions) are not numbered.","title":"Tips & Tricks"},{"location":"contributing/text-extraction/#formatting-guidelines","text":"How far do we go? Should we strip punctuation? Newlines? Whitespace? Section headings? Try not to be too aggressive with punctuation stripping, search indices/etc. can easily strip it later, but it can be handy if someone decides they want to search for things like \"\u00a7 143C-4-8.b\" Ideally leave newlines as-is since it makes looking at changes a lot nicer for humans and stripping newlines out for final products (search/text comparison/etc.) is trivial. Collapsing spaces/etc. is recommended, but not required. Removal of section headers/etc. is fine, but if the only reason you're writing a new function instead of using a common one is to do this, reconsider. When in doubt, ask , you may have encountered something we haven't considered yet and we can discuss the best practice and add it here. Should we include bill digests? There isn't a need to, but it doesn't hurt if separating the two is difficult.","title":"Formatting Guidelines"},{"location":"contributing/writing-a-committee-scraper/","text":"Writing a Committee Scraper \u00b6 For Those Completely New to Writing Scrapers \u00b6 What do we mean when we use the term \u201cweb scraping\u201d? What is a scraper? How do you write a web scraper? An overview from the source above: When you run the code for web scraping, a request is sent to the URL that you have mentioned. As a response to the request, the server sends the data and allows you to read the HTML or XML page. The code then parses the HTML or XML page, finds the data and extracts it. To extract data using web scraping with python, you need to follow these basic steps: Find the URL that you want to scrape Find the data you want to extract Write the code Run the code and extract the data Store the data in the required format For those new to Open States: \u00b6 What is Open States? What is an open source project? How do you contribute to Open States? What projects are we looking for contributions on? Committee Scrapers Other issues Writing Committee Scrapers \u00b6 Background: What are legislative committees? \u00b6 This information isn\u2019t necessary for writing scrapers , but if you\u2019re interested in more context or want to know why committee data is important, this is where you\u2019ll find it! Legislative committees are groups of legislators selected by House and Senate leadership to consider legislation concerning a certain subject or set of subjects (ex: Appropriations, Environment & Natural Resources, Criminal Justice & Public Safety). While most state legislatures have separate House and Senate committees, a few states (CT, ME, MA, and NE) have just one set of committees. Most legislators serve on between 2 and 5 committees each session. Standing committees are permanent committees that are created at the beginning of the legislative session in the House, Senate, or Joint Rules. Other committee types (select committees, interim committees, study committees) are time-limited and often created by legislation or resolutions. Because the vast majority of legislation goes through the committee process, and because the committee membership has such an outsized role on whether that legislation lives or dies, it's incredibly important that people have accurate and up to date information about committees, their membership, and their work. Writing a Committee Scraper, Step by Step \u00b6 How do I set up my environment? Make sure you've walked through our installation prerequisities Where should my code live? All committee scrapers live in the scrapers_next directory of the openstates_scrapers repository When you're ready to commit your code, this is where your new committee scraper will live too! What are the steps for working on my committee scraper locally? Fork the openstates-scrapers repository ( what is a fork ?) Find the state you want to write a committee scraper for under the scrapers_next directory Create a committees.py file inside of that state's directory Write your committee scraper code! What information does a committee scraper need to grab? Name of the committee Chamber (upper/lower/etc) Classification (committee, subcommittee) Parent (if it is a subcommittee) Sources (home page for the list of committees, specific page for that committee, etc) Members (name and role on the committee) Example of complete data: { \"name\" : \"Revenue\" , \"chamber\" : \"upper\" , \"classification\" : \"committee\" , \"parent\" : null , \"sources\" : [ { \"url\" : \"https://www.ilga.gov/senate/committees/default.asp\" , \"note\" : \"homepage\" }, { \"url\" : \"https://www.ilga.gov/senate/committees/members.asp?CommitteeID=2688\" , \"note\" : \"\" }], \"links\" : [], \"other_names\" : [], \"members\" : [ { \"name\" : \"Mattie Hunter\" , \"role\" : \"Chair\" , \"person_id\" : null }, { \"name\" : \"Steve Stadelman\" , \"role\" : \"Vice-Chair\" , \"person_id\" : null }, { \"name\" : \"Robert F. Martwick\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Cristina H. Pacione-Zayas\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Robert Peters\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Elgie R. Sims, Jr.\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Donald P. DeWitte\" , \"role\" : \"Minority\" , \"person_id\" : null }, { \"name\" : \"Dale Fowler\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Jil Tracy\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Sally J. Turner\" , \"role\" : \"Member\" , \"person_id\" : null }], \"extras\" : {} } What does a complete committee scraper look like? An example committee scraper The way committee scrapers look will vary depending on how the website for the state is set up. Most of them will have a list of committees for the House, a list of committees for the Senate, and individual pages for each committee that includes details about membership and/or subcommittees I know what information I want to grab, but how do I grab it? Grab the information you need using selectors (<- a guide to selectors) Getting the selector/XPath on chrome: Right click the item you want to grab Click \u201cinspect\u201d A panel will pop up and the element that you\u2019re inspecting should be highlighted. Click on the three dots on the left side of the highlight Hover over \u201ccopy\u201d and click on either selector (for a CSS selector) or XPath to copy it There aren\u2019t any rules about whether XPath or CSS selectors are better. Use whichever helps you grab the information you need! General Scraper Writing Tips \u00b6 What are helpful tools for writing scrapers? The most helpful tool that you can use to write a committee scraper is the python package spatula A walkthrough on how to write a scraper using spatula How do I run a scraper? Follow the instructions here . If you haven\u2019t read the Getting Started , make sure you do that first! The command for running a spatula scraper is poetry run spatula scrape {directory your python file is in}.{state your scraper is for}.committees.{function (optional)} Example: To run the entire committee scraper for Missouri, the command would be poetry run spatula scrape scrapers_next.mo.committees Help! My code doesn't work! Stuck? Think it through and ask us for help! What\u2019s not working? What do you think should happen? What\u2019s actually happening? Exception handling can be great for weird cases, like a specific committee\u2019s information being unavailable Spatula lets you easily skip over weird cases Debug and quickly test if your CSS or Xpath selector is accurate with spatula test for a single page Before running spatula scrape, you can test the logic of what you\u2019ve already written (and essentially get a quick \u201cpreview\u201d) in the terminal. This is especially helpful for making sure you\u2019re using the correct CSS or Xpath selector for an element. You can set an example_source as a default URL to keep running spatula scrape on Example use case: After scraping a list of committees on one page, I now need specific information about each committee. Each committee has its own webpage (that we navigate to from that list of committees), and the format for each webpage is fairly similar (this is most likely the case!). I can set the example_source as a single committee\u2019s more-detailed webpage, and as I write that part of the scraper, I can keep running spatula test to see in my terminal that I\u2019m grabbing the correct information from that specific webpage. This should save time and prevent some headaches!","title":"Writing a Committee Scraper"},{"location":"contributing/writing-a-committee-scraper/#writing-a-committee-scraper","text":"","title":"Writing a Committee Scraper"},{"location":"contributing/writing-a-committee-scraper/#for-those-completely-new-to-writing-scrapers","text":"What do we mean when we use the term \u201cweb scraping\u201d? What is a scraper? How do you write a web scraper? An overview from the source above: When you run the code for web scraping, a request is sent to the URL that you have mentioned. As a response to the request, the server sends the data and allows you to read the HTML or XML page. The code then parses the HTML or XML page, finds the data and extracts it. To extract data using web scraping with python, you need to follow these basic steps: Find the URL that you want to scrape Find the data you want to extract Write the code Run the code and extract the data Store the data in the required format","title":"For Those Completely New to Writing Scrapers"},{"location":"contributing/writing-a-committee-scraper/#for-those-new-to-open-states","text":"What is Open States? What is an open source project? How do you contribute to Open States? What projects are we looking for contributions on? Committee Scrapers Other issues","title":"For those new to Open States:"},{"location":"contributing/writing-a-committee-scraper/#writing-committee-scrapers","text":"","title":"Writing Committee Scrapers"},{"location":"contributing/writing-a-committee-scraper/#background-what-are-legislative-committees","text":"This information isn\u2019t necessary for writing scrapers , but if you\u2019re interested in more context or want to know why committee data is important, this is where you\u2019ll find it! Legislative committees are groups of legislators selected by House and Senate leadership to consider legislation concerning a certain subject or set of subjects (ex: Appropriations, Environment & Natural Resources, Criminal Justice & Public Safety). While most state legislatures have separate House and Senate committees, a few states (CT, ME, MA, and NE) have just one set of committees. Most legislators serve on between 2 and 5 committees each session. Standing committees are permanent committees that are created at the beginning of the legislative session in the House, Senate, or Joint Rules. Other committee types (select committees, interim committees, study committees) are time-limited and often created by legislation or resolutions. Because the vast majority of legislation goes through the committee process, and because the committee membership has such an outsized role on whether that legislation lives or dies, it's incredibly important that people have accurate and up to date information about committees, their membership, and their work.","title":"Background: What are legislative committees?"},{"location":"contributing/writing-a-committee-scraper/#writing-a-committee-scraper-step-by-step","text":"How do I set up my environment? Make sure you've walked through our installation prerequisities Where should my code live? All committee scrapers live in the scrapers_next directory of the openstates_scrapers repository When you're ready to commit your code, this is where your new committee scraper will live too! What are the steps for working on my committee scraper locally? Fork the openstates-scrapers repository ( what is a fork ?) Find the state you want to write a committee scraper for under the scrapers_next directory Create a committees.py file inside of that state's directory Write your committee scraper code! What information does a committee scraper need to grab? Name of the committee Chamber (upper/lower/etc) Classification (committee, subcommittee) Parent (if it is a subcommittee) Sources (home page for the list of committees, specific page for that committee, etc) Members (name and role on the committee) Example of complete data: { \"name\" : \"Revenue\" , \"chamber\" : \"upper\" , \"classification\" : \"committee\" , \"parent\" : null , \"sources\" : [ { \"url\" : \"https://www.ilga.gov/senate/committees/default.asp\" , \"note\" : \"homepage\" }, { \"url\" : \"https://www.ilga.gov/senate/committees/members.asp?CommitteeID=2688\" , \"note\" : \"\" }], \"links\" : [], \"other_names\" : [], \"members\" : [ { \"name\" : \"Mattie Hunter\" , \"role\" : \"Chair\" , \"person_id\" : null }, { \"name\" : \"Steve Stadelman\" , \"role\" : \"Vice-Chair\" , \"person_id\" : null }, { \"name\" : \"Robert F. Martwick\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Cristina H. Pacione-Zayas\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Robert Peters\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Elgie R. Sims, Jr.\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Donald P. DeWitte\" , \"role\" : \"Minority\" , \"person_id\" : null }, { \"name\" : \"Dale Fowler\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Jil Tracy\" , \"role\" : \"Member\" , \"person_id\" : null }, { \"name\" : \"Sally J. Turner\" , \"role\" : \"Member\" , \"person_id\" : null }], \"extras\" : {} } What does a complete committee scraper look like? An example committee scraper The way committee scrapers look will vary depending on how the website for the state is set up. Most of them will have a list of committees for the House, a list of committees for the Senate, and individual pages for each committee that includes details about membership and/or subcommittees I know what information I want to grab, but how do I grab it? Grab the information you need using selectors (<- a guide to selectors) Getting the selector/XPath on chrome: Right click the item you want to grab Click \u201cinspect\u201d A panel will pop up and the element that you\u2019re inspecting should be highlighted. Click on the three dots on the left side of the highlight Hover over \u201ccopy\u201d and click on either selector (for a CSS selector) or XPath to copy it There aren\u2019t any rules about whether XPath or CSS selectors are better. Use whichever helps you grab the information you need!","title":"Writing a Committee Scraper, Step by Step"},{"location":"contributing/writing-a-committee-scraper/#general-scraper-writing-tips","text":"What are helpful tools for writing scrapers? The most helpful tool that you can use to write a committee scraper is the python package spatula A walkthrough on how to write a scraper using spatula How do I run a scraper? Follow the instructions here . If you haven\u2019t read the Getting Started , make sure you do that first! The command for running a spatula scraper is poetry run spatula scrape {directory your python file is in}.{state your scraper is for}.committees.{function (optional)} Example: To run the entire committee scraper for Missouri, the command would be poetry run spatula scrape scrapers_next.mo.committees Help! My code doesn't work! Stuck? Think it through and ask us for help! What\u2019s not working? What do you think should happen? What\u2019s actually happening? Exception handling can be great for weird cases, like a specific committee\u2019s information being unavailable Spatula lets you easily skip over weird cases Debug and quickly test if your CSS or Xpath selector is accurate with spatula test for a single page Before running spatula scrape, you can test the logic of what you\u2019ve already written (and essentially get a quick \u201cpreview\u201d) in the terminal. This is especially helpful for making sure you\u2019re using the correct CSS or Xpath selector for an element. You can set an example_source as a default URL to keep running spatula scrape on Example use case: After scraping a list of committees on one page, I now need specific information about each committee. Each committee has its own webpage (that we navigate to from that list of committees), and the format for each webpage is fairly similar (this is most likely the case!). I can set the example_source as a single committee\u2019s more-detailed webpage, and as I write that part of the scraper, I can keep running spatula test to see in my terminal that I\u2019m grabbing the correct information from that specific webpage. This should save time and prevent some headaches!","title":"General Scraper Writing Tips"},{"location":"data/","text":"Understanding the Data \u00b6 Open States data adheres to a schema that has evolved over our 11+ years of working with legislative data. Our goal is to provide as much uniformity across states as possible while still allowing for the wide diversity of legislative processes between the states. These docs both catalog the schema and attempt to explain some of those choices, particularly where they might be surprising. Main Concepts \u00b6 The main concepts are defined below. You'll notice these concepts mostly correspond to the v2 GraphQL root nodes. Concept Definition Jurisdiction Essentially just another word for \"State\" in our context. (Includes DC and Puerto Rico.) Session A period of time in a legislature where the same members serve together, typically punctuated by elections. All bills in a session will be uniquely numbered. (e.g. HB 1 in the 2017 session is typically not the same bill as in the 2019 session) Bill Represents all types of legislation whether it is a bill, resolution, etc. Vote A vote among members of the legislature, typically an entire chamber but can also be a committee vote. Person Any person that is associated with the legislature. Organization A generic term used to represent a few different concepts: legislatures, chambers, committees, and political parties. Post A particular role within an organization, typically used to represent a seat in the legislature. (e.g. the District 4 post in the North Carolina Senate Organization) Membership Ties a Person to a Post for a duration of time.","title":"Understanding the Data"},{"location":"data/#understanding-the-data","text":"Open States data adheres to a schema that has evolved over our 11+ years of working with legislative data. Our goal is to provide as much uniformity across states as possible while still allowing for the wide diversity of legislative processes between the states. These docs both catalog the schema and attempt to explain some of those choices, particularly where they might be surprising.","title":"Understanding the Data"},{"location":"data/#main-concepts","text":"The main concepts are defined below. You'll notice these concepts mostly correspond to the v2 GraphQL root nodes. Concept Definition Jurisdiction Essentially just another word for \"State\" in our context. (Includes DC and Puerto Rico.) Session A period of time in a legislature where the same members serve together, typically punctuated by elections. All bills in a session will be uniquely numbered. (e.g. HB 1 in the 2017 session is typically not the same bill as in the 2019 session) Bill Represents all types of legislation whether it is a bill, resolution, etc. Vote A vote among members of the legislature, typically an entire chamber but can also be a committee vote. Person Any person that is associated with the legislature. Organization A generic term used to represent a few different concepts: legislatures, chambers, committees, and political parties. Post A particular role within an organization, typically used to represent a seat in the legislature. (e.g. the District 4 post in the North Carolina Senate Organization) Membership Ties a Person to a Post for a duration of time.","title":"Main Concepts"},{"location":"data/categorization/","text":"Categorization \u00b6 One of the ways that we add value to the data we provide is by attempting to classify bills, actions, and votes across states. This allows us to let states use their own names for these things, but for us to try to provide some mapping to a common simplified view of the legislative process. Bill Types \u00b6 State legislatures deal with more than bills. Despite the name of the bill objects in our data we take in all types of legislation that a state might produce. Generally looking at the bill_id will help you determine the type of legislation, but to make things easier across states we provide a type field on bills. This field is a list with one (or more) of the following values: Common Values: bill resolution joint resolution concurrent resolution constitutional amendment Some states also make use of additional types such as 'contract', 'nomination', 'memorial' and more. Action Types \u00b6 Although most states follow very similar parlimentary procedure the names that their bill status systems use for various actions almost never match up. To make analysis and the building of certain types of tools easier we attempt to classify common actions. In using our data you'll find these values in the classification field of actions. filing - bill is filed (where this is a separate action from introduction) introduction - introduced, typically the first action reading-1 - first reading (often same as introduction) reading-2 - second reading reading-3 - third reading (often same as passage) passage - bill is passed by the chamber failure - bill fails to proceed from the chamber withdrawal - bill is withdrawn substitution - a substitution is made to the bill text deferral - consideration of the bill was deferred receipt - a bill was received by another chamber referral - a bill was sent somewhere for consideration referral-committee - a bill was sent to a committee for consideration became-law - the bill became law (through signature or inaction) amendment-introduction - an amendment is introduced amendment-passage - an amendment passes amendment-withdrawal - an amendment is withdrawn amendment-failure - an amendment fails to pass amendment-amendment - an amendment is amended amendment-deferral - consideration of an amendment is deferred committee-passage - the bill passes the current committee (unknown outcome, typically favorble) committee-passage-favorable - the bill passes the current committee favorably committee-passage-unfavorable - the bill passes the current committee with an unfavorable report committee-failure - the bill fails to advance out of committee executive-receipt - the bill is sent to the governor executive-signature - the governor signs the bill executive-veto - the governor vetos the bill executive-veto-line-item - the governor uses a line-item veto to strike part of a bill veto-override-passage - a veto override vote occurred and succeeded veto-override-failure - a veto override vote occurred and failed Vote Types \u00b6 Similarly to actions, we make an effort to categorize the motion being voted upon. You'll find these values in the categorization field on VoteEvents. Possible values: bill-passage - This is a vote to pass (either out of committee or a chamber) amendment-passage - Vote on amending a bill veto-override - Vote to override an executive veto","title":"Categorization"},{"location":"data/categorization/#categorization","text":"One of the ways that we add value to the data we provide is by attempting to classify bills, actions, and votes across states. This allows us to let states use their own names for these things, but for us to try to provide some mapping to a common simplified view of the legislative process.","title":"Categorization"},{"location":"data/categorization/#bill-types","text":"State legislatures deal with more than bills. Despite the name of the bill objects in our data we take in all types of legislation that a state might produce. Generally looking at the bill_id will help you determine the type of legislation, but to make things easier across states we provide a type field on bills. This field is a list with one (or more) of the following values: Common Values: bill resolution joint resolution concurrent resolution constitutional amendment Some states also make use of additional types such as 'contract', 'nomination', 'memorial' and more.","title":"Bill Types"},{"location":"data/categorization/#action-types","text":"Although most states follow very similar parlimentary procedure the names that their bill status systems use for various actions almost never match up. To make analysis and the building of certain types of tools easier we attempt to classify common actions. In using our data you'll find these values in the classification field of actions. filing - bill is filed (where this is a separate action from introduction) introduction - introduced, typically the first action reading-1 - first reading (often same as introduction) reading-2 - second reading reading-3 - third reading (often same as passage) passage - bill is passed by the chamber failure - bill fails to proceed from the chamber withdrawal - bill is withdrawn substitution - a substitution is made to the bill text deferral - consideration of the bill was deferred receipt - a bill was received by another chamber referral - a bill was sent somewhere for consideration referral-committee - a bill was sent to a committee for consideration became-law - the bill became law (through signature or inaction) amendment-introduction - an amendment is introduced amendment-passage - an amendment passes amendment-withdrawal - an amendment is withdrawn amendment-failure - an amendment fails to pass amendment-amendment - an amendment is amended amendment-deferral - consideration of an amendment is deferred committee-passage - the bill passes the current committee (unknown outcome, typically favorble) committee-passage-favorable - the bill passes the current committee favorably committee-passage-unfavorable - the bill passes the current committee with an unfavorable report committee-failure - the bill fails to advance out of committee executive-receipt - the bill is sent to the governor executive-signature - the governor signs the bill executive-veto - the governor vetos the bill executive-veto-line-item - the governor uses a line-item veto to strike part of a bill veto-override-passage - a veto override vote occurred and succeeded veto-override-failure - a veto override vote occurred and failed","title":"Action Types"},{"location":"data/categorization/#vote-types","text":"Similarly to actions, we make an effort to categorize the motion being voted upon. You'll find these values in the categorization field on VoteEvents. Possible values: bill-passage - This is a vote to pass (either out of committee or a chamber) amendment-passage - Vote on amending a bill veto-override - Vote to override an executive veto","title":"Vote Types"},{"location":"data/session-naming/","text":"Session Naming \u00b6 States name their sessions drastically differently, and sometimes inconsistently even within their own site. (49th vs 2008 Regular Session). As our goal is to help smooth these inconsistencies we put forward this guide to naming sessions within state metadata. (See https://github.com/sunlightlabs/openstates/issues/81 for discussion on the topic) Default Session Names \u00b6 The sessions list within terms is dangerous to change as all bill data is keyed off it. As a rule these should be short and generally useful for the scraper to make the appropriate decisions on what data to scrape. If a state calls its 1st special session in 2010 '2010E1' this is a perfectly acceptable name for the session in the metadata. Similarly 49th-regular, 2009-Special-B, etc. are fine names. Generally names with spaces should be avoided simply for ease of construction of URLs, etc. In states where spaces are already in use it is fine to continue to use them. The one caveat is that if a state uses a unique ID that has no bearing on the session itself such as '7323' for the 2011 session, this should not be used. Instead add some mapping that maps a session name that is descriptive to their internal ids. Session Display Names \u00b6 Because the most convenient name to refer to a session is often far from what a user might expect to see upon opening a mobile application, the session_details dict supports a display_name key. Suitable display names are descriptive but also short and obey a given style. General Rules \u00b6 All sessions should be in title case. Fewer than 20 characters is highly preferable. Months should be abbreviated to 3 letters (Jan., Feb., Jun., Dec.) Ordinals \u00b6 If no special sessions are present: [Ordinal] Legislature If special sessions are present: [Ordinal] Regular Session [Ordinal], [Ordinal] Special Session Examples: 82nd Legislature 82nd Regular Session 82nd, 3rd Special Session Years \u00b6 [Year/Year-Range] Regular Session [Year/Year-Range], [Ordinal] Special Session [Mon. Year] Special Session Examples: 2010 Regular Session 2011-2012, 4th Special Session Dec. 2011 Special Session","title":"Session Naming"},{"location":"data/session-naming/#session-naming","text":"States name their sessions drastically differently, and sometimes inconsistently even within their own site. (49th vs 2008 Regular Session). As our goal is to help smooth these inconsistencies we put forward this guide to naming sessions within state metadata. (See https://github.com/sunlightlabs/openstates/issues/81 for discussion on the topic)","title":"Session Naming"},{"location":"data/session-naming/#default-session-names","text":"The sessions list within terms is dangerous to change as all bill data is keyed off it. As a rule these should be short and generally useful for the scraper to make the appropriate decisions on what data to scrape. If a state calls its 1st special session in 2010 '2010E1' this is a perfectly acceptable name for the session in the metadata. Similarly 49th-regular, 2009-Special-B, etc. are fine names. Generally names with spaces should be avoided simply for ease of construction of URLs, etc. In states where spaces are already in use it is fine to continue to use them. The one caveat is that if a state uses a unique ID that has no bearing on the session itself such as '7323' for the 2011 session, this should not be used. Instead add some mapping that maps a session name that is descriptive to their internal ids.","title":"Default Session Names"},{"location":"data/session-naming/#session-display-names","text":"Because the most convenient name to refer to a session is often far from what a user might expect to see upon opening a mobile application, the session_details dict supports a display_name key. Suitable display names are descriptive but also short and obey a given style.","title":"Session Display Names"},{"location":"data/session-naming/#general-rules","text":"All sessions should be in title case. Fewer than 20 characters is highly preferable. Months should be abbreviated to 3 letters (Jan., Feb., Jun., Dec.)","title":"General Rules"},{"location":"data/session-naming/#ordinals","text":"If no special sessions are present: [Ordinal] Legislature If special sessions are present: [Ordinal] Regular Session [Ordinal], [Ordinal] Special Session Examples: 82nd Legislature 82nd Regular Session 82nd, 3rd Special Session","title":"Ordinals"},{"location":"data/session-naming/#years","text":"[Year/Year-Range] Regular Session [Year/Year-Range], [Ordinal] Special Session [Mon. Year] Special Session Examples: 2010 Regular Session 2011-2012, 4th Special Session Dec. 2011 Special Session","title":"Years"},{"location":"openstates.org/scheduled-tasks/","text":"Openstates.org scheduled tasks \u00b6 Openstates.org is a django application that has some simple scheduled tasks that execute using Django admin commands . Jobs and Locations \u00b6 Containerized versions of the actual scheduled jobs are managed in Github . These are not the jobs executed in production. We deploy jobs to the Openstates.org host using ansible . openstates.org runs in AWS. Access credentials/location/etc. can be found in the AWS console. Currently Scheduled Jobs \u00b6 Subscription Processing \u00b6 Defined here Tool that processes search subscriptions for users. Currently (2022-08-02) scheduled to run once a day (12:30 UTC) Aggregate API Usage \u00b6 Defined here Tool that generates some internal stats for user interactions with Openstates. Currently (2022-08-02) scheduled to run every 2 hours (39 */2) System Maintenance Jobs \u00b6 Let's Encrypt certificate collection/rotation Nginx maintenance (tied to certificate rotation)","title":"Openstates.org scheduled tasks"},{"location":"openstates.org/scheduled-tasks/#openstatesorg-scheduled-tasks","text":"Openstates.org is a django application that has some simple scheduled tasks that execute using Django admin commands .","title":"Openstates.org scheduled tasks"},{"location":"openstates.org/scheduled-tasks/#jobs-and-locations","text":"Containerized versions of the actual scheduled jobs are managed in Github . These are not the jobs executed in production. We deploy jobs to the Openstates.org host using ansible . openstates.org runs in AWS. Access credentials/location/etc. can be found in the AWS console.","title":"Jobs and Locations"},{"location":"openstates.org/scheduled-tasks/#currently-scheduled-jobs","text":"","title":"Currently Scheduled Jobs"},{"location":"openstates.org/scheduled-tasks/#subscription-processing","text":"Defined here Tool that processes search subscriptions for users. Currently (2022-08-02) scheduled to run once a day (12:30 UTC)","title":"Subscription Processing"},{"location":"openstates.org/scheduled-tasks/#aggregate-api-usage","text":"Defined here Tool that generates some internal stats for user interactions with Openstates. Currently (2022-08-02) scheduled to run every 2 hours (39 */2)","title":"Aggregate API Usage"},{"location":"openstates.org/scheduled-tasks/#system-maintenance-jobs","text":"Let's Encrypt certificate collection/rotation Nginx maintenance (tied to certificate rotation)","title":"System Maintenance Jobs"}]} \ No newline at end of file diff --git a/sitemap.xml b/sitemap.xml index a9862c4..a5dc8aa 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -2,172 +2,122 @@ https://docs.openstates.org/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/code-of-conduct/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/api-v2/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/api-v2/changelog/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/api-v2/examples/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/api-v2/other/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/api-v2/root-nodes/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/api-v2/types/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/api-v3/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/api-v3/changelog/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/contributing/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/contributing/documentation/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/contributing/local-database/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/contributing/openstates-org/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/contributing/people/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/contributing/scrapers/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/contributing/state-specific/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/contributing/testing-scrapers/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/contributing/text-extraction/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/contributing/writing-a-committee-scraper/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/data/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/data/categorization/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/data/session-naming/ - 2023-07-13 - daily - - - https://docs.openstates.org/enhancement-proposals/ - 2023-07-13 - daily - - - https://docs.openstates.org/enhancement-proposals/001-purpose-and-process/ - 2023-07-13 - daily - - - https://docs.openstates.org/enhancement-proposals/002-legal-citations/ - 2023-07-13 - daily - - - https://docs.openstates.org/enhancement-proposals/003-manual-people-data-tools/ - 2023-07-13 - daily - - - https://docs.openstates.org/enhancement-proposals/004-committee-data/ - 2023-07-13 - daily - - - https://docs.openstates.org/enhancement-proposals/005-dedupe-key/ - 2023-07-13 - daily - - - https://docs.openstates.org/enhancement-proposals/006-new-people-offices/ - 2023-07-13 - daily - - - https://docs.openstates.org/enhancement-proposals/007-events/ - 2023-07-13 - daily - - - https://docs.openstates.org/enhancement-proposals/008-active-sessions/ - 2023-07-13 - daily - - - https://docs.openstates.org/enhancement-proposals/template/ - 2023-07-13 + 2024-03-28 daily https://docs.openstates.org/openstates.org/scheduled-tasks/ - 2023-07-13 + 2024-03-28 daily \ No newline at end of file diff --git a/sitemap.xml.gz b/sitemap.xml.gz index 0d7f6e48ed220377fe5d7c054cbbd0b9a2bfa271..dd50f2222977012e9826676da3eb11402045d85f 100644 GIT binary patch literal 409 zcmV;K0cQRmiwFo+tp#QR|8r?{Wo=<_E_iKh0M(blZi6rkhVOd{#68f_9lD9q+n!*1 zfHb%eEhJfNO1r0@fo?;)?J|W(D8%_=eXeJh3kY`-%GtVC6epzgTAYih*=CQ%HBnQ-MRwi*Mh)qxg%CAiaE^rKn-g~+ z>FT2<+(yk7`Hkj;O+g=f;;Oa+LKgHLSl;ELyaHYpOCxGY@+c8j0Q|EjB5pPoz#bfz zq7{Im3es~oxdznHF3^M0C#_Zh%*JsRD~d6}g#>>=s+HujA?PW{lDqt_(?kGY0gsb@ zOuG@2DPA@%=#Pkt*#wG!;~Q95*+ttW{92CB?YcKm3zGQu5BXr45UBbDkbvrBHxK{- DnqI!R literal 529 zcmV+s0`C1EiwFp58?a;o|8r?{Wo=<_E_iKh0M(emZrd;nhVOlfz<0(@vaRjlxVJt* z_W-KII6`Gflju0v(~q*-%^0x5E`wyiutibCXF?`_yM4M2W^=(1wX?@+w{5BoTIrOw z{jvJ?@(d5vQ+L;n!9=7DJKK1y&P4ueT~E`rjhb)}K5kv;>nP7Wi1oX%ZnlqAcemjq z1z7Ipqcu);DN5up%1K`C(MA%4OQg0=9_s}n(%@9LZ}tah?qPRemv`IBO`e7A@f7fT zr-U}Qwq98llKL0Qzd~_Z5Zno*vuc!-C#3KiF8i$6mWamnSySRw4BlYO8q(1jL)L`B zIReXXPTq;6o6nl?kTqN8Z!{Nd3i@>TG-6EOz+lCYtx7=4u_PvK06sYHqA;-})TY^i9|MmN zgjH!qa^A~&ggqGSS@1}`D73=x4=@NjGXEkfX59wlTxmP$)2c$}TLk|`{}~jj(IdR! ztt|2{U~&4`JnqEW2@Bgf_kb6+6-NC4Nkm;(jx3$3D15@f8}`YQ`Hx}xjjcw53P{Aa T|B&bT5|O%}j~4&{@>2yu