From dd3a98b1798f8d53a0d58347ccda38cf0a873906 Mon Sep 17 00:00:00 2001 From: Peter Wolanin <107691+pwolanin@users.noreply.github.com> Date: Tue, 10 Mar 2026 20:45:15 -0400 Subject: [PATCH 1/8] Start on py updates using claude --- data-scripts/Pipfile | 16 +++ data-scripts/Pipfile.lock | 231 ++++++++++++++++++++++++++++++++++ data-scripts/README.md | 20 +++ data-scripts/requirements.txt | 5 - 4 files changed, 267 insertions(+), 5 deletions(-) create mode 100644 data-scripts/Pipfile create mode 100644 data-scripts/Pipfile.lock delete mode 100644 data-scripts/requirements.txt diff --git a/data-scripts/Pipfile b/data-scripts/Pipfile new file mode 100644 index 00000000..c1e9662a --- /dev/null +++ b/data-scripts/Pipfile @@ -0,0 +1,16 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +click = "*" +petl = "*" +ratelimit = "*" +tqdm = "*" +contentful-management = "*" + +[dev-packages] + +[requires] +python_version = "3" diff --git a/data-scripts/Pipfile.lock b/data-scripts/Pipfile.lock new file mode 100644 index 00000000..e114e4fa --- /dev/null +++ b/data-scripts/Pipfile.lock @@ -0,0 +1,231 @@ +{ + "_meta": { + "hash": { + "sha256": "c7ae2b4bb4927208da8238b4b11b2c0949ff809b5ef94aa9606370917624c28a" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", + "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7" + ], + "markers": "python_version >= '3.7'", + "version": "==2026.2.25" + }, + "charset-normalizer": { + "hashes": [ + "sha256:014837af6fabf57121b6254fa8ade10dceabc3528b27b721a64bbc7b8b1d4eb4", + "sha256:01a1ed54b953303ca7e310fafe0fe347aab348bd81834a0bcd602eb538f89d66", + "sha256:0294916d6ccf2d069727d65973c3a1ca477d68708db25fd758dd28b0827cff54", + "sha256:02a9d1b01c1e12c27883b0c9349e0bcd9ae92e727ff1a277207e1a262b1cbf05", + "sha256:036c079aa08a6a592b82487f97c60b439428320ed1b2ea0b3912e99d30c77765", + "sha256:039215608ac7b358c4da0191d10fc76868567fbf276d54c14721bdedeb6de064", + "sha256:0625665e4ebdddb553ab185de5db7054393af8879fb0c87bd5690d14379d6819", + "sha256:0a45e504f5e1be0bd385935a8e1507c442349ca36f511a47057a71c9d1d6ea9e", + "sha256:0b362bcd27819f9c07cbf23db4e0e8cd4b44c5ecd900c2ff907b2b92274a7412", + "sha256:0c300cefd9b0970381a46394902cd18eaf2aa00163f999590ace991989dcd0fc", + "sha256:1088345bcc93c58d8d8f3d783eca4a6e7a7752bbff26c3eee7e73c597c191c2e", + "sha256:10b473fc8dca1c3ad8559985794815f06ca3fc71942c969129070f2c3cdf7281", + "sha256:131716d6786ad5e3dc542f5cc6f397ba3339dc0fb87f87ac30e550e8987756af", + "sha256:14498a429321de554b140013142abe7608f9d8ccc04d7baf2ad60498374aefa2", + "sha256:149ec69866c3d6c2fb6f758dbc014ecb09f30b35a5ca90b6a8a2d4e54e18fdfe", + "sha256:165c7b21d19365464e8f70e5ce5e12524c58b48c78c1f5a57524603c1ab003f8", + "sha256:1827734a5b308b65ac54e86a618de66f935a4f63a8a462ff1e19a6788d6c2262", + "sha256:19092dde50335accf365cce21998a1c6dd8eafd42c7b226eb54b2747cdce2fac", + "sha256:1a374cc0b88aa710e8865dc1bd6edb3743c59f27830f0293ab101e4cf3ce9f85", + "sha256:1d1401945cb77787dbd3af2446ff2d75912327c4c3a1526ab7955ecf8600687c", + "sha256:1f2da5cbb9becfcd607757a169e38fb82aa5fd86fae6653dea716e7b613fe2cf", + "sha256:259cd1ca995ad525f638e131dbcc2353a586564c038fc548a3fe450a91882139", + "sha256:2820a98460c83663dd8ec015d9ddfd1e4879f12e06bb7d0500f044fb477d2770", + "sha256:28269983f25a4da0425743d0d257a2d6921ea7d9b83599d4039486ec5b9f911d", + "sha256:2b970382e4a36bed897c19f310f31d7d13489c11b4f468ddfba42d41cddfb918", + "sha256:2da4eedcb6338e2321e831a0165759c0c620e37f8cd044a263ff67493be8ffb3", + "sha256:30987f4a8ed169983f93e1be8ffeea5214a779e27ed0b059835c7afe96550ad7", + "sha256:30a2b1a48478c3428d047ed9690d57c23038dac838a87ad624c85c0a78ebeb39", + "sha256:340810d34ef83af92148e96e3e44cb2d3f910d2bf95e5618a5c467d9f102231d", + "sha256:3f64c6bf8f32f9133b668c7f7a7cbdbc453412bc95ecdbd157f3b1e377a92990", + "sha256:4167a621a9a1a986c73777dbc15d4b5eac8ac5c10393374109a343d4013ec765", + "sha256:4354e401eb6dab9aed3c7b4030514328a6c748d05e1c3e19175008ca7de84fb1", + "sha256:4481e6da1830c8a1cc0b746b47f603b653dadb690bcd851d039ffaefe70533aa", + "sha256:4b8551b6e6531e156db71193771c93bda78ffc4d1e6372517fe58ad3b91e4659", + "sha256:4cd966c2559f501c6fd69294d082c2934c8dd4719deb32c22961a5ac6db0df1d", + "sha256:50bcbca6603c06a1dcc7b056ed45c37715fb5d2768feb3bcd37d2313c587a5b9", + "sha256:530beedcec9b6e027e7a4b6ce26eed36678aa39e17da85e6e03d7bd9e8e9d7c9", + "sha256:568e3c34b58422075a1b49575a6abc616d9751b4d61b23f712e12ebb78fe47b2", + "sha256:573ef5814c4b7c0d59a7710aa920eaaaef383bd71626aa420fba27b5cab92e8d", + "sha256:58ad8270cfa5d4bef1bc85bd387217e14ff154d6630e976c6f56f9a040757475", + "sha256:597d10dec876923e5c59e48dbd366e852eacb2b806029491d307daea6b917d7c", + "sha256:5bcb3227c3d9aaf73eaaab1db7ccd80a8995c509ee9941e2aae060ca6e4e5d81", + "sha256:5cffde4032a197bd3b42fd0b9509ec60fb70918d6970e4cc773f20fc9180ca67", + "sha256:5fea359734b140d0d6741189fea5478c6091b54ffc69d7ce119e0a05637d8c99", + "sha256:60d68e820af339df4ae8358c7a2e7596badeb61e544438e489035f9fbf3246a5", + "sha256:610f72c0ee565dfb8ae1241b666119582fdbfe7c0975c175be719f940e110694", + "sha256:65a126fb4b070d05340a84fc709dd9e7c75d9b063b610ece8a60197a291d0adf", + "sha256:65b3c403a5b6b8034b655e7385de4f72b7b244869a22b32d4030b99a60593eca", + "sha256:66dee73039277eb35380d1b82cccc69cc82b13a66f9f4a18da32d573acf02b7c", + "sha256:708c7acde173eedd4bfa4028484426ba689d2103b28588c513b9db2cd5ecde9c", + "sha256:728c6a963dfab66ef865f49286e45239384249672cd598576765acc2a640a636", + "sha256:754f96058e61a5e22e91483f823e07df16416ce76afa4ebf306f8e1d1296d43f", + "sha256:75dfd1afe0b1647449e852f4fb428195a7ed0588947218f7ba929f6538487f02", + "sha256:75ee9c1cce2911581a70a3c0919d8bccf5b1cbc9b0e5171400ec736b4b569497", + "sha256:76a9d0de4d0eab387822e7b35d8f89367dd237c72e82ab42b9f7bf5e15ada00f", + "sha256:77be992288f720306ab4108fe5c74797de327f3248368dfc7e1a916d6ed9e5a2", + "sha256:7ad83b8f9379176c841f8865884f3514d905bcd2a9a3b210eaa446e7d2223e4d", + "sha256:8197abe5ca1ffb7d91e78360f915eef5addff270f8a71c1fc5be24a56f3e4873", + "sha256:82cc7c2ad42faec8b574351f8bc2a0c049043893853317bd9bb309f5aba6cb5a", + "sha256:8a28afb04baa55abf26df544e3e5c6534245d3daa5178bc4a8eeb48202060d0e", + "sha256:8b78d8a609a4b82c273257ee9d631ded7fac0d875bdcdccc109f3ee8328cfcb1", + "sha256:8ce11cd4d62d11166f2b441e30ace226c19a3899a7cf0796f668fba49a9fb123", + "sha256:8fff79bf5978c693c9b1a4d71e4a94fddfb5fe744eb062a318e15f4a2f63a550", + "sha256:92263f7eca2f4af326cd20de8d16728d2602f7cfea02e790dcde9d83c365d7cc", + "sha256:93b3b2cc5cf1b8743660ce77a4f45f3f6d1172068207c1defc779a36eea6bb36", + "sha256:95adae7b6c42a6c5b5b559b1a99149f090a57128155daeea91732c8d970d8644", + "sha256:97ab7787092eb9b50fb47fa04f24c75b768a606af1bcba1957f07f128a7219e4", + "sha256:9db5e3fcdcee89a78c04dffb3fe33c79f77bd741a624946db2591c81b2fc85b0", + "sha256:a118e2e0b5ae6b0120d5efa5f866e58f2bb826067a646431da4d6a2bdae7950e", + "sha256:a2aecdb364b8a1802afdc7f9327d55dad5366bc97d8502d0f5854e50712dbc5f", + "sha256:a66aa5022bf81ab4b1bebfb009db4fd68e0c6d4307a1ce5ef6a26e5878dfc9e4", + "sha256:a68766a3c58fde7f9aaa22b3786276f62ab2f594efb02d0a1421b6282e852e98", + "sha256:aa2f963b4da26daf46231d9b9e0e2c9408a751f8f0d0f44d2de56d3caf51d294", + "sha256:aa92ec1102eaff840ccd1021478af176a831f1bccb08e526ce844b7ddda85c22", + "sha256:ac59c15e3f1465f722607800c68713f9fbc2f672b9eb649fe831da4019ae9b23", + "sha256:ae8b03427410731469c4033934cf473426faff3e04b69d2dfb64a4281a3719f8", + "sha256:afca7f78067dd27c2b848f1b234623d26b87529296c6c5652168cc1954f2f3b2", + "sha256:b2d37d78297b39a9eb9eb92c0f6df98c706467282055419df141389b23f93362", + "sha256:b3e71afc578b98512bfe7bdb822dd6bc57d4b0093b4b6e5487c1e96ad4ace242", + "sha256:ba20bdf69bd127f66d0174d6f2a93e69045e0b4036dc1ca78e091bcc765830c4", + "sha256:c108f8619e504140569ee7de3f97d234f0fbae338a7f9f360455071ef9855a95", + "sha256:c23eb3263356d94858655b3e63f85ac5d50970c6e8febcdde7830209139cc37d", + "sha256:c5af897b45fa606b12464ccbe0014bbf8c09191e0a66aab6aa9d5cf6e77e0c94", + "sha256:c7a80a9242963416bd81f99349d5f3fce1843c303bd404f204918b6d75a75fd6", + "sha256:c7e84e0c0005e3bdc1a9211cd4e62c78ba80bc37b2365ef4410cd2007a9047f2", + "sha256:cace89841c0599d736d3d74a27bc5821288bb47c5441923277afc6059d7fbcb4", + "sha256:cd2d0f0ec9aa977a27731a3209ebbcacebebaf41f902bd453a928bfd281cf7f8", + "sha256:d01de5e768328646e6a3fa9e562706f8f6641708c115c62588aef2b941a4f88e", + "sha256:d1028de43596a315e2720a9849ee79007ab742c06ad8b45a50db8cdb7ed4a82a", + "sha256:d27ce22ec453564770d29d03a9506d449efbb9fa13c00842262b2f6801c48cce", + "sha256:d29dd9c016f2078b43d0c357511e87eee5b05108f3dd603423cb389b89813969", + "sha256:d31f0d1671e1534e395f9eb84a68e0fb670e1edb1fe819a9d7f564ae3bc4e53f", + "sha256:d4eb8ac7469b2a5d64b5b8c04f84d8bf3ad340f4514b98523805cbf46e3b3923", + "sha256:d5e52d127045d6ae01a1e821acfad2f3a1866c54d0e837828538fabe8d9d1bd6", + "sha256:d77f97e515688bd615c1d1f795d540f32542d514242067adcb8ef532504cb9ee", + "sha256:d8ed79b8f6372ca4254955005830fd61c1ccdd8c0fac6603e2c145c61dd95db6", + "sha256:dc57a0baa3eeedd99fafaef7511b5a6ef4581494e8168ee086031744e2679467", + "sha256:e09f671a54ce70b79a1fc1dc6da3072b7ef7251fadb894ed92d9aa8218465a5f", + "sha256:e22d1059b951e7ae7c20ef6b06afd10fb95e3c41bf3c4fbc874dba113321c193", + "sha256:e37bd100d2c5d3ba35db9c7c5ba5a9228cbcffe5c4778dc824b164e5257813d7", + "sha256:e51ae7d81c825761d941962450f50d041db028b7278e7b08930b4541b3e45cb9", + "sha256:e545b51da9f9af5c67815ca0eb40676c0f016d0b0381c86f20451e35696c5f95", + "sha256:e6302ca4ae283deb0af68d2fbf467474b8b6aedcd3dab4db187e07f94c109763", + "sha256:e71bbb595973622b817c042bd943c3f3667e9c9983ce3d205f973f486fec98a7", + "sha256:ec56a2266f32bc06ed3c3e2a8f58417ce02f7e0356edc89786e52db13c593c98", + "sha256:ed1a9a204f317ef879b32f9af507d47e49cd5e7f8e8d5d96358c98373314fc60", + "sha256:ed97c282ee4f994ef814042423a529df9497e3c666dca19be1d4cd1129dc7ade", + "sha256:ed98364e1c262cf5f9363c3eca8c2df37024f52a8fa1180a3610014f26eac51c", + "sha256:ee57b926940ba00bca7ba7041e665cc956e55ef482f851b9b65acb20d867e7a2", + "sha256:f1d725b754e967e648046f00c4facc42d414840f5ccc670c5670f59f83693e4f", + "sha256:f8102ae93c0bc863b1d41ea0f4499c20a83229f52ed870850892df555187154a", + "sha256:fc1c64934b8faf7584924143eb9db4770bbdb16659626e1a1a4d9efbcb68d947", + "sha256:ff95a9283de8a457e6b12989de3f9f5193430f375d64297d323a615ea52cbdb3" + ], + "markers": "python_version >= '3.7'", + "version": "==3.4.5" + }, + "click": { + "hashes": [ + "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", + "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6" + ], + "index": "pypi", + "markers": "python_version >= '3.10'", + "version": "==8.3.1" + }, + "contentful-management": { + "hashes": [ + "sha256:571855f24d5a67d6d0a372674200918ace2365db5709780008d6b826a0ddceed", + "sha256:be675e07fd87951c912b72b663459b7ee4d8823a849c8e94722ee34888a1e14f" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==2.15.1" + }, + "idna": { + "hashes": [ + "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", + "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902" + ], + "markers": "python_version >= '3.8'", + "version": "==3.11" + }, + "petl": { + "hashes": [ + "sha256:53785128bcdf46eb4472638ad572acc6d87cc83f80b567fed06ee4a947eea5d1", + "sha256:802696187c2ef35894c4acf3c0ff9fecff6035cb335944c194416b9a18e8390b" + ], + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.7.17" + }, + "python-dateutil": { + "hashes": [ + "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.9.0.post0" + }, + "ratelimit": { + "hashes": [ + "sha256:af8a9b64b821529aca09ebaf6d8d279100d766f19e90b5059ac6a718ca6dee42" + ], + "index": "pypi", + "version": "==2.2.1" + }, + "requests": { + "hashes": [ + "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", + "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422" + ], + "markers": "python_version >= '3.8'", + "version": "==2.32.4" + }, + "six": { + "hashes": [ + "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.17.0" + }, + "tqdm": { + "hashes": [ + "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", + "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==4.67.3" + }, + "urllib3": { + "hashes": [ + "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", + "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4" + ], + "markers": "python_version >= '3.9'", + "version": "==2.6.3" + } + }, + "develop": {} +} diff --git a/data-scripts/README.md b/data-scripts/README.md index 6c56c531..d1d754d6 100644 --- a/data-scripts/README.md +++ b/data-scripts/README.md @@ -2,6 +2,26 @@ Handy little python scripts to clean and merge voter turnout and registration data. Also a script to migrate content. +## Install dependencies + +This project uses [pipenv](https://pipenv.pypa.io/) to manage Python dependencies. + +```bash +# Install pipenv if you don't have it +pip install --user pipenv + +# Install dependencies +pipenv install + +# Run a script +pipenv run python cli.py --help + +# Start a virtual environment to run multiple commands +pipenv shell +``` + +## Usage + ``` Usage: cli.py [OPTIONS] COMMAND [ARGS]... diff --git a/data-scripts/requirements.txt b/data-scripts/requirements.txt deleted file mode 100644 index 5e45501b..00000000 --- a/data-scripts/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -click==6.7 -petl==1.1.1 -ratelimit==1.4.1 -contentful_management==1.4.0 -tqdm==4.19.2 From 0a057ca113257b65d16b033e891702f7bf914881 Mon Sep 17 00:00:00 2001 From: Peter Wolanin <107691+pwolanin@users.noreply.github.com> Date: Tue, 17 Mar 2026 20:17:59 -0400 Subject: [PATCH 2/8] add minimum versions --- data-scripts/Pipfile | 10 +++++----- data-scripts/README.md | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/data-scripts/Pipfile b/data-scripts/Pipfile index c1e9662a..45da723f 100644 --- a/data-scripts/Pipfile +++ b/data-scripts/Pipfile @@ -4,11 +4,11 @@ verify_ssl = true name = "pypi" [packages] -click = "*" -petl = "*" -ratelimit = "*" -tqdm = "*" -contentful-management = "*" +click = ">=8.3.1" +petl = ">=1.7.17" +ratelimit = ">=2.2.1" +tqdm = ">=4.67.3" +contentful-management = ">=2.15.1" [dev-packages] diff --git a/data-scripts/README.md b/data-scripts/README.md index d1d754d6..23030902 100644 --- a/data-scripts/README.md +++ b/data-scripts/README.md @@ -13,6 +13,9 @@ pip install --user pipenv # Install dependencies pipenv install +# Or to install the versions in the lock file without updating +pipenv sync + # Run a script pipenv run python cli.py --help From e30e1ebf5cd78535821f48cf950158c6a9e4d9e8 Mon Sep 17 00:00:00 2001 From: Peter Wolanin <107691+pwolanin@users.noreply.github.com> Date: Tue, 7 Apr 2026 20:48:38 -0400 Subject: [PATCH 3/8] trying to get import and export to work for leaders --- data-scripts/cli.py | 36 +++++++++++++++++--- data-scripts/contentful.py | 33 +++++++++++++++--- data-scripts/leaders.py | 70 ++++++++++++++++++++++++++------------ 3 files changed, 108 insertions(+), 31 deletions(-) diff --git a/data-scripts/cli.py b/data-scripts/cli.py index 3b0515b8..3a8adbcd 100644 --- a/data-scripts/cli.py +++ b/data-scripts/cli.py @@ -7,8 +7,8 @@ from turnout import process_turnout from committee import process_committee from divisions import process_divisions -from leaders import process_leaders -from contentful import process_import, process_drop +from leaders import process_leaders, export_leaders, collapse_party +from contentful import process_import, process_fetch, process_drop @click.group() def cli(): @@ -55,9 +55,37 @@ def leaders(leaders_file): help='Contentful.com Content type ID') @click.option('--apikey', 'api_key', required=True, help='Contentful.com API key') -def import_contentful(import_file, space_id, content_type, api_key): +@click.option('--environment', '-e', 'environment_id', default='master', + help='Contentful.com environment ID (default: master)') +@click.option('--update', '-u', is_flag=True, default=False, + help='Update existing entries instead of creating new ones') +def import_contentful(import_file, space_id, content_type, api_key, + environment_id, update): """Imports a JSON file to a contentful.com space""" - process_import(import_file, space_id, content_type, api_key) + process_import(import_file, space_id, content_type, api_key, + environment_id, update) + +@cli.command('export_leaders') +@click.option('--space', 'space_id', required=True, + help='Contentful.com Space ID') +@click.option('--apikey', 'api_key', required=True, + help='Contentful.com API key') +@click.option('--environment', '-e', 'environment_id', default='master', + help='Contentful.com environment ID (default: master)') +@click.option('--party', '-p', type=click.Choice( + ['democratic', 'republican', 'D', 'R'], case_sensitive=False), + default=None, help='Filter by party (democratic/D or republican/R)') +@click.option('--out', '-o', 'output_file', type=click.Path(), + required=True, help='Output CSV file path') +def export_leaders_cmd(space_id, api_key, environment_id, + party, output_file): + """Exports leaders from a contentful.com space to CSV""" + records = process_fetch(space_id, 'wardLeader', api_key, environment_id) + table = export_leaders(records) + if party: + table = table.select(lambda row: row['Party'] == collapse_party(party)) + table.tocsv(output_file) + click.echo(f'Exported {etl.nrows(table)} records to {output_file}') @cli.command('drop') @click.option('--space', 'space_id', required=True, diff --git a/data-scripts/contentful.py b/data-scripts/contentful.py index 41812dc6..a9fe705e 100644 --- a/data-scripts/contentful.py +++ b/data-scripts/contentful.py @@ -4,9 +4,10 @@ from tqdm import tqdm from ratelimit import rate_limited -def process_import(filepath, space_id, content_type_id, api_key): +def process_import(filepath, space_id, content_type_id, api_key, + environment_id='master', update=False): client = Client(api_key) - space = client.entries(space_id) + space = client.entries(space_id, environment_id) with open(filepath) as import_file: records = json.load(import_file) @@ -16,15 +17,23 @@ def process_import(filepath, space_id, content_type_id, api_key): if 'ID' in record.keys(): entry_id = record['ID'] record.pop('ID') + fields = {key: {'en-US': value} for (key, value) in record.items()} entry_data = { 'content_type_id': content_type_id, - 'fields': {key: {'en-US': value} for (key, value) in record.items()} + 'fields': fields } try: - entry = space.create(entry_id, entry_data) + if update and entry_id: + entry = space.find(entry_id) + for key, value in fields.items(): + entry.fields()[key] = value + entry.save() + else: + entry = space.create(entry_id, entry_data) except Exception as err: - print('Creation failed', json.dumps(record)) + action = 'Update' if update and entry_id else 'Creation' + print(f'{action} failed', json.dumps(record)) print(err) else: try: @@ -39,6 +48,20 @@ def delete_entry(entry): entry.delete() +def process_fetch(space_id, content_type_id, api_key, environment_id='master'): + """Fetches all entries for a content type and returns them as a list of dicts""" + client = Client(api_key) + content_type = client.content_types(space_id, environment_id).find(content_type_id) + entries = content_type.entries().all({'limit': 1000}) + + records = [] + for entry in entries: + record = {} + for field_name, field_value in entry.fields().items(): + record[field_name] = field_value + records.append(record) + return records + def process_drop(space_id, content_type_id, api_key): client = Client(api_key) content_type = client.content_types(space_id).find(content_type_id) diff --git a/data-scripts/leaders.py b/data-scripts/leaders.py index 294ca2c5..5d21de10 100644 --- a/data-scripts/leaders.py +++ b/data-scripts/leaders.py @@ -2,6 +2,34 @@ import petl as etl +# Mapping from CSV column names to Contentful field names +FIELD_MAP = { + 'Name': 'fullName', + 'Ward': 'ward', + 'Sub-Ward': 'subWard', + 'Party': 'party', + 'Nickname': 'nickname', + 'Phones': 'phone', + 'Address': 'address', + 'Gender': 'gender', + 'Ward of Residence': 'wardOfResidence', + 'Year of Birth': 'yearOfBirth', + 'Occupation': 'occupation', + 'LinkedIn': 'linkedin', + 'Facebook': 'facebook', + 'Twitter': 'twitter', + 'Email': 'email', + 'Photo': 'photoUrl', + 'Divisions': 'divisionCount', + 'Committee People': 'committeePersonCount', + 'Party Registered': 'registeredVotersParty', + 'Total Registered': 'registeredVotersTotal', + 'Party Turnout': 'turnoutParty', + 'Total Turnout': 'turnoutTotal', +} + +REVERSE_FIELD_MAP = {v: k for k, v in FIELD_MAP.items()} + def remove_dash_lines(value): """Some social media values are '---------'""" return None if value[:2] == '--' else value @@ -12,33 +40,21 @@ def expand_party(party): def expand_gender(gender): return 'Male' if gender == 'M' else 'Female' if gender == 'F' else None +def collapse_party(party): + return 'D' if party == 'democratic' else 'R' if party == 'republican' else party + +def collapse_gender(gender): + return 'M' if gender == 'Male' else 'F' if gender == 'Female' else gender + def empty_to_none(value): return None if value == '' else value +def none_to_empty(value): + return '' if value is None else value + def process_leaders(filepath): table = etl.fromcsv(filepath) \ - .rename({'Name': 'fullName', - 'Ward': 'ward', - 'Sub-Ward': 'subWard', - 'Party': 'party', - 'Nickname': 'nickname', - 'Phones': 'phone', - 'Address': 'address', - 'Gender': 'gender', - 'Ward of Residence': 'wardOfResidence', - 'Year of Birth': 'yearOfBirth', - 'Occupation': 'occupation', - 'LinkedIn': 'linkedin', - 'Facebook': 'facebook', - 'Twitter': 'twitter', - 'Email': 'email', - 'Photo': 'photoUrl', - 'Divisions': 'divisionCount', - 'Committee People': 'committeePersonCount', - 'Party Registered': 'registeredVotersParty', - 'Total Registered': 'registeredVotersTotal', - 'Party Turnout': 'turnoutParty', - 'Total Turnout': 'turnoutTotal'}) \ + .rename(FIELD_MAP) \ .cutout('Lat', 'Lng', 'Last Voted', 'Photo Offset', '2014 General Party Turnout', '2014 General Total Turnout') \ .convert(('ward', 'wardOfResidence', 'yearOfBirth', @@ -51,3 +67,13 @@ def process_leaders(filepath): .convertall(empty_to_none) return table + +def export_leaders(records): + """Converts Contentful leader records back to CSV-style table""" + table = etl.fromdicts(records) \ + .convert('party', collapse_party) \ + .convert('gender', collapse_gender) \ + .convertall(none_to_empty) \ + .rename(REVERSE_FIELD_MAP) + + return table From 7b38a6a63a76f506be7964f74cad2865c6b78d63 Mon Sep 17 00:00:00 2001 From: Peter Wolanin <107691+pwolanin@users.noreply.github.com> Date: Tue, 7 Apr 2026 20:53:07 -0400 Subject: [PATCH 4/8] Add some AI guidance --- AGENTS.md | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ CLAUDE.md | 1 + 2 files changed, 81 insertions(+) create mode 100644 AGENTS.md create mode 120000 CLAUDE.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..dc71b7a5 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,80 @@ +# AGENTS.md + +## Project Overview + +Philly Ward Leaders is a civic transparency web app showing Philadelphia's 69 Ward Leaders and their political influence. The frontend is a Vue 3 SPA that pulls data from Contentful CMS and static GeoJSON files. The `data-scripts/` subdirectory contains Python CLI tools for cleaning source data and importing it into Contentful. + +## Common Commands + +### Frontend (Node.js 20, run from repo root) + +```bash +npm ci # Install dependencies +npm run dev # Dev server on localhost:8080 (hot reload) +npm run build # Production build to ./build +npm run lint # ESLint (JS + Vue files) +npm run lint:fix # ESLint auto-fix +npm run format:check # Prettier check +npm run format # Prettier auto-fix +npm run cy:run # Cypress component tests (headless) +npm run cy:e2e # Cypress e2e tests (needs serve-http running) +npm run cy:open # Cypress interactive UI +``` + +### Data Scripts (Python 3 + pipenv, run from `data-scripts/`) + +```bash +pipenv install # Install dependencies +pipenv sync # Install exact locked versions +pipenv run python cli.py --help # Show available commands +pipenv run python cli.py leaders # Convert leaders CSV to JSON +pipenv run python cli.py committee # Clean committee person data +pipenv run python cli.py divisions -o # Split ward boundary GeoJSON +pipenv run python cli.py voters -r -t # Combine voter data +pipenv run python cli.py import --space ID --content-type ID --apikey KEY +``` + +### Docker (alternative local dev) + +```bash +docker compose build && docker compose up -d +docker compose exec ward-leaders bash # Then run npm/pipenv commands inside +``` + +## CI Pipeline + +GitHub Actions on push/PR to `master` (`.github/workflows/node.js.yml`): +1. `npm run format:check` +2. `npm run build` +3. `npm run cy:run` (component tests) +4. Cypress e2e tests (starts `serve-http`, runs e2e suite) + +## Architecture + +### Frontend (Vue 3 + Vite) + +- **Entry:** `src/main.js` creates the Vue app with Vue Router and Vuex store +- **Views** (`src/views/`): Page-level components for each route — splash, ward-leader-list, ward-leader (detail), city-map, content-page (CMS-driven), feedback +- **Components** (`src/components/`): Reusable UI — ward-map (Leaflet), baseball-card, geocoder, nav-bar, stats-bar, notification +- **Store** (`src/store/`): Vuex with actions (async API calls), mutations, getters. Key state: leaders list, currentLeader, wardBoundaries, citywideBoundaries, contentPage +- **API** (`src/api/index.js`): Contentful SDK + axios wrapper +- **Config** (`src/config.js`): Public Contentful space/access token, AIS API key +- **Routing** (`src/router/index.js`): 7 routes including `/leaders/:party`, `/leaders/:party/:ward/:slug`, `/map`, and dynamic `/:slug` for CMS content +- **Styling:** SCSS with Bulma CSS framework +- **Maps:** Leaflet with vue-leaflet for interactive ward boundary maps +- **Build output:** `./build` directory + +### Data Scripts (Python CLI) + +- **`cli.py`**: Click CLI entry point grouping all subcommands +- **Processing modules**: `leaders.py`, `committee.py`, `divisions.py`, `registry.py`, `turnout.py` — each uses PETL for ETL pipelines +- **`contentful.py`**: Contentful Management API integration for import/drop operations +- **`input_data/`**: Source CSV and GeoJSON files + +### Data Flow + +Raw CSV/GeoJSON in `data-scripts/input_data/` -> Python scripts transform -> JSON output or Contentful import -> Frontend fetches from Contentful API at runtime + static GeoJSON from `public/data/` + +## Linting + +ESLint config (`.eslintrc.json`) extends: `plugin:vue/strongly-recommended`, `airbnb`, `plugin:cypress/recommended`, `prettier`. Prettier has final say on formatting. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file From d5abbb79773d2589f95b44987d57c4d0db93ae72 Mon Sep 17 00:00:00 2001 From: Peter Wolanin Date: Wed, 8 Apr 2026 21:30:23 -0400 Subject: [PATCH 5/8] Include the ID in fetched data ddd --- data-scripts/contentful.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data-scripts/contentful.py b/data-scripts/contentful.py index a9fe705e..beb028a0 100644 --- a/data-scripts/contentful.py +++ b/data-scripts/contentful.py @@ -57,6 +57,7 @@ def process_fetch(space_id, content_type_id, api_key, environment_id='master'): records = [] for entry in entries: record = {} + record['ID'] = entry.id for field_name, field_value in entry.fields().items(): record[field_name] = field_value records.append(record) From 84ed92c8509b102098b6b5828a10dac750ba61dc Mon Sep 17 00:00:00 2001 From: Peter Wolanin Date: Tue, 21 Apr 2026 19:58:38 -0400 Subject: [PATCH 6/8] supply headers for export --- data-scripts/contentful.py | 6 ++++-- data-scripts/leaders.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/data-scripts/contentful.py b/data-scripts/contentful.py index beb028a0..27e7bbeb 100644 --- a/data-scripts/contentful.py +++ b/data-scripts/contentful.py @@ -1,4 +1,5 @@ import json +import pprint from contentful_management import Client from tqdm import tqdm @@ -58,8 +59,9 @@ def process_fetch(space_id, content_type_id, api_key, environment_id='master'): for entry in entries: record = {} record['ID'] = entry.id - for field_name, field_value in entry.fields().items(): - record[field_name] = field_value + """pprint.pprint(entry.__dict__)""" + for field_name, field_value in entry.raw.get('fields').items(): + record[field_name] = field_value['en-US'] records.append(record) return records diff --git a/data-scripts/leaders.py b/data-scripts/leaders.py index 5d21de10..c88f9f12 100644 --- a/data-scripts/leaders.py +++ b/data-scripts/leaders.py @@ -29,6 +29,7 @@ } REVERSE_FIELD_MAP = {v: k for k, v in FIELD_MAP.items()} +REVERSE_HEADERS = list(FIELD_MAP.values()) def remove_dash_lines(value): """Some social media values are '---------'""" @@ -70,7 +71,7 @@ def process_leaders(filepath): def export_leaders(records): """Converts Contentful leader records back to CSV-style table""" - table = etl.fromdicts(records) \ + table = etl.fromdicts(records, REVERSE_HEADERS) \ .convert('party', collapse_party) \ .convert('gender', collapse_gender) \ .convertall(none_to_empty) \ From 0323a9b1ed18ece41a4b4c3bf14574872245f604 Mon Sep 17 00:00:00 2001 From: Peter Wolanin Date: Tue, 21 Apr 2026 20:59:39 -0400 Subject: [PATCH 7/8] sort export, filter columns on import --- data-scripts/leaders.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-scripts/leaders.py b/data-scripts/leaders.py index c88f9f12..a0d4b5d9 100644 --- a/data-scripts/leaders.py +++ b/data-scripts/leaders.py @@ -56,8 +56,7 @@ def none_to_empty(value): def process_leaders(filepath): table = etl.fromcsv(filepath) \ .rename(FIELD_MAP) \ - .cutout('Lat', 'Lng', 'Last Voted', 'Photo Offset', - '2014 General Party Turnout', '2014 General Total Turnout') \ + .cut(REVERSE_HEADERS) \ .convert(('ward', 'wardOfResidence', 'yearOfBirth', 'divisionCount', 'committeePersonCount', 'registeredVotersParty', 'registeredVotersTotal', @@ -72,6 +71,7 @@ def process_leaders(filepath): def export_leaders(records): """Converts Contentful leader records back to CSV-style table""" table = etl.fromdicts(records, REVERSE_HEADERS) \ + .sort(['party', 'ward', 'subWard']) \ .convert('party', collapse_party) \ .convert('gender', collapse_gender) \ .convertall(none_to_empty) \ From fc73611176f53de175bee45e33567edb856844cc Mon Sep 17 00:00:00 2001 From: Peter Wolanin Date: Tue, 21 Apr 2026 21:06:39 -0400 Subject: [PATCH 8/8] retain ID field --- data-scripts/leaders.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data-scripts/leaders.py b/data-scripts/leaders.py index a0d4b5d9..2250a53d 100644 --- a/data-scripts/leaders.py +++ b/data-scripts/leaders.py @@ -4,6 +4,7 @@ # Mapping from CSV column names to Contentful field names FIELD_MAP = { + 'ID': 'ID', 'Name': 'fullName', 'Ward': 'ward', 'Sub-Ward': 'subWard',