"
+ )
+ for row in rows
+ for source_key, source_name in [
+ _html_source_metadata(row, default_source_key, default_source_name)
+ ]
+ for reported_display in [_html_reported_date(row)]
+ for breach_display in [str(row.get("date_of_breach", "") or "")]
+ for affected_display in [_html_affected_count(row)]
+ for affected_numeric in [_affected_count_numeric(row)]
+ )
+
+ return f"""
+
+
+
+
+ {safe_title}
+
+
+
+
+
+
Breach Monitoring
+
{safe_title}
+ {source_markup}
+
Last updated: {html.escape(generated_at)}
+
+
+
+
+ {" ".join(source_buttons)}
+
+
{html.escape(summary_text)}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ {table_rows}
+
+
+
+
No records match the selected source filter.
+
+
Page 1 of 1
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+
+def to_report(
+ records: Iterable[Row], title: str = "Breach Monitoring Report", source_url: str | None = None
+) -> str:
+ rows = list(records)
+ if not rows:
+ return f"# {title}\n\nNo records found."
+
+ lines = [f"# {title}", ""]
+ if source_url:
+ host = urlparse(source_url).netloc or source_url
+ lines.append(f"Source: [{host}]({source_url})")
+ lines.append("")
+
+ lines.append("## Summary")
+ lines.extend(_report_summary(rows))
+ lines.append("")
+ lines.append("## Breach Notices")
+ lines.append("")
+
+ for idx, row in enumerate(rows, start=1):
+ name = str(row.get("organization_name", "") or "Unknown organization")
+ lines.append(f"### {idx}. {name}")
+ lines.append(f"- Date reported: {row.get('date_reported', '') or 'Unknown'}")
+ lines.append(f"- Date of breach: {row.get('date_of_breach', '') or 'Unknown'}")
+ lines.append(
+ "- Persons affected: "
+ f"{row.get('persons_affected', '') or row.get('total_persons_affected', '') or row.get('number_affected', '') or row.get('individuals_affected', '') or 'Unknown'}"
+ )
+ lines.append(
+ f"- Information compromised: {row.get('information_compromised', '') or 'Not listed'}"
+ )
+ if row.get("organization_name_url"):
+ lines.append(f"- Notice letter: {row['organization_name_url']}")
+ elif row.get("notice_url"):
+ lines.append(f"- Notice letter: {row['notice_url']}")
+ lines.append("")
+
+ return "\n".join(lines).strip()
+
+
def to_markdown(records: Iterable[Row]) -> str:
rows = list(records)
if not rows:
return "No records found."
+
columns = _display_columns(rows)
header = "| " + " | ".join(_humanize_column(column) for column in columns) + " |"
divider = "| " + " | ".join(["---"] * len(columns)) + " |"
@@ -93,27 +993,15 @@ def to_markdown(records: Iterable[Row]) -> str:
return "\n".join([header, divider, *body])
-def _to_csv(records: list[dict[str, Any]]) -> str:
- if not records:
- return ""
- columns: list[str] = []
- for row in records:
- for key in row:
- if key not in columns:
- columns.append(key)
- buffer = StringIO()
- writer = csv.DictWriter(buffer, fieldnames=columns, lineterminator="\n")
- writer.writeheader()
- writer.writerows(records)
- return buffer.getvalue()
-
-
def write_output(
records: Sequence[Row],
output_format: str,
out_file: str | None,
*,
title: str = "Breach Monitoring Report",
+ source_url: str | None = None,
+ source_key: str | None = None,
+ source_name: str | None = None,
) -> None:
public_records = _public_records(records)
@@ -121,8 +1009,31 @@ def write_output(
payload = json.dumps(public_records, indent=2)
elif output_format == "markdown":
payload = to_markdown(public_records)
+ elif output_format == "html":
+ payload = to_html(
+ records,
+ title=title,
+ source_url=source_url,
+ default_source_key=source_key,
+ default_source_name=source_name,
+ )
+ elif output_format == "report":
+ payload = to_report(public_records, title=title, source_url=source_url)
elif output_format == "csv":
- payload = _to_csv(public_records)
+ if not public_records:
+ payload = ""
+ else:
+ columns: list[str] = []
+ for row in public_records:
+ for key in row:
+ if key not in columns:
+ columns.append(key)
+
+ buffer = StringIO()
+ writer = csv.DictWriter(buffer, fieldnames=columns, lineterminator="\n")
+ writer.writeheader()
+ writer.writerows(public_records)
+ payload = buffer.getvalue()
else:
raise ValueError(f"Unsupported output format: {output_format}")
From 4c0995eedda9d5c58beb1dc9a17a5755b7a0ed48 Mon Sep 17 00:00:00 2001
From: noderaven <130021665+noderaven@users.noreply.github.com>
Date: Thu, 25 Jun 2026 01:13:50 +0000
Subject: [PATCH 2/4] Wire report and html into the CLI
---
breach_scraper/cli.py | 4 +++-
pyproject.toml | 2 ++
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/breach_scraper/cli.py b/breach_scraper/cli.py
index 2d882db..c17b443 100644
--- a/breach_scraper/cli.py
+++ b/breach_scraper/cli.py
@@ -31,7 +31,9 @@ def build_arg_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Scrape and combine breach notification data from supported sources."
)
- parser.add_argument("--output", choices=["json", "csv", "markdown"], default="json")
+ parser.add_argument(
+ "--output", choices=["json", "csv", "markdown", "report", "html"], default="json"
+ )
parser.add_argument("--out-file", help="Optional output file path.")
parser.add_argument(
"--start-date",
diff --git a/pyproject.toml b/pyproject.toml
index 5799e31..fefcff1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -41,6 +41,8 @@ select = ["E", "F", "I", "UP", "B", "SIM", "C4"]
[tool.ruff.lint.per-file-ignores]
# Test fixtures embed long inline HTML/CSV sample strings.
"tests/**" = ["E501"]
+# output.py embeds an HTML/CSS/JS web-UI template with unavoidably long lines.
+"breach_scraper/output.py" = ["E501"]
[tool.mypy]
python_version = "3.10"
From 0dc97162de2b4cd86377617f1b28a3474c387bfd Mon Sep 17 00:00:00 2001
From: noderaven <130021665+noderaven@users.noreply.github.com>
Date: Thu, 25 Jun 2026 01:13:50 +0000
Subject: [PATCH 3/4] Add report/html CLI tests and refresh README for
multi-source
---
README.md | 74 +++++++++++++++++++++++++++++++++--------------
tests/test_cli.py | 18 ++++++++++++
2 files changed, 70 insertions(+), 22 deletions(-)
diff --git a/README.md b/README.md
index 8cb1d84..89bc239 100644
--- a/README.md
+++ b/README.md
@@ -4,9 +4,20 @@
[](LICENSE)
[](pyproject.toml)
-A Python tool for scraping breach websites to provide a nice summary. The first
-supported source is the Washington State Attorney General data breach
-notifications page. Runtime dependencies: none (standard library only).
+A Python tool for scraping breach-notification websites and combining them into a
+single summary. Runtime dependencies: none (standard library only).
+
+## Sources
+
+| Key | Source |
+| --- | --- |
+| `wa_atg` | Washington Attorney General |
+| `ca_oag` | California Attorney General |
+| `or_doj` | Oregon Department of Justice |
+| `maine_ag` | Maine Attorney General |
+| `hhs_ocr` | HHS OCR Breach Portal (federal) |
+
+List them anytime with `breach-scraper --list-sources`.
## Install
@@ -19,28 +30,44 @@ This installs the `breach-scraper` console command.
## Usage
```bash
-# Fetch live and print JSON (default), limited to 10 rows
-breach-scraper --output json --limit 10
+# Combine all sources for the last 6 months (default), as JSON
+breach-scraper
-# Markdown / CSV to a file
-breach-scraper --output markdown --out-file wa_breaches.md
-breach-scraper --output csv --out-file wa_breaches.csv
+# A single source, an explicit date range, Markdown to a file
+breach-scraper --source wa_atg --start-date 2025-01-01 --end-date 2025-06-30 \
+ --output markdown --out-file breaches.md
-# Offline: parse a previously saved page (no network)
-breach-scraper --input-html saved_page.html --output json
+# A readable text summary, or a self-contained sortable HTML dashboard
+breach-scraper --source ca_oag --output report
+breach-scraper --output html --out-file breaches.html
-# Override the User-Agent or retry count
-breach-scraper --user-agent "my-agent/1.0" --retries 5
+# Offline: parse a previously saved page for one source (no network)
+breach-scraper --source wa_atg --input-html saved_page.html
+
+# Override the User-Agent / retry count; fail hard if any source errors
+breach-scraper --user-agent "my-agent/1.0" --retries 5 --strict --verbose
```
-If the source returns HTTP 403, the tool prints an actionable message; use
-`--input-html` with a saved copy of the page, a different network, or a
-different `--user-agent`.
+### Options
+
+- `--source KEY` (repeatable): limit to specific sources. Default: all.
+- `--list-sources`: print supported source keys and exit.
+- `--start-date` / `--end-date` (`YYYY-MM-DD`): inclusive date window. Default: last 6 months.
+- `--output {json,csv,markdown,report,html}`: output format. Default: `json`.
+- `--out-file PATH`: write output to a file instead of stdout.
+- `--input-html PATH`: parse a saved page offline (requires exactly one `--source`).
+- `--user-agent STR`, `--retries N`: request tuning.
+- `--include-undated`: keep records with no parseable date.
+- `--strict`: exit non-zero if any source fails. `--verbose`: per-source progress on stderr.
+
+If a source returns HTTP 403, the tool prints an actionable message; use
+`--input-html` with a saved copy, a different network, or a different `--user-agent`.
-## Output fields
+## Output formats
-Column names from the HTML table are normalized to `snake_case`. Cells that
-contain links also emit a `_url` field.
+- `json` / `csv` / `markdown`: the combined records (column names normalized to `snake_case`).
+- `report`: a text summary with totals plus per-notice detail.
+- `html`: a self-contained, sortable web dashboard (no external assets).
## Development
@@ -53,8 +80,11 @@ bandit -r breach_scraper
python -m unittest discover -s tests -v
```
-## Source / maintenance notes
+## Notes
-- Source URL: `https://www.atg.wa.gov/data-breach-notifications`
-- The scraper depends on the page exposing a parseable HTML table; if the WA AG
- changes the table structure or field names, parsing may need updates.
+- Standard library only at runtime; sources fetch through a shared HTTP helper with
+ retries, exponential backoff, and helpful HTTP 403 handling.
+- Each source exposes `DEFAULT_URL`, `fetch_html`, and `parse_breach_table`, registered in
+ `breach_scraper/registry.py`. Adding a source is a small, self-contained module.
+- Scraping depends on the upstream pages' structure; if a site changes, that source's
+ parser may need updates.
diff --git a/tests/test_cli.py b/tests/test_cli.py
index e289ddc..45cf920 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -72,6 +72,24 @@ def test_offline_csv_has_unix_line_endings(self) -> None:
self.assertNotIn("\r", out)
self.assertIn("date_reported,organization_name", out)
+ def test_report_output(self) -> None:
+ rc, out, _ = run(
+ ["--input-html", str(FIXTURE), "--source", "wa_atg", "--output", "report", *RANGE]
+ )
+ self.assertEqual(rc, 0)
+ self.assertIn("## Summary", out)
+ self.assertIn("Total breach notices reviewed: 2", out)
+
+ def test_html_output_is_ascii_and_sortable(self) -> None:
+ rc, out, _ = run(
+ ["--input-html", str(FIXTURE), "--source", "wa_atg", "--output", "html", *RANGE]
+ )
+ self.assertEqual(rc, 0)
+ self.assertIn("