Add robustness and engineering polish

.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+      - name: Ruff lint
+        run: ruff check .
+      - name: Ruff format check
+        run: ruff format --check .
+      - name: Mypy
+        run: mypy
+      - name: Bandit
+        run: bandit -r breach_scraper
+      - name: Tests
+        run: python -m unittest discover -s tests -v

.gitignore

@@ -1,2 +1,8 @@
 __pycache__/
 *.pyc
+.venv/
+.mypy_cache/
+.ruff_cache/
+dist/
+build/
+*.egg-info/

README.md

@@ -1,28 +1,60 @@
 # breach-web-scraper
-A Python tool for scraping breach websites to provide a nice summary.
 
-## WA AG scraper (initial source)
-This repository now includes a scraper for Washington Attorney General data breach notifications:
+[![CI](https://github.com/noderaven/breach-web-scraper/actions/workflows/ci.yml/badge.svg)](https://github.com/noderaven/breach-web-scraper/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+[![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](pyproject.toml)
 
-- Source URL: `https://www.atg.wa.gov/data-breach-notifications`
-- Script: `scraper/wa_atg_scraper.py`
+A Python tool for scraping breach websites to provide a nice summary. The first
+supported source is the Washington State Attorney General data breach
+notifications page. Runtime dependencies: none (standard library only).
+
+## Install
 
-### Usage
 ```bash
-python scraper/wa_atg_scraper.py --output json --limit 10
-python scraper/wa_atg_scraper.py --output markdown --out-file wa_breaches.md
-python scraper/wa_atg_scraper.py --output csv --out-file wa_breaches.csv
+pip install .
 ```
 
-### Output fields
-The parser normalizes column names from the HTML table to `snake_case`. For cells containing links, it also emits a `<column>_url` field.
+This installs the `breach-scraper` console command.
 
-### Known hurdles / maintenance notes
-- The scraper depends on the page containing a parseable HTML table.
-- If WA AG changes table structure or field names, parsing/normalization may need updates.
-- For production automation, add retries/backoff, persistence, and monitoring around this script.
+## Usage
 
-### Tests
 ```bash
-python -m unittest discover -s tests
+# Fetch live and print JSON (default), limited to 10 rows
+breach-scraper --output json --limit 10
+
+# Markdown / CSV to a file
+breach-scraper --output markdown --out-file wa_breaches.md
+breach-scraper --output csv --out-file wa_breaches.csv
+
+# Offline: parse a previously saved page (no network)
+breach-scraper --input-html saved_page.html --output json
+
+# Override the User-Agent or retry count
+breach-scraper --user-agent "my-agent/1.0" --retries 5
 ```
+
+If the source returns HTTP 403, the tool prints an actionable message; use
+`--input-html` with a saved copy of the page, a different network, or a
+different `--user-agent`.
+
+## Output fields
+
+Column names from the HTML table are normalized to `snake_case`. Cells that
+contain links also emit a `<column>_url` field.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+ruff check .
+ruff format --check .
+mypy
+bandit -r breach_scraper
+python -m unittest discover -s tests -v
+```
+
+## Source / maintenance notes
+
+- Source URL: `https://www.atg.wa.gov/data-breach-notifications`
+- The scraper depends on the page exposing a parseable HTML table; if the WA AG
+  changes the table structure or field names, parsing may need updates.

scraper/wa_atg_scraper.py → breach_scraper/wa_atg_scraper.py

@@ -1,7 +1,8 @@
 """Scraper for Washington Attorney General breach notifications.
 
 This module fetches the WA AG data breach page, extracts the breach table,
-normalizes the records, and can emit JSON/CSV/Markdown output.
+normalizes the records, and can emit JSON/CSV/Markdown output. It also supports
+an offline mode that parses a previously saved copy of the page.
 """
 
 from __future__ import annotations
@@ -11,15 +12,25 @@
 import json
 import re
 import sys
+import time
+from collections.abc import Iterable
 from dataclasses import dataclass, field
 from html.parser import HTMLParser
+from http.client import HTTPException
 from pathlib import Path
-from typing import Iterable
+from urllib.error import HTTPError, URLError
 from urllib.parse import urljoin
 from urllib.request import Request, urlopen
 
 DEFAULT_URL = "https://www.atg.wa.gov/data-breach-notifications"
 
+# The WA AG site returns HTTP 403 for clients that do not look like a browser,
+# so a browser-like User-Agent is the default; override it with --user-agent.
+DEFAULT_USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+)
+
 
 def _clean_text(value: str) -> str:
     return re.sub(r"\s+", " ", value or "").strip()
@@ -101,7 +112,12 @@ def handle_endtag(self, tag: str) -> None:
                     self.tables.append(self.current_table)
             return
 
-        if tag in {"th", "td"} and self.in_cell and self.current_cell and self.current_cell_tag == tag:
+        if (
+            tag in {"th", "td"}
+            and self.in_cell
+            and self.current_cell
+            and self.current_cell_tag == tag
+        ):
             self.current_row.append((tag, self.current_cell))
             self.in_cell = False
             self.current_cell_tag = None
@@ -111,25 +127,63 @@ def handle_endtag(self, tag: str) -> None:
         if tag == "tr" and self.in_row:
             self.in_row = False
             if self.current_row:
-                row_type = "header" if all(cell_tag == "th" for cell_tag, _ in self.current_row) else "data"
+                is_header = all(cell_tag == "th" for cell_tag, _ in self.current_row)
+                row_type = "header" if is_header else "data"
                 self.current_table.append((row_type, self.current_row))
 
     def handle_data(self, data: str) -> None:
         if self.in_cell and self.current_cell:
             self.current_cell.add_text(data)
 
 
-def fetch_html(url: str = DEFAULT_URL, timeout: int = 30) -> str:
+def fetch_html(
+    url: str = DEFAULT_URL,
+    *,
+    timeout: int = 30,
+    retries: int = 3,
+    backoff: float = 0.5,
+    user_agent: str | None = None,
+) -> str:
+    """Fetch page HTML, retrying transient errors with exponential backoff."""
     request = Request(
         url,
         headers={
-            "User-Agent": "Mozilla/5.0 (compatible; breach-web-scraper/1.0; +https://example.com)",
-            "Accept": "text/html,application/xhtml+xml",
+            "User-Agent": user_agent or DEFAULT_USER_AGENT,
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
         },
     )
-    with urlopen(request, timeout=timeout) as response:  # nosec B310 - expected for HTTP fetch
-        content_type = response.headers.get_content_charset() or "utf-8"
-        return response.read().decode(content_type, errors="replace")
+    attempts = max(1, retries)
+    last_error: Exception | None = None
+    for attempt in range(attempts):
+        try:
+            # urlopen here targets a vetted HTTP(S) URL, which is the intended use.
+            with urlopen(request, timeout=timeout) as response:  # nosec B310
+                charset = response.headers.get_content_charset() or "utf-8"
+                body: bytes = response.read()
+                try:
+                    return body.decode(charset, errors="replace")
+                except LookupError:
+                    # Server advertised an unknown charset; fall back to UTF-8.
+                    return body.decode("utf-8", errors="replace")
+        except HTTPError as exc:
+            if exc.code == 403:
+                raise RuntimeError(
+                    "Request blocked with HTTP 403. This source may require "
+                    "browser-like access from your network. Try: (1) run with "
+                    "--input-html using a saved copy of the page, (2) run from a "
+                    "different network, or (3) pass a different --user-agent."
+                ) from exc
+            if 500 <= exc.code < 600:
+                last_error = exc
+            else:
+                raise RuntimeError(f"Failed to fetch source page: HTTP {exc.code}.") from exc
+        except (URLError, TimeoutError, HTTPException) as exc:
+            last_error = exc
+        if attempt < attempts - 1:
+            time.sleep(min(backoff * (2**attempt), 30.0))
+    raise RuntimeError(
+        f"Failed to fetch source page after {attempts} attempt(s): {last_error}"
+    ) from last_error
 
 
 def parse_breach_table(html: str, base_url: str = DEFAULT_URL) -> list[dict[str, str]]:
@@ -212,18 +266,33 @@ def write_output(records: list[dict[str, str]], output_format: str, out_file: st
 def main(argv: list[str] | None = None) -> int:
     arg_parser = argparse.ArgumentParser(description="Scrape WA ATG breach notifications table.")
     arg_parser.add_argument("--url", default=DEFAULT_URL, help="Source page URL.")
+    arg_parser.add_argument(
+        "--input-html",
+        help="Path to a previously saved HTML page (offline mode; --url is ignored).",
+    )
+    arg_parser.add_argument("--user-agent", help="Override the request User-Agent header.")
+    arg_parser.add_argument(
+        "--retries", type=int, default=3, help="Max fetch attempts on transient errors."
+    )
     arg_parser.add_argument("--output", choices=["json", "csv", "markdown"], default="json")
     arg_parser.add_argument("--out-file", help="Optional output file path.")
     arg_parser.add_argument("--limit", type=int, default=0, help="Optional max records to output.")
     args = arg_parser.parse_args(argv)
 
-    html = fetch_html(args.url)
-    records = parse_breach_table(html, base_url=args.url)
+    try:
+        if args.input_html:
+            html = Path(args.input_html).read_text(encoding="utf-8")
+        else:
+            html = fetch_html(args.url, retries=args.retries, user_agent=args.user_agent)
+        records = parse_breach_table(html, base_url=args.url)
 
-    if args.limit and args.limit > 0:
-        records = records[: args.limit]
+        if args.limit and args.limit > 0:
+            records = records[: args.limit]
 
-    write_output(records, output_format=args.output, out_file=args.out_file)
+        write_output(records, output_format=args.output, out_file=args.out_file)
+    except (RuntimeError, OSError, ValueError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
     return 0
 
 

pyproject.toml

@@ -0,0 +1,44 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "breach-web-scraper"
+version = "0.1.0"
+description = "A Python tool for scraping breach websites to provide a nice summary."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [{ name = "noderaven" }]
+keywords = ["breach", "scraper", "security", "washington", "data-breach"]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
+  "Topic :: Security",
+]
+
+[project.urls]
+Homepage = "https://github.com/noderaven/breach-web-scraper"
+Repository = "https://github.com/noderaven/breach-web-scraper"
+
+[project.scripts]
+breach-scraper = "breach_scraper.wa_atg_scraper:main"
+
+[project.optional-dependencies]
+dev = ["ruff>=0.5", "mypy>=1.10", "bandit>=1.7"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["breach_scraper"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM", "C4"]
+
+[tool.mypy]
+python_version = "3.10"
+strict = true
+files = ["breach_scraper"]