diff --git a/.gitattributes b/.gitattributes index 9c8348a..a4a6637 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1 @@ -docs.html.pythonmod/* linguist-documentation +client-unbound/docs.pythonmod/* linguist-documentation diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 249e395..c470722 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,10 +5,9 @@ on: pull_request: jobs: - pure: - # The pure suites (wire, expiry, validation, persist files, PF struct - # layout) are host-independent. The PF ioctl and live-firewall suites skip - # themselves here and must be run on an OpenBSD host. + python: + # Host-independent suites: protocol, client and server. The PF ioctl and + # live-firewall suites skip here by design and need an OpenBSD host. runs-on: ubuntu-latest strategy: matrix: @@ -19,20 +18,32 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: python -m pip install --upgrade pip lz4 pyyaml pytest + run: | + python -m pip install --upgrade pip pytest + python -m pip install -r server-python/requirements.txt + python -m pip install -r client-unbound/requirements.txt - name: Run tests - run: python -m pytest tests/ -v + run: python -m pytest -v - name: Check both daemons parse - run: python -m py_compile pfui_firewall.py pfui_unbound.py + run: python -m py_compile server-python/pfui_firewall.py client-unbound/pfui_unbound.py + + server-c: + # Builds the C framing implementation and runs the same conformance vectors + # as the Python one, so the two cannot drift apart unnoticed. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build and run protocol vectors + run: make -C server-c test shell: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Syntax-check the install and ops scripts + - name: Syntax-check the installers and ops scripts run: | - for f in *.sh; do bash -n "$f"; done + for f in *.sh client-unbound/tools/*.sh; do bash -n "$f"; done - name: Syntax-check the rc.d scripts run: | sudo apt-get update -qq && sudo apt-get install -y -qq ksh - for f in rc.d/*; do ksh -n "$f"; done + for f in client-unbound/rc.d/* server-python/rc.d/*; do ksh -n "$f"; done diff --git a/DECISIONS.md b/DECISIONS.md index 166204b..d8aea60 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -62,6 +62,6 @@ mismatched pair fails closed in either direction. ## Unbound runs unchrooted -`chroot: ""` in the shipped resolver config. `pfui_unbound.py` imports `lz4` and +`chroot: ""` in the shipped resolver config. `client-unbound/pfui_unbound.py` imports `lz4` and `yaml` at module load, and those would have to exist inside the chroot. Revisit only alongside a plan for the module's dependencies. diff --git a/INSTALL.md b/INSTALL.md index 792e3d4..eb1f4e1 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -34,7 +34,7 @@ export PKG_PATH=http://ftp.openbsd.org/pub/OpenBSD/%v/packages/%a/ pkg_add -i bash # All edge OpenBSD Firewalls -pfui_firewall_install.sh +install-server-python.sh # All internal DNS Servers install_pfui_unbound.sh @@ -80,7 +80,7 @@ pkg_add -i swig git bash cmake libconfig libiconv bison gawk mawk ``` #### TODO: PFUI_Unbound Dependencies -python3 -m pip install -r ./requirements-unbound.txt +python3 -m pip install -r ./client-unbound/requirements.txt ### PFUI_Unbound - Download Unbound source ``` diff --git a/README.md b/README.md index be17a10..bd73921 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,24 @@ restricted to only trusted DNS resolvers (with corporate/community DNS Blocklist Ie, Users cannot bypass an administrator's DNS blocking attempts using 'DNS over TCP/HTTPS' (DoT/DoH), or other any other tunneling technology etc. +------ +## Repository layout + +| Path | What it is | +|------|------------| +| [protocol/](protocol/) | The wire protocol: specification, conformance vectors, and the Python reference implementation shared by clients and servers | +| [client-unbound/](client-unbound/) | PFUI client as an Unbound pythonmod plugin | +| [server-python/](server-python/) | PFUI server for OpenBSD PF, in Python | +| [server-c/](server-c/) | PFUI server in C. Framing only so far | +| `install-client-unbound.sh` | Installs the Unbound client on a resolver | +| `install-server-python.sh` | Installs the Python server on a PF firewall | +| [examples/pf.conf](examples/pf.conf) | Example PF ruleset, applies to any server implementation | + +Clients and servers share only the protocol. Adding support for another resolver +means a new `client-/`; a second server implementation means a new +`server-/`. Both cases are conformance-tested against +[protocol/vectors/](protocol/vectors/) rather than against each other. + ------ ## PFUI Firewall comprises two parts @@ -80,7 +98,7 @@ up to OpenBSD 7.4, Unbound 1.18, Python 3.10 ``` pkg_add bash git clone https://github.com/andylemin/PFUI.git && cd PFUI -doas ./pfui_firewall_install.sh +doas ./install-server-python.sh ``` * 1b) Now add IP Reputation Block Lists to PF Firewalls (optional/recommended);\ https://www.geoghegan.ca/pfbadhost.html \ @@ -99,7 +117,7 @@ sysctl net.inet.tcp.ackonpush=1 ``` pkg_add bash git clone https://github.com/andylemin/PFUI.git && cd PFUI -doas ./pfui_unbound_install.sh +doas ./install-client-unbound.sh ``` Note the following lines in the example Unbound `/var/unbound/etc/pfui_unbound.conf` file after install (copy these to your own Unbound config or use the example); ``` @@ -247,7 +265,7 @@ This will avoid issues with local firewall services, which are assumed to be tru ------ ### Docs; -The Unbound "Python Module" [documentation](docs.html.pythonmod/index.html) has been included here for reference +The Unbound "Python Module" [documentation](client-unbound/docs.pythonmod/index.html) has been included here for reference (requires compiling from source) and all rights remain with Unbound arthor's Nlnetlabs. The Python Module documentation for Unbound was built with SWIG on: Sep 3 13:18 2019 diff --git a/client-unbound/README.md b/client-unbound/README.md new file mode 100644 index 0000000..05c18c5 --- /dev/null +++ b/client-unbound/README.md @@ -0,0 +1,29 @@ +# PFUI_Unbound (client) + +The PFUI client for [Unbound](https://nlnetlabs.nl/projects/unbound/about/). +Installed on the resolvers, it forwards successfully resolved IPs and their TTLs +to every PFUI server. + +`pfui_unbound.py` is an Unbound **pythonmod plugin**, not a script. Unbound +executes it in the embedded interpreter's `__main__` namespace, injects +`log_info`, `log_err` and the `MODULE_*` constants, and calls `init`, +`init_standard`, `deinit`, `inform_super`, `operate` and +`inplace_cache_callback` itself. Nothing in PFUI calls those, which is why +`tests/test_unbound_module.py` pins their signatures: a rename here would surface +only on a running resolver. + +| Path | What it is | +|------|------------| +| `pfui_unbound.py` | The plugin | +| `pfui_unbound.yml` | PFUI client configuration (firewall list, transport, timeouts) | +| `examples/pfui_unbound.conf` | Example Unbound config with the module enabled | +| `rc.d/pfui_unbound` | OpenBSD rc.d script for the Unbound build with pythonmod | +| `tools/` | Root hints and DNS blocklist updaters, run from cron | +| `docs.pythonmod/` | Vendored Unbound pythonmod documentation, from NLnet Labs | + +Install with `../install-client-unbound.sh` from the repository root. The wire +format it speaks is specified in [../protocol/PROTOCOL.md](../protocol/PROTOCOL.md); +the installer places `pfui_wire.py` beside the plugin. + +Adding a client for another resolver means a new `client-/` directory +that speaks the same protocol. Nothing in this directory is shared. diff --git a/docs.html.pythonmod/.buildinfo b/client-unbound/docs.pythonmod/.buildinfo similarity index 100% rename from docs.html.pythonmod/.buildinfo rename to client-unbound/docs.pythonmod/.buildinfo diff --git a/docs.html.pythonmod/.doctrees/environment.pickle b/client-unbound/docs.pythonmod/.doctrees/environment.pickle similarity index 100% rename from docs.html.pythonmod/.doctrees/environment.pickle rename to client-unbound/docs.pythonmod/.doctrees/environment.pickle diff --git a/docs.html.pythonmod/.doctrees/examples/example0.doctree b/client-unbound/docs.pythonmod/.doctrees/examples/example0.doctree similarity index 100% rename from docs.html.pythonmod/.doctrees/examples/example0.doctree rename to client-unbound/docs.pythonmod/.doctrees/examples/example0.doctree diff --git a/docs.html.pythonmod/.doctrees/examples/example1.doctree b/client-unbound/docs.pythonmod/.doctrees/examples/example1.doctree similarity index 100% rename from docs.html.pythonmod/.doctrees/examples/example1.doctree rename to client-unbound/docs.pythonmod/.doctrees/examples/example1.doctree diff --git a/docs.html.pythonmod/.doctrees/examples/example2.doctree b/client-unbound/docs.pythonmod/.doctrees/examples/example2.doctree similarity index 100% rename from docs.html.pythonmod/.doctrees/examples/example2.doctree rename to client-unbound/docs.pythonmod/.doctrees/examples/example2.doctree diff --git a/docs.html.pythonmod/.doctrees/examples/example3.doctree b/client-unbound/docs.pythonmod/.doctrees/examples/example3.doctree similarity index 100% rename from docs.html.pythonmod/.doctrees/examples/example3.doctree rename to client-unbound/docs.pythonmod/.doctrees/examples/example3.doctree diff --git a/docs.html.pythonmod/.doctrees/examples/example4.doctree b/client-unbound/docs.pythonmod/.doctrees/examples/example4.doctree similarity index 100% rename from docs.html.pythonmod/.doctrees/examples/example4.doctree rename to client-unbound/docs.pythonmod/.doctrees/examples/example4.doctree diff --git a/docs.html.pythonmod/.doctrees/examples/example5.doctree b/client-unbound/docs.pythonmod/.doctrees/examples/example5.doctree similarity index 100% rename from docs.html.pythonmod/.doctrees/examples/example5.doctree rename to client-unbound/docs.pythonmod/.doctrees/examples/example5.doctree diff --git a/docs.html.pythonmod/.doctrees/examples/example6.doctree b/client-unbound/docs.pythonmod/.doctrees/examples/example6.doctree similarity index 100% rename from docs.html.pythonmod/.doctrees/examples/example6.doctree rename to client-unbound/docs.pythonmod/.doctrees/examples/example6.doctree diff --git a/docs.html.pythonmod/.doctrees/examples/index.doctree b/client-unbound/docs.pythonmod/.doctrees/examples/index.doctree similarity index 100% rename from docs.html.pythonmod/.doctrees/examples/index.doctree rename to client-unbound/docs.pythonmod/.doctrees/examples/index.doctree diff --git a/docs.html.pythonmod/.doctrees/index.doctree b/client-unbound/docs.pythonmod/.doctrees/index.doctree similarity index 100% rename from docs.html.pythonmod/.doctrees/index.doctree rename to client-unbound/docs.pythonmod/.doctrees/index.doctree diff --git a/docs.html.pythonmod/.doctrees/install.doctree b/client-unbound/docs.pythonmod/.doctrees/install.doctree similarity index 100% rename from docs.html.pythonmod/.doctrees/install.doctree rename to client-unbound/docs.pythonmod/.doctrees/install.doctree diff --git a/docs.html.pythonmod/.doctrees/modules/config.doctree b/client-unbound/docs.pythonmod/.doctrees/modules/config.doctree similarity index 100% rename from docs.html.pythonmod/.doctrees/modules/config.doctree rename to client-unbound/docs.pythonmod/.doctrees/modules/config.doctree diff --git a/docs.html.pythonmod/.doctrees/modules/env.doctree b/client-unbound/docs.pythonmod/.doctrees/modules/env.doctree similarity index 100% rename from docs.html.pythonmod/.doctrees/modules/env.doctree rename to client-unbound/docs.pythonmod/.doctrees/modules/env.doctree diff --git a/docs.html.pythonmod/.doctrees/modules/functions.doctree b/client-unbound/docs.pythonmod/.doctrees/modules/functions.doctree similarity index 100% rename from docs.html.pythonmod/.doctrees/modules/functions.doctree rename to client-unbound/docs.pythonmod/.doctrees/modules/functions.doctree diff --git a/docs.html.pythonmod/.doctrees/modules/index.doctree b/client-unbound/docs.pythonmod/.doctrees/modules/index.doctree similarity index 100% rename from docs.html.pythonmod/.doctrees/modules/index.doctree rename to client-unbound/docs.pythonmod/.doctrees/modules/index.doctree diff --git a/docs.html.pythonmod/.doctrees/modules/struct.doctree b/client-unbound/docs.pythonmod/.doctrees/modules/struct.doctree similarity index 100% rename from docs.html.pythonmod/.doctrees/modules/struct.doctree rename to client-unbound/docs.pythonmod/.doctrees/modules/struct.doctree diff --git a/docs.html.pythonmod/.doctrees/usecase.doctree b/client-unbound/docs.pythonmod/.doctrees/usecase.doctree similarity index 100% rename from docs.html.pythonmod/.doctrees/usecase.doctree rename to client-unbound/docs.pythonmod/.doctrees/usecase.doctree diff --git a/docs.html.pythonmod/_static/ajax-loader.gif b/client-unbound/docs.pythonmod/_static/ajax-loader.gif similarity index 100% rename from docs.html.pythonmod/_static/ajax-loader.gif rename to client-unbound/docs.pythonmod/_static/ajax-loader.gif diff --git a/docs.html.pythonmod/_static/basic.css b/client-unbound/docs.pythonmod/_static/basic.css similarity index 100% rename from docs.html.pythonmod/_static/basic.css rename to client-unbound/docs.pythonmod/_static/basic.css diff --git a/docs.html.pythonmod/_static/classic.css b/client-unbound/docs.pythonmod/_static/classic.css similarity index 100% rename from docs.html.pythonmod/_static/classic.css rename to client-unbound/docs.pythonmod/_static/classic.css diff --git a/docs.html.pythonmod/_static/comment-bright.png b/client-unbound/docs.pythonmod/_static/comment-bright.png similarity index 100% rename from docs.html.pythonmod/_static/comment-bright.png rename to client-unbound/docs.pythonmod/_static/comment-bright.png diff --git a/docs.html.pythonmod/_static/comment-close.png b/client-unbound/docs.pythonmod/_static/comment-close.png similarity index 100% rename from docs.html.pythonmod/_static/comment-close.png rename to client-unbound/docs.pythonmod/_static/comment-close.png diff --git a/docs.html.pythonmod/_static/comment.png b/client-unbound/docs.pythonmod/_static/comment.png similarity index 100% rename from docs.html.pythonmod/_static/comment.png rename to client-unbound/docs.pythonmod/_static/comment.png diff --git a/docs.html.pythonmod/_static/doctools.js b/client-unbound/docs.pythonmod/_static/doctools.js similarity index 100% rename from docs.html.pythonmod/_static/doctools.js rename to client-unbound/docs.pythonmod/_static/doctools.js diff --git a/docs.html.pythonmod/_static/down-pressed.png b/client-unbound/docs.pythonmod/_static/down-pressed.png similarity index 100% rename from docs.html.pythonmod/_static/down-pressed.png rename to client-unbound/docs.pythonmod/_static/down-pressed.png diff --git a/docs.html.pythonmod/_static/down.png b/client-unbound/docs.pythonmod/_static/down.png similarity index 100% rename from docs.html.pythonmod/_static/down.png rename to client-unbound/docs.pythonmod/_static/down.png diff --git a/docs.html.pythonmod/_static/file.png b/client-unbound/docs.pythonmod/_static/file.png similarity index 100% rename from docs.html.pythonmod/_static/file.png rename to client-unbound/docs.pythonmod/_static/file.png diff --git a/docs.html.pythonmod/_static/jquery.js b/client-unbound/docs.pythonmod/_static/jquery.js similarity index 100% rename from docs.html.pythonmod/_static/jquery.js rename to client-unbound/docs.pythonmod/_static/jquery.js diff --git a/docs.html.pythonmod/_static/minus.png b/client-unbound/docs.pythonmod/_static/minus.png similarity index 100% rename from docs.html.pythonmod/_static/minus.png rename to client-unbound/docs.pythonmod/_static/minus.png diff --git a/docs.html.pythonmod/_static/plus.png b/client-unbound/docs.pythonmod/_static/plus.png similarity index 100% rename from docs.html.pythonmod/_static/plus.png rename to client-unbound/docs.pythonmod/_static/plus.png diff --git a/docs.html.pythonmod/_static/pygments.css b/client-unbound/docs.pythonmod/_static/pygments.css similarity index 100% rename from docs.html.pythonmod/_static/pygments.css rename to client-unbound/docs.pythonmod/_static/pygments.css diff --git a/docs.html.pythonmod/_static/readme b/client-unbound/docs.pythonmod/_static/readme similarity index 100% rename from docs.html.pythonmod/_static/readme rename to client-unbound/docs.pythonmod/_static/readme diff --git a/docs.html.pythonmod/_static/searchtools.js b/client-unbound/docs.pythonmod/_static/searchtools.js similarity index 100% rename from docs.html.pythonmod/_static/searchtools.js rename to client-unbound/docs.pythonmod/_static/searchtools.js diff --git a/docs.html.pythonmod/_static/sidebar.js b/client-unbound/docs.pythonmod/_static/sidebar.js similarity index 100% rename from docs.html.pythonmod/_static/sidebar.js rename to client-unbound/docs.pythonmod/_static/sidebar.js diff --git a/docs.html.pythonmod/_static/underscore.js b/client-unbound/docs.pythonmod/_static/underscore.js similarity index 100% rename from docs.html.pythonmod/_static/underscore.js rename to client-unbound/docs.pythonmod/_static/underscore.js diff --git a/docs.html.pythonmod/_static/up-pressed.png b/client-unbound/docs.pythonmod/_static/up-pressed.png similarity index 100% rename from docs.html.pythonmod/_static/up-pressed.png rename to client-unbound/docs.pythonmod/_static/up-pressed.png diff --git a/docs.html.pythonmod/_static/up.png b/client-unbound/docs.pythonmod/_static/up.png similarity index 100% rename from docs.html.pythonmod/_static/up.png rename to client-unbound/docs.pythonmod/_static/up.png diff --git a/docs.html.pythonmod/_static/websupport.js b/client-unbound/docs.pythonmod/_static/websupport.js similarity index 100% rename from docs.html.pythonmod/_static/websupport.js rename to client-unbound/docs.pythonmod/_static/websupport.js diff --git a/docs.html.pythonmod/examples/example0.html b/client-unbound/docs.pythonmod/examples/example0.html similarity index 100% rename from docs.html.pythonmod/examples/example0.html rename to client-unbound/docs.pythonmod/examples/example0.html diff --git a/docs.html.pythonmod/examples/example1.html b/client-unbound/docs.pythonmod/examples/example1.html similarity index 100% rename from docs.html.pythonmod/examples/example1.html rename to client-unbound/docs.pythonmod/examples/example1.html diff --git a/docs.html.pythonmod/examples/example2.html b/client-unbound/docs.pythonmod/examples/example2.html similarity index 100% rename from docs.html.pythonmod/examples/example2.html rename to client-unbound/docs.pythonmod/examples/example2.html diff --git a/docs.html.pythonmod/examples/example3.html b/client-unbound/docs.pythonmod/examples/example3.html similarity index 100% rename from docs.html.pythonmod/examples/example3.html rename to client-unbound/docs.pythonmod/examples/example3.html diff --git a/docs.html.pythonmod/examples/example4.html b/client-unbound/docs.pythonmod/examples/example4.html similarity index 100% rename from docs.html.pythonmod/examples/example4.html rename to client-unbound/docs.pythonmod/examples/example4.html diff --git a/docs.html.pythonmod/examples/example5.html b/client-unbound/docs.pythonmod/examples/example5.html similarity index 100% rename from docs.html.pythonmod/examples/example5.html rename to client-unbound/docs.pythonmod/examples/example5.html diff --git a/docs.html.pythonmod/examples/example6.html b/client-unbound/docs.pythonmod/examples/example6.html similarity index 100% rename from docs.html.pythonmod/examples/example6.html rename to client-unbound/docs.pythonmod/examples/example6.html diff --git a/docs.html.pythonmod/examples/index.html b/client-unbound/docs.pythonmod/examples/index.html similarity index 100% rename from docs.html.pythonmod/examples/index.html rename to client-unbound/docs.pythonmod/examples/index.html diff --git a/docs.html.pythonmod/genindex.html b/client-unbound/docs.pythonmod/genindex.html similarity index 100% rename from docs.html.pythonmod/genindex.html rename to client-unbound/docs.pythonmod/genindex.html diff --git a/docs.html.pythonmod/index.html b/client-unbound/docs.pythonmod/index.html similarity index 100% rename from docs.html.pythonmod/index.html rename to client-unbound/docs.pythonmod/index.html diff --git a/docs.html.pythonmod/install.html b/client-unbound/docs.pythonmod/install.html similarity index 100% rename from docs.html.pythonmod/install.html rename to client-unbound/docs.pythonmod/install.html diff --git a/docs.html.pythonmod/modules/config.html b/client-unbound/docs.pythonmod/modules/config.html similarity index 100% rename from docs.html.pythonmod/modules/config.html rename to client-unbound/docs.pythonmod/modules/config.html diff --git a/docs.html.pythonmod/modules/env.html b/client-unbound/docs.pythonmod/modules/env.html similarity index 100% rename from docs.html.pythonmod/modules/env.html rename to client-unbound/docs.pythonmod/modules/env.html diff --git a/docs.html.pythonmod/modules/functions.html b/client-unbound/docs.pythonmod/modules/functions.html similarity index 100% rename from docs.html.pythonmod/modules/functions.html rename to client-unbound/docs.pythonmod/modules/functions.html diff --git a/docs.html.pythonmod/modules/index.html b/client-unbound/docs.pythonmod/modules/index.html similarity index 100% rename from docs.html.pythonmod/modules/index.html rename to client-unbound/docs.pythonmod/modules/index.html diff --git a/docs.html.pythonmod/modules/struct.html b/client-unbound/docs.pythonmod/modules/struct.html similarity index 100% rename from docs.html.pythonmod/modules/struct.html rename to client-unbound/docs.pythonmod/modules/struct.html diff --git a/docs.html.pythonmod/objects.inv b/client-unbound/docs.pythonmod/objects.inv similarity index 100% rename from docs.html.pythonmod/objects.inv rename to client-unbound/docs.pythonmod/objects.inv diff --git a/docs.html.pythonmod/search.html b/client-unbound/docs.pythonmod/search.html similarity index 100% rename from docs.html.pythonmod/search.html rename to client-unbound/docs.pythonmod/search.html diff --git a/docs.html.pythonmod/searchindex.js b/client-unbound/docs.pythonmod/searchindex.js similarity index 100% rename from docs.html.pythonmod/searchindex.js rename to client-unbound/docs.pythonmod/searchindex.js diff --git a/docs.html.pythonmod/usecase.html b/client-unbound/docs.pythonmod/usecase.html similarity index 100% rename from docs.html.pythonmod/usecase.html rename to client-unbound/docs.pythonmod/usecase.html diff --git a/examples/pfui_unbound.conf b/client-unbound/examples/pfui_unbound.conf similarity index 100% rename from examples/pfui_unbound.conf rename to client-unbound/examples/pfui_unbound.conf diff --git a/pfui_unbound.py b/client-unbound/pfui_unbound.py similarity index 99% rename from pfui_unbound.py rename to client-unbound/pfui_unbound.py index 2931d64..636302c 100644 --- a/pfui_unbound.py +++ b/client-unbound/pfui_unbound.py @@ -31,7 +31,7 @@ # installer puts the shared pfui/ package beside this config, so derive it from # there: one hardcoded location, same as CONFIG_LOCATION itself. sys.path.insert(0, dirname(CONFIG_LOCATION)) -from pfui.wire import encode_payload, frame # noqa: E402 +from pfui_wire import encode_payload, frame # noqa: E402 from socket import ( AF_INET, diff --git a/pfui_unbound.yml b/client-unbound/pfui_unbound.yml similarity index 100% rename from pfui_unbound.yml rename to client-unbound/pfui_unbound.yml diff --git a/rc.d/openbsd_pfui_unbound b/client-unbound/rc.d/pfui_unbound similarity index 100% rename from rc.d/openbsd_pfui_unbound rename to client-unbound/rc.d/pfui_unbound diff --git a/requirements-unbound.txt b/client-unbound/requirements.txt similarity index 100% rename from requirements-unbound.txt rename to client-unbound/requirements.txt diff --git a/tests/test_unbound.py b/client-unbound/tests/test_unbound.py similarity index 98% rename from tests/test_unbound.py rename to client-unbound/tests/test_unbound.py index 9ad6674..178a2c7 100755 --- a/tests/test_unbound.py +++ b/client-unbound/tests/test_unbound.py @@ -15,7 +15,7 @@ import pytest -from pfui.wire import encode +from pfui_wire import encode HOST = os.environ.get("PFUI_FW_HOST") PORT = int(os.environ.get("PFUI_FW_PORT", 10001)) diff --git a/tests/test_unbound_faults.py b/client-unbound/tests/test_unbound_faults.py similarity index 98% rename from tests/test_unbound_faults.py rename to client-unbound/tests/test_unbound_faults.py index a835288..d7331ae 100755 --- a/tests/test_unbound_faults.py +++ b/client-unbound/tests/test_unbound_faults.py @@ -12,7 +12,7 @@ import lz4.frame import pytest -from pfui.wire import HEADER, MAX_MESSAGE, encode +from pfui_wire import HEADER, MAX_MESSAGE, encode HOST = os.environ.get("PFUI_FW_HOST") PORT = int(os.environ.get("PFUI_FW_PORT", 10001)) diff --git a/tests/test_unbound_module.py b/client-unbound/tests/test_unbound_module.py similarity index 95% rename from tests/test_unbound_module.py rename to client-unbound/tests/test_unbound_module.py index 9d01356..99750d2 100644 --- a/tests/test_unbound_module.py +++ b/client-unbound/tests/test_unbound_module.py @@ -19,7 +19,7 @@ import pytest -REPO = Path(__file__).resolve().parent.parent +COMPONENT = Path(__file__).resolve().parent.parent # Unbound's pythonmod injects these; supply stubs so call-time lookups resolve INJECTED = { @@ -37,9 +37,9 @@ @pytest.fixture(scope="module") def plugin(): - sys.path.insert(0, str(REPO)) + sys.path.insert(0, str(COMPONENT)) spec = importlib.util.spec_from_file_location( - "pfui_unbound_plugin", REPO / "pfui_unbound.py" + "pfui_unbound_plugin", COMPONENT / "pfui_unbound.py" ) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) @@ -60,8 +60,8 @@ def test_module_loads_without_dunder_file(): the fixture above cannot catch a dependency on it; exec the source in a bare namespace instead. """ - source = (REPO / "pfui_unbound.py").read_text() - sys.path.insert(0, str(REPO)) + source = (COMPONENT / "pfui_unbound.py").read_text() + sys.path.insert(0, str(COMPONENT)) namespace = {"__name__": "pfui_unbound_no_file"} # deliberately no __file__ exec(compile(source, "pfui_unbound.py", "exec"), namespace) assert callable(namespace["operate"]) diff --git a/update_dns_blocklist.sh b/client-unbound/tools/update_dns_blocklist.sh similarity index 100% rename from update_dns_blocklist.sh rename to client-unbound/tools/update_dns_blocklist.sh diff --git a/update_root_hints.sh b/client-unbound/tools/update_root_hints.sh similarity index 100% rename from update_root_hints.sh rename to client-unbound/tools/update_root_hints.sh diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..426ba4c --- /dev/null +++ b/conftest.py @@ -0,0 +1,18 @@ +"""Put each component on sys.path the same way it is laid out when installed. + +The installers place the shared pfui_wire module beside each daemon, so both a +daemon and its tests import it as a top-level module. Mirroring that here keeps +the test imports identical to production rather than merely equivalent. +""" + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent + +for path in ( + ROOT / "protocol" / "python", # shared: pfui_wire + ROOT / "server-python", # pfui.store / pfui.validate / pfui.pf_ioctl + ROOT / "client-unbound", +): + sys.path.insert(0, str(path)) diff --git a/pfui_unbound_install.sh b/install-client-unbound.sh similarity index 93% rename from pfui_unbound_install.sh rename to install-client-unbound.sh index 60a6e35..297ed6b 100755 --- a/pfui_unbound_install.sh +++ b/install-client-unbound.sh @@ -140,7 +140,7 @@ echo echo "PFUIDNS: Installing PFUI Python dependencies" # The resolver runs unchrooted (chroot: "" in pfui_unbound.conf), so the module # imports these from the system interpreter's site-packages -python3 -m pip install -r "${DIR}/requirements-unbound.txt" \ +python3 -m pip install -r "${DIR}/client-unbound/requirements.txt" \ || die "cannot install Python dependencies" if [[ "$OS" = "OpenBSD" ]]; then @@ -205,26 +205,25 @@ if [[ "$OS" = "OpenBSD" ]]; then read -p "Would you like to install the example pfui_unbound.yml (existing will be backed up) y/n: " yn if [[ "$yn" = "y" ]]; then [ -f "${TARGET}/pfui_unbound.yml" ] && mv "${TARGET}/pfui_unbound.yml" "${TARGET}/pfui_unbound.yml.${HOUR}" - install -m 644 -o root -g wheel "${DIR}"/pfui_unbound.yml ${TARGET}/pfui_unbound.yml + install -m 644 -o root -g wheel "${DIR}"/client-unbound/pfui_unbound.yml ${TARGET}/pfui_unbound.yml fi echo "Default pfui_unbound config: ${TARGET}/pfui_unbound.yml" # Install PFUI_Unbound module script - install -m 644 -o root -g wheel "${DIR}"/pfui_unbound.py ${TARGET}/pfui_unbound.py - # Shared modules; pfui_unbound.py adds its own directory to sys.path to reach these - install -d -m 755 -o root -g wheel ${TARGET}/pfui - install -m 644 -o root -g wheel "${DIR}"/pfui/__init__.py "${DIR}"/pfui/wire.py ${TARGET}/pfui/ + install -m 644 -o root -g wheel "${DIR}"/client-unbound/pfui_unbound.py ${TARGET}/pfui_unbound.py + # Shared protocol module; pfui_unbound.py adds its own directory to sys.path + install -m 644 -o root -g wheel "${DIR}"/protocol/python/pfui_wire.py ${TARGET}/ # Install PFUI_Unbound RC script # root-owned: rcctl runs this as root, and a file's owner can always chmod it - install -m 555 -o root -g wheel "${DIR}"/rc.d/openbsd_pfui_unbound /etc/rc.d/pfui_unbound + install -m 555 -o root -g wheel "${DIR}"/client-unbound/rc.d/pfui_unbound /etc/rc.d/pfui_unbound echo echo "PFUIDNS: Installing Root Hints and example DNS-BL" [ -f "${TARGET}/update_root_hints.sh" ] && mv "${TARGET}/update_root_hints.sh" "${TARGET}/update_root_hints.sh.${HOUR}" # root-owned: these run from cron with the privilege to write /var/unbound and restart the service - install -m 755 -o root -g wheel "${DIR}"/update_root_hints.sh ${TARGET}/update_root_hints.sh + install -m 755 -o root -g wheel "${DIR}"/client-unbound/tools/update_root_hints.sh ${TARGET}/update_root_hints.sh [ -f "${TARGET}/update_dns_blocklist.sh" ] && mv "${TARGET}/update_dns_blocklist.sh" "${TARGET}/update_dns_blocklist.sh.${HOUR}" - install -m 755 -o root -g wheel "${DIR}"/update_dns_blocklist.sh ${TARGET}/update_dns_blocklist.sh + install -m 755 -o root -g wheel "${DIR}"/client-unbound/tools/update_dns_blocklist.sh ${TARGET}/update_dns_blocklist.sh echo "New scripts: ${TARGET}/update_root_hints.sh, ${TARGET}/update_dns_blocklist.sh" # Install Unbound example configuration with PFUI_Unbound enabled @@ -233,7 +232,7 @@ if [[ "$OS" = "OpenBSD" ]]; then if [[ "$yn" = "y" ]]; then echo "Installing example ${TARGET}/pfui_unbound.conf" [ -f "${TARGET}/pfui_unbound.conf" ] && mv "${TARGET}/pfui_unbound.conf" "${TARGET}/pfui_unbound.conf.${HOUR}" - install -m 644 -o root -g wheel "${DIR}/examples/pfui_unbound.conf" \ + install -m 644 -o root -g wheel "${DIR}/client-unbound/examples/pfui_unbound.conf" \ "${TARGET}/pfui_unbound.conf" || die "cannot install pfui_unbound.conf" fi echo "Default pfui_unbound config: ${TARGET}/pfui_unbound.conf" diff --git a/pfui_firewall_install.sh b/install-server-python.sh similarity index 85% rename from pfui_firewall_install.sh rename to install-server-python.sh index 25b7ae1..c2f9dc5 100755 --- a/pfui_firewall_install.sh +++ b/install-server-python.sh @@ -74,24 +74,28 @@ if [[ "$OS" = "OpenBSD" ]]; then # Not a virtualenv: the daemon runs under /usr/local/bin/python3 (see its # shebang), so the dependencies have to be importable there. The previous # venv was never wired into the daemon and was the only place these landed. - python3 -m pip install -r "${DIR}/requirements-firewall.txt" \ + python3 -m pip install -r "${DIR}/server-python/requirements.txt" \ || die "cannot install Python dependencies" echo "PFUIFW: Installing PFUI Firewall Service (will backup any existing pfui_firewall configuration)" - install -m 755 -o root -g wheel "${DIR}"/pfui_firewall.py /usr/local/sbin/pfui_firewall \ + install -m 755 -o root -g wheel "${DIR}"/server-python/pfui_firewall.py /usr/local/sbin/pfui_firewall \ || die "cannot install the daemon" # Shared modules must sit beside the daemon; a script's own directory is sys.path[0] install -d -m 755 -o root -g wheel /usr/local/sbin/pfui - install -m 644 -o root -g wheel "${DIR}"/pfui/__init__.py "${DIR}"/pfui/wire.py \ - "${DIR}"/pfui/store.py "${DIR}"/pfui/validate.py /usr/local/sbin/pfui/ \ + # Shared protocol module sits beside the daemon, as a top-level module + install -m 644 -o root -g wheel "${DIR}"/protocol/python/pfui_wire.py \ + /usr/local/sbin/ || die "cannot install the shared wire module" + install -m 644 -o root -g wheel "${DIR}"/server-python/pfui/__init__.py \ + "${DIR}"/server-python/pfui/store.py "${DIR}"/server-python/pfui/validate.py \ + "${DIR}"/server-python/pfui/pf_ioctl.py /usr/local/sbin/pfui/ \ || die "cannot install the pfui modules" [ -f /etc/pfui_firewall.yml ] && cp -p /etc/pfui_firewall.yml "/etc/pfui_firewall.yml.${HOUR}" # root-owned: the daemon only reads this, and CTL: PFCTL makes it a command source - install -m 644 -o root -g wheel "${DIR}"/pfui_firewall.yml /etc/pfui_firewall.yml + install -m 644 -o root -g wheel "${DIR}"/server-python/pfui_firewall.yml /etc/pfui_firewall.yml echo "PFUIFW: PFUI_Firewall default configuration file located at '/etc/pfui_firewall.yml' (please configure)" # root-owned: rcctl runs this as root, and a file's owner can always chmod it - install -m 555 -o root -g wheel "${DIR}"/rc.d/pfui_firewall /etc/rc.d/pfui_firewall \ + install -m 555 -o root -g wheel "${DIR}"/server-python/rc.d/pfui_firewall /etc/rc.d/pfui_firewall \ || die "cannot install the rc.d script" install -m 644 -o root -g wheel "${DIR}"/examples/pf.conf /etc/pf-pfui-example.conf diff --git a/protocol/PROTOCOL.md b/protocol/PROTOCOL.md new file mode 100644 index 0000000..3746c30 --- /dev/null +++ b/protocol/PROTOCOL.md @@ -0,0 +1,144 @@ +# PFUI wire protocol + +Normative description of what a PFUI client sends to a PFUI server. Any client +(`client-unbound/`, or a future client for another resolver) and any server +(`server-python/`, `server-c/`) must agree with this document, and both are +tested against the shared vectors in `vectors/`. + +Version: 1. There is no version field on the wire; see +[Compatibility](#compatibility). + +## Transport + +TCP is the supported transport. UDP exists for lab use and is disabled unless +the server sets `ALLOW_INSECURE_UDP`, because a datagram source address is not +verified and the protocol has no authentication. + +Neither transport is authenticated or encrypted. An IP must reach a PF table +microseconds before the client connects to it, and a handshake would spend that +budget. Access control is therefore the packet filter's job: restrict the +server's listening port to the known resolvers. + +## Framing + +A TCP message is exactly one frame: + +``` + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| length (uint32, BE) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| payload (length bytes) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +``` + +- `length` is the payload byte count, big-endian (network byte order), not + including the 4 header bytes. +- A receiver MUST read exactly 4 bytes, then exactly `length` bytes. +- A receiver MUST reject `length == 0` and `length > MAX_MESSAGE` + (1048576, 1 MiB) **before** buffering any payload, and MUST close the + connection. +- A receiver MUST NOT scan the payload for a terminator. The payload is + arbitrary binary and may contain any byte sequence, including the `EOT` that + earlier releases used as a footer. +- Nothing follows the payload. One connection carries one frame. +- If the peer closes before a complete 4-byte header arrives, there is no + message. That is not a protocol error: the receiver closes the connection and + carries on. A payload that stops short of `length`, by contrast, IS an error, + because the sender declared bytes it did not send. + +UDP datagrams are self-delimiting and carry the payload **alone**, with no +length prefix. + +## Payload + +The payload is UTF-8 JSON, optionally compressed with lz4 frame format. Whether +compression is in use is configuration on both sides (`COMPRESS`), not signalled +on the wire: a mismatch is a misconfiguration and shows up as a decode failure. + +A receiver MUST bound decompression to `MAX_MESSAGE` and abort while expanding, +not after: a small frame can otherwise expand to gigabytes before any size check +runs. + +### Message object + +```json +{ + "kind": "rr", + "AF4": [{"ip": "8.8.8.8", "ttl": 3600, "qname": "example.com."}], + "AF6": [{"ip": "2001:4860:4860::8888", "ttl": 3600, "qname": "example.com."}] +} +``` + +| Field | Type | Meaning | +|-------|------|---------| +| `kind` | `"rr"` or `"cache"` | How to read every `ttl` in this message. Required. | +| `AF4` | array | IPv4 records. May be empty or absent. | +| `AF6` | array | IPv6 records. May be empty or absent. | +| `AF*[].ip` | string | Address in presentation form. | +| `AF*[].ttl` | integer | See `kind`. `0` is valid and means do-not-cache. | +| `AF*[].qname` | string | Query name, for logging. Optional. | + +`kind` is what removes the guesswork that used to come from TTL magnitude: + +- `"rr"` — `ttl` is a **relative** DNS TTL in seconds, read from a fresh + resolution. The server expires the entry at + `received_at + ttl * TTL_MULTIPLIER`. +- `"cache"` — `ttl` is an **absolute** unix timestamp, because the resolver + reported a cache entry's expiry rather than a countdown. The server expires + the entry when that timestamp passes. + +A server MUST reject a message with a missing or unrecognised `kind` rather than +infer one. A server MUST accept a message whose `AF4`/`AF6` are both empty only +by rejecting it as having nothing to act on. + +### Server obligations for records + +A server MUST NOT install a non-global address. Sentinels (`0.0.0.0`, `::` and +every other spelling of them), private, loopback, link-local, CGNAT, ULA, +IPv4-mapped and multicast addresses MUST be dropped: a DNS answer pointing +inside the network must not authorise egress. A server SHOULD canonicalise +addresses before use, so its own state and `pfctl -T show` cannot disagree about +IPv6 spelling. + +## Replies + +| Reply | Meaning | +|-------|---------| +| `ACKDATA` | UDP only. The datagram decoded to a valid message. Sent **after** validation, never on receipt. | +| `ACKUPDATE` | The PF tables have been updated. The client may release the DNS answer. | +| any other | Refusal, with a short human-readable reason (`Missing kind`, `Bad frame`, `Bad length`, `Truncated`, `Failed to decode`, `Invalid datatype`, `Empty payload`, `Socket timeout`). | + +A client SHOULD treat anything other than `ACKUPDATE` as a failed update and log +the reason, which is what a version skew looks like from the client side. + +## Compatibility + +There is no version negotiation. The framing change from the `EOT` footer, and +the addition of `kind`, are both breaking: a mismatched pair fails closed in +either direction, so client and server must be deployed from the same release. + +If a future change needs negotiation, add a field to the JSON payload rather +than the frame header; the framing layer is deliberately dumb. + +## Conformance vectors + +`vectors/framing.tsv` is language-neutral and covers the framing layer only, so +an implementation that has no JSON parser yet can still be checked. Columns: + +| Column | Meaning | +|--------|---------| +| `name` | Identifier for the case | +| `frame_hex` | Bytes on the wire, hex | +| `expect` | `ok`, `short`, or a rejection reason (`length`, `truncated`) | +| `payload_hex` | Expected payload for `ok`, else `-` | + +Fields are tab-separated and never empty: `-` stands for empty or not +applicable, so a reader that collapses runs of whitespace cannot silently shift +columns. + +`vectors/messages.json` covers the payload layer: each entry has the decoded +message and, where the encoding is byte-stable, the exact frame. Compressed +frames are round-trip only, because lz4 output is not guaranteed identical +across library versions. diff --git a/protocol/README.md b/protocol/README.md new file mode 100644 index 0000000..dcc87f4 --- /dev/null +++ b/protocol/README.md @@ -0,0 +1,23 @@ +# PFUI protocol + +What every PFUI client and server must agree on. + +- [PROTOCOL.md](PROTOCOL.md) — the normative specification +- `vectors/framing.tsv` — framing conformance vectors, language-neutral +- `vectors/messages.json` — payload conformance vectors +- `python/pfui_wire.py` — reference implementation, used by both + `client-unbound/` and `server-python/` + +Only code that both a client and a server need lives here. Server-only logic +(expiry, address validation, PF ioctl) belongs to the server, and the C framing +implementation lives with `server-c/` because nothing else uses it. + +Both implementations run `vectors/framing.tsv`: + +``` +python -m pytest protocol/python/tests +make -C server-c test +``` + +A change to the framing or the message object means changing PROTOCOL.md, the +vectors, and every implementation together. diff --git a/pfui/wire.py b/protocol/python/pfui_wire.py similarity index 100% rename from pfui/wire.py rename to protocol/python/pfui_wire.py diff --git a/protocol/python/tests/test_vectors.py b/protocol/python/tests/test_vectors.py new file mode 100644 index 0000000..7714109 --- /dev/null +++ b/protocol/python/tests/test_vectors.py @@ -0,0 +1,61 @@ +"""Conformance tests against the shared vectors in ../../vectors/. + +The same framing.tsv is read by server-c's test suite, so a divergence between +the two implementations fails here or there rather than in production. +""" + +import json +from pathlib import Path + +import pytest + +from pfui_wire import MAX_MESSAGE, WireError, decode_stream, encode + +VECTORS = Path(__file__).resolve().parents[3] / "protocol" / "vectors" + + +def framing_rows(): + rows = [] + for line in (VECTORS / "framing.tsv").read_text().splitlines(): + if not line or line.startswith("#"): + continue + name, frame_hex, expect, payload_hex = line.split("\t") + # "-" means empty or not applicable; the format has no empty fields, so + # that a whitespace-splitting reader cannot silently shift columns + blob = b"" if frame_hex == "-" else bytes.fromhex(frame_hex) + rows.append((name, blob, expect, payload_hex)) + return rows + + +def message_vectors(): + return json.loads((VECTORS / "messages.json").read_text())["vectors"] + + +@pytest.mark.parametrize( + "name,blob,expect,payload_hex", framing_rows(), ids=lambda v: v if isinstance(v, str) else "" +) +def test_framing_vector(name, blob, expect, payload_hex): + if expect == "ok": + # This layer is about bytes, not JSON: check the declared length and the + # payload the framing yields + assert int.from_bytes(blob[:4], "big") == len(blob) - 4 + assert blob[4:].hex() == payload_hex + elif expect == "short": + # No complete header arrived, so there is no message; not an error + assert decode_stream(blob, compress=False) is None + else: + with pytest.raises(WireError): + decode_stream(blob, compress=False) + + +@pytest.mark.parametrize("vector", message_vectors(), ids=lambda v: v["name"]) +def test_message_vector(vector): + msg = vector["message"] + if vector.get("byte_exact"): + assert encode(msg, compress=vector["compress"]).hex() == vector["frame_hex"] + blob = encode(msg, compress=vector["compress"]) + assert decode_stream(blob, compress=vector["compress"]) == msg + + +def test_vectors_agree_on_max_message(): + assert json.loads((VECTORS / "messages.json").read_text())["max_message"] == MAX_MESSAGE diff --git a/tests/test_wire.py b/protocol/python/tests/test_wire.py similarity index 99% rename from tests/test_wire.py rename to protocol/python/tests/test_wire.py index 778e770..4368fd8 100644 --- a/tests/test_wire.py +++ b/protocol/python/tests/test_wire.py @@ -3,7 +3,7 @@ import lz4.frame import pytest -from pfui.wire import ( +from pfui_wire import ( HEADER, MAX_MESSAGE, WireError, diff --git a/protocol/vectors/framing.tsv b/protocol/vectors/framing.tsv new file mode 100644 index 0000000..21560fe --- /dev/null +++ b/protocol/vectors/framing.tsv @@ -0,0 +1,12 @@ +# PFUI framing conformance vectors. See ../PROTOCOL.md +# name frame_hex expect payload_hex +minimal_payload 000000027b7d ok 7b7d +json_uncompressed 0000001f7b226b696e64223a227272222c22414634223a5b5d2c22414636223a5b5d7d ok 7b226b696e64223a227272222c22414634223a5b5d2c22414636223a5b5d7d +payload_containing_eot 0000000b7b2271223a22454f54227d ok 7b2271223a22454f54227d +payload_all_zero_bytes 0000001000000000000000000000000000000000 ok 00000000000000000000000000000000 +payload_high_bytes 00000010f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff ok f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff +length_zero 00000000 length - +length_over_max 0010000178787878 length - +header_short 000001 short - +payload_truncated 00000008616263 truncated - +empty_input - short - diff --git a/protocol/vectors/messages.json b/protocol/vectors/messages.json new file mode 100644 index 0000000..ac32e09 --- /dev/null +++ b/protocol/vectors/messages.json @@ -0,0 +1,173 @@ +{ + "note": "frame_hex is byte-exact only for uncompressed payloads; compressed frames are round-trip only", + "max_message": 1048576, + "vectors": [ + { + "name": "rr_single_a", + "message": { + "kind": "rr", + "AF4": [ + { + "ip": "8.8.8.8", + "ttl": 3600, + "qname": "example.com." + } + ], + "AF6": [] + }, + "compress": false, + "byte_exact": true, + "frame_hex": "0000005b7b226b696e64223a20227272222c2022414634223a205b7b226970223a2022382e382e382e38222c202274746c223a20333630302c2022716e616d65223a20226578616d706c652e636f6d2e227d5d2c2022414636223a205b5d7d" + }, + { + "name": "rr_single_a_lz4", + "message": { + "kind": "rr", + "AF4": [ + { + "ip": "8.8.8.8", + "ttl": 3600, + "qname": "example.com." + } + ], + "AF6": [] + }, + "compress": true, + "byte_exact": false + }, + { + "name": "rr_dual_stack", + "message": { + "kind": "rr", + "AF4": [ + { + "ip": "1.1.1.1", + "ttl": 300 + } + ], + "AF6": [ + { + "ip": "2001:4860:4860::8888", + "ttl": 300 + } + ] + }, + "compress": false, + "byte_exact": true, + "frame_hex": "0000006b7b226b696e64223a20227272222c2022414634223a205b7b226970223a2022312e312e312e31222c202274746c223a203330307d5d2c2022414636223a205b7b226970223a2022323030313a343836303a343836303a3a38383838222c202274746c223a203330307d5d7d" + }, + { + "name": "rr_dual_stack_lz4", + "message": { + "kind": "rr", + "AF4": [ + { + "ip": "1.1.1.1", + "ttl": 300 + } + ], + "AF6": [ + { + "ip": "2001:4860:4860::8888", + "ttl": 300 + } + ] + }, + "compress": true, + "byte_exact": false + }, + { + "name": "cache_absolute_expiry", + "message": { + "kind": "cache", + "AF4": [ + { + "ip": "1.0.0.1", + "ttl": 1675846753 + } + ], + "AF6": [] + }, + "compress": false, + "byte_exact": true, + "frame_hex": "0000004b7b226b696e64223a20226361636865222c2022414634223a205b7b226970223a2022312e302e302e31222c202274746c223a20313637353834363735337d5d2c2022414636223a205b5d7d" + }, + { + "name": "cache_absolute_expiry_lz4", + "message": { + "kind": "cache", + "AF4": [ + { + "ip": "1.0.0.1", + "ttl": 1675846753 + } + ], + "AF6": [] + }, + "compress": true, + "byte_exact": false + }, + { + "name": "rr_zero_ttl", + "message": { + "kind": "rr", + "AF4": [ + { + "ip": "8.8.4.4", + "ttl": 0 + } + ], + "AF6": [] + }, + "compress": false, + "byte_exact": true, + "frame_hex": "0000003f7b226b696e64223a20227272222c2022414634223a205b7b226970223a2022382e382e342e34222c202274746c223a20307d5d2c2022414636223a205b5d7d" + }, + { + "name": "rr_zero_ttl_lz4", + "message": { + "kind": "rr", + "AF4": [ + { + "ip": "8.8.4.4", + "ttl": 0 + } + ], + "AF6": [] + }, + "compress": true, + "byte_exact": false + }, + { + "name": "rr_long_ttl_not_a_timestamp", + "message": { + "kind": "rr", + "AF4": [ + { + "ip": "8.8.8.8", + "ttl": 604800 + } + ], + "AF6": [] + }, + "compress": false, + "byte_exact": true, + "frame_hex": "000000447b226b696e64223a20227272222c2022414634223a205b7b226970223a2022382e382e382e38222c202274746c223a203630343830307d5d2c2022414636223a205b5d7d" + }, + { + "name": "rr_long_ttl_not_a_timestamp_lz4", + "message": { + "kind": "rr", + "AF4": [ + { + "ip": "8.8.8.8", + "ttl": 604800 + } + ], + "AF6": [] + }, + "compress": true, + "byte_exact": false + } + ] +} diff --git a/server-c/Makefile b/server-c/Makefile new file mode 100644 index 0000000..ed5e2d6 --- /dev/null +++ b/server-c/Makefile @@ -0,0 +1,27 @@ +# PFUI_Firewall, C implementation. Framing layer only so far; see README.md. + +CC ?= cc +CFLAGS ?= -std=c99 -O2 -Wall -Wextra -Werror -pedantic +CPPFLAGS += -Iinclude +BUILD := build +VECTORS := ../protocol/vectors/framing.tsv + +.PHONY: all test clean + +all: $(BUILD)/test_vectors + +$(BUILD): + mkdir -p $(BUILD) + +$(BUILD)/pfui_wire.o: src/pfui_wire.c include/pfui_wire.h | $(BUILD) + $(CC) $(CFLAGS) $(CPPFLAGS) -c src/pfui_wire.c -o $@ + +$(BUILD)/test_vectors: tests/test_vectors.c $(BUILD)/pfui_wire.o + $(CC) $(CFLAGS) $(CPPFLAGS) tests/test_vectors.c $(BUILD)/pfui_wire.o -o $@ + +# Runs the same vectors as protocol/python/tests/test_vectors.py +test: $(BUILD)/test_vectors + $(BUILD)/test_vectors $(VECTORS) + +clean: + rm -rf $(BUILD) diff --git a/server-c/README.md b/server-c/README.md new file mode 100644 index 0000000..9a610ab --- /dev/null +++ b/server-c/README.md @@ -0,0 +1,39 @@ +# PFUI_Firewall (C) + +A C implementation of the PFUI server, intended to replace the Python daemon on +the firewall itself. The Python version wraps the PF ioctl structures through +ctypes, so the kernel interface is the part that most wants to be C. + +## Status + +Framing only. `src/pfui_wire.c` implements the length-prefixed frame described in +[../protocol/PROTOCOL.md](../protocol/PROTOCOL.md), and nothing else exists yet: +no listener, no JSON parsing, no Redis, no PF ioctl. + +It is here now so the protocol spec has a second implementation checking it. The +framing vectors in `../protocol/vectors/framing.tsv` are run by both this suite +and the Python one, so the two cannot drift apart unnoticed. + +## Build and test + +``` +make test +``` + +Requires only a C99 compiler. The test binary takes the vector file as its first +argument, defaulting to the shared copy. + +## Still to write + +Roughly in dependency order: + +- JSON payload parsing, and lz4 decompression bounded to `PFUI_MAX_MESSAGE` +- address validation (globally routable unicast only, canonicalised) +- PF table ioctl: `DIOCRADDADDRS`, `DIOCRDELADDRS`, and `DIOCRGETADDRS`, which + the Python server never implemented and currently shells out to `pfctl` for +- expiry state, whether in Redis or in-process +- persist-file writer with the same locking discipline as the Python version +- listener with bounded concurrency, privilege drop, and an rc.d script + +`../server-python/` remains the reference for all of the above, and +`../protocol/PROTOCOL.md` is normative where the two disagree. diff --git a/server-c/include/pfui_wire.h b/server-c/include/pfui_wire.h new file mode 100644 index 0000000..b4cd7cb --- /dev/null +++ b/server-c/include/pfui_wire.h @@ -0,0 +1,47 @@ +/* + * PFUI wire framing. See ../../protocol/PROTOCOL.md for the normative spec and + * ../../protocol/vectors/framing.tsv for the shared conformance vectors. + * + * This is the framing layer only: it hands back the payload as opaque bytes and + * takes no view on JSON or lz4. + */ + +#ifndef PFUI_WIRE_H +#define PFUI_WIRE_H + +#include +#include + +#define PFUI_HEADER_LEN 4u +#define PFUI_MAX_MESSAGE (1u << 20) /* 1 MiB, must match the Python side */ + +typedef enum { + PFUI_OK = 0, + PFUI_SHORT, /* no complete header yet; peer closed or more to come */ + PFUI_BAD_LENGTH,/* declared length is zero or above PFUI_MAX_MESSAGE */ + PFUI_TRUNCATED, /* header is complete but the payload is not */ + PFUI_NOSPACE /* caller's buffer cannot hold the frame */ +} pfui_status; + +/* + * Read the declared payload length from a 4-byte big-endian header. Rejects the + * length before a caller has any reason to allocate for it. + */ +pfui_status pfui_header_decode(const uint8_t *buf, size_t len, uint32_t *out_len); + +/* + * Point *payload at the payload inside a complete frame. No copy is made, so + * *payload remains valid only as long as buf does. + */ +pfui_status pfui_frame_decode(const uint8_t *buf, size_t len, + const uint8_t **payload, uint32_t *payload_len); + +/* + * Write header + payload into out. Returns the total bytes written via *written. + */ +pfui_status pfui_frame_encode(uint8_t *out, size_t out_cap, const uint8_t *payload, + uint32_t payload_len, size_t *written); + +const char *pfui_status_str(pfui_status status); + +#endif /* PFUI_WIRE_H */ diff --git a/server-c/src/pfui_wire.c b/server-c/src/pfui_wire.c new file mode 100644 index 0000000..01e545e --- /dev/null +++ b/server-c/src/pfui_wire.c @@ -0,0 +1,85 @@ +#include "pfui_wire.h" + +#include + +pfui_status +pfui_header_decode(const uint8_t *buf, size_t len, uint32_t *out_len) +{ + uint32_t declared; + + if (buf == NULL || out_len == NULL) + return PFUI_SHORT; + if (len < PFUI_HEADER_LEN) + return PFUI_SHORT; + + declared = ((uint32_t)buf[0] << 24) | ((uint32_t)buf[1] << 16) | + ((uint32_t)buf[2] << 8) | (uint32_t)buf[3]; + + /* Checked before the caller reads or allocates the payload */ + if (declared == 0 || declared > PFUI_MAX_MESSAGE) + return PFUI_BAD_LENGTH; + + *out_len = declared; + return PFUI_OK; +} + +pfui_status +pfui_frame_decode(const uint8_t *buf, size_t len, const uint8_t **payload, + uint32_t *payload_len) +{ + uint32_t declared; + pfui_status status; + + if (payload == NULL || payload_len == NULL) + return PFUI_SHORT; + + status = pfui_header_decode(buf, len, &declared); + if (status != PFUI_OK) + return status; + + if (len - PFUI_HEADER_LEN < declared) + return PFUI_TRUNCATED; + + *payload = buf + PFUI_HEADER_LEN; + *payload_len = declared; + return PFUI_OK; +} + +pfui_status +pfui_frame_encode(uint8_t *out, size_t out_cap, const uint8_t *payload, + uint32_t payload_len, size_t *written) +{ + if (out == NULL || payload == NULL || written == NULL) + return PFUI_NOSPACE; + if (payload_len == 0 || payload_len > PFUI_MAX_MESSAGE) + return PFUI_BAD_LENGTH; + if (out_cap < (size_t)payload_len + PFUI_HEADER_LEN) + return PFUI_NOSPACE; + + out[0] = (uint8_t)(payload_len >> 24); + out[1] = (uint8_t)(payload_len >> 16); + out[2] = (uint8_t)(payload_len >> 8); + out[3] = (uint8_t)(payload_len); + memcpy(out + PFUI_HEADER_LEN, payload, payload_len); + + *written = (size_t)payload_len + PFUI_HEADER_LEN; + return PFUI_OK; +} + +const char * +pfui_status_str(pfui_status status) +{ + switch (status) { + case PFUI_OK: + return "ok"; + case PFUI_SHORT: + return "short"; + case PFUI_BAD_LENGTH: + return "length"; + case PFUI_TRUNCATED: + return "truncated"; + case PFUI_NOSPACE: + return "nospace"; + } + return "unknown"; +} diff --git a/server-c/tests/test_vectors.c b/server-c/tests/test_vectors.c new file mode 100644 index 0000000..1adfd95 --- /dev/null +++ b/server-c/tests/test_vectors.c @@ -0,0 +1,124 @@ +/* + * Runs the shared framing vectors from ../../protocol/vectors/framing.tsv. + * + * The Python implementation runs the same file, so a divergence between the two + * fails here or in protocol/python/tests/test_vectors.py, not in production. + */ + +#include "pfui_wire.h" + +#include +#include +#include + +#define MAX_HEX (2 * (PFUI_MAX_MESSAGE + PFUI_HEADER_LEN) + 1) + +static int failures; +static int checks; + +static size_t +unhex(const char *hex, uint8_t *out, size_t out_cap) +{ + size_t len = strlen(hex), i; + + if (strcmp(hex, "-") == 0 || len % 2 != 0 || len / 2 > out_cap) + return 0; + for (i = 0; i < len / 2; i++) { + unsigned int byte; + if (sscanf(hex + 2 * i, "%2x", &byte) != 1) + return 0; + out[i] = (uint8_t)byte; + } + return len / 2; +} + +static void +check(const char *name, int condition, const char *detail) +{ + checks++; + if (!condition) { + failures++; + fprintf(stderr, "FAIL %s: %s\n", name, detail); + } +} + +static void +run_vector(const char *name, const char *frame_hex, const char *expect, + const char *payload_hex) +{ + static uint8_t frame[PFUI_MAX_MESSAGE + PFUI_HEADER_LEN]; + static uint8_t want[PFUI_MAX_MESSAGE]; + const uint8_t *payload = NULL; + uint32_t payload_len = 0; + size_t frame_len, want_len; + pfui_status status; + + frame_len = strcmp(frame_hex, "-") == 0 ? 0 : unhex(frame_hex, frame, sizeof(frame)); + status = pfui_frame_decode(frame, frame_len, &payload, &payload_len); + + if (strcmp(expect, "ok") == 0) { + check(name, status == PFUI_OK, pfui_status_str(status)); + if (status != PFUI_OK) + return; + want_len = unhex(payload_hex, want, sizeof(want)); + check(name, payload_len == want_len, "payload length mismatch"); + check(name, payload_len == want_len && + memcmp(payload, want, want_len) == 0, "payload bytes mismatch"); + + /* Re-encoding the payload must reproduce the frame byte for byte */ + { + static uint8_t round[PFUI_MAX_MESSAGE + PFUI_HEADER_LEN]; + size_t written = 0; + status = pfui_frame_encode(round, sizeof(round), payload, + payload_len, &written); + check(name, status == PFUI_OK, "re-encode failed"); + check(name, written == frame_len && + memcmp(round, frame, frame_len) == 0, "round trip differs"); + } + } else { + check(name, strcmp(pfui_status_str(status), expect) == 0, + pfui_status_str(status)); + } +} + +int +main(int argc, char **argv) +{ + const char *path = argc > 1 ? argv[1] : "../protocol/vectors/framing.tsv"; + char line[MAX_HEX + 256]; + FILE *fp; + + fp = fopen(path, "r"); + if (fp == NULL) { + fprintf(stderr, "cannot open vectors: %s\n", path); + return 2; + } + + while (fgets(line, sizeof(line), fp) != NULL) { + char *name, *frame_hex, *expect, *payload_hex, *newline; + + if (line[0] == '#' || line[0] == '\n') + continue; + newline = strchr(line, '\n'); + if (newline != NULL) + *newline = '\0'; + + name = strtok(line, "\t"); + frame_hex = strtok(NULL, "\t"); + expect = strtok(NULL, "\t"); + payload_hex = strtok(NULL, "\t"); + if (name == NULL || frame_hex == NULL || expect == NULL || + payload_hex == NULL) { + fprintf(stderr, "malformed vector line\n"); + failures++; + continue; + } + if (frame_hex[0] == '\0') + frame_hex = "-"; + run_vector(name, frame_hex, expect, payload_hex); + } + fclose(fp); + + printf("%d checks, %d failures\n", checks, failures); + return failures == 0 ? 0 : 1; +} diff --git a/server-python/README.md b/server-python/README.md new file mode 100644 index 0000000..5c8ed41 --- /dev/null +++ b/server-python/README.md @@ -0,0 +1,22 @@ +# PFUI_Firewall (Python) + +The PFUI server: a daemon for OpenBSD PF firewalls that receives resolved IPs +from the clients and installs them into PF tables via the `/dev/pf` ioctl +interface, tracking expiry in Redis and mirroring the tables to persist files. + +| Path | What it is | +|------|------------| +| `pfui_firewall.py` | The daemon (`start`/`stop`/`kill`/`restart`/`check`) | +| `pfui_firewall.yml` | Configuration, installed to `/etc/pfui_firewall.yml` | +| `pfui/pf_ioctl.py` | ctypes PF structures, `DIOCRADDADDRS`/`DIOCRDELADDRS`, `pfctl` fallback | +| `pfui/store.py` | Redis expiry decision | +| `pfui/validate.py` | Ingress address validation | +| `rc.d/pfui_firewall` | OpenBSD rc.d script | + +Install with `../install-server-python.sh` from the repository root. The wire +format is specified in [../protocol/PROTOCOL.md](../protocol/PROTOCOL.md); the +installer places `pfui_wire.py` beside the daemon, and the `pfui` package holds +only what is server-specific. + +`../server-c/` is a second implementation of the same protocol, currently +framing only. Where the two disagree, the protocol spec is normative. diff --git a/pfui/__init__.py b/server-python/pfui/__init__.py similarity index 100% rename from pfui/__init__.py rename to server-python/pfui/__init__.py diff --git a/pfui/pf_ioctl.py b/server-python/pfui/pf_ioctl.py similarity index 100% rename from pfui/pf_ioctl.py rename to server-python/pfui/pf_ioctl.py diff --git a/pfui/store.py b/server-python/pfui/store.py similarity index 100% rename from pfui/store.py rename to server-python/pfui/store.py diff --git a/pfui/validate.py b/server-python/pfui/validate.py similarity index 100% rename from pfui/validate.py rename to server-python/pfui/validate.py diff --git a/pfui_firewall.py b/server-python/pfui_firewall.py similarity index 99% rename from pfui_firewall.py rename to server-python/pfui_firewall.py index 55f92e6..0a1705e 100644 --- a/pfui_firewall.py +++ b/server-python/pfui_firewall.py @@ -47,7 +47,7 @@ from pfui.pf_ioctl import table_pop, table_push from pfui.store import expired_keys from pfui.validate import extract -from pfui.wire import MAX_MESSAGE, WireError, decode, read_frame +from pfui_wire import MAX_MESSAGE, WireError, decode, read_frame CONFIG_LOCATION = "/etc/pfui_firewall.yml" diff --git a/pfui_firewall.yml b/server-python/pfui_firewall.yml similarity index 100% rename from pfui_firewall.yml rename to server-python/pfui_firewall.yml diff --git a/rc.d/pfui_firewall b/server-python/rc.d/pfui_firewall similarity index 100% rename from rc.d/pfui_firewall rename to server-python/rc.d/pfui_firewall diff --git a/requirements-firewall.txt b/server-python/requirements.txt similarity index 100% rename from requirements-firewall.txt rename to server-python/requirements.txt diff --git a/tests/test_expiry.py b/server-python/tests/test_expiry.py similarity index 100% rename from tests/test_expiry.py rename to server-python/tests/test_expiry.py diff --git a/tests/test_file_store.py b/server-python/tests/test_file_store.py similarity index 96% rename from tests/test_file_store.py rename to server-python/tests/test_file_store.py index 5235072..dc6960c 100644 --- a/tests/test_file_store.py +++ b/server-python/tests/test_file_store.py @@ -9,12 +9,12 @@ import pytest -REPO = Path(__file__).resolve().parent.parent +COMPONENT = Path(__file__).resolve().parent.parent def _load_daemon(): """Import pfui_firewall without its OpenBSD-only runtime deps.""" - sys.path.insert(0, str(REPO)) + sys.path.insert(0, str(COMPONENT)) for missing in ("redis", "service"): if missing not in sys.modules: module = type(sys)(missing) @@ -25,7 +25,7 @@ def _load_daemon(): module.find_syslog = lambda: None sys.modules[missing] = module spec = importlib.util.spec_from_file_location( - "pfui_firewall", REPO / "pfui_firewall.py" + "pfui_firewall", COMPONENT / "pfui_firewall.py" ) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) diff --git a/tests/test_ioctl.py b/server-python/tests/test_ioctl.py similarity index 100% rename from tests/test_ioctl.py rename to server-python/tests/test_ioctl.py diff --git a/tests/test_pf_structs.py b/server-python/tests/test_pf_structs.py similarity index 100% rename from tests/test_pf_structs.py rename to server-python/tests/test_pf_structs.py diff --git a/tests/test_validate.py b/server-python/tests/test_validate.py similarity index 100% rename from tests/test_validate.py rename to server-python/tests/test_validate.py