Rewrite answer generator code to be more like the TS version

.coveragerc

@@ -3,7 +3,7 @@
 [run]
 branch = true
 parallel = true
-source = test
+source = src
 
 [report]
 sort = cover

.env-template

@@ -0,0 +1,25 @@
+#######################
+# openAI setup
+#######################
+#OPENAI_API_KEY=...
+#OPENAI_MODEL=gpt-4o
+
+#######################
+# Azure OpenAI setup
+#######################
+# gpt-4o
+#AZURE_OPENAI_API_KEY=...
+#AZURE_OPENAI_ENDPOINT="..."
+# text-embedding
+#AZURE_OPENAI_API_KEY_EMBEDDING=...
+#AZURE_OPENAI_ENDPOINT_EMBEDDING="..."
+
+#######################
+# Outlook mail dump (tools/mail/outlook_dump.py)
+#######################
+# User email address for login (pre-fills the sign-in page)
+OUTLOOK_CLIENT_ID=user@email.com
+# Azure AD app registration client ID (GUID)
+OUTLOOK_APPLICATION_CLIENT_ID=...
+# Azure AD tenant ID (GUID)
+OUTLOOK_TENANT_ID=...

.github/workflows/ci.yml

@@ -4,7 +4,7 @@ name: CI
 on:
   push:
     branches: [ "main" ]
-  pull_request_target:
+  pull_request:
     branches: [ "main" ]
   workflow_dispatch:     # manual run
 
@@ -15,36 +15,35 @@ concurrency:
 permissions:
   pull-requests: read
   contents: read
-  id-token: write
   actions: read
 
 jobs:
 
-  permissions-check:
-    runs-on: ubuntu-latest
-    steps:
-
-      # The following two steps (permissions checks) ensure that only users with write access can run this workflow on a PR (except the merge queue bot)
-      # PRs from forks we check the permissions of the user that triggered the workflow (github.triggering_actor)
-      # This means that if a user without write access opens a PR from a fork, they cannot run this workflow
-      # Users with write access can still run this workflow on a PR from a fork
-      # For PRs from the same repo, we allow the workflow to run as normal
-      - name: Get User Permission
-        if: ${{ github.event_name == 'pull_request_target' || github.triggering_actor != 'github-merge-queue[bot]' }}
-        id: checkAccess
-        uses: actions-cool/check-user-permission@v2
-        with:
-          require: write
-          username: ${{ github.triggering_actor }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Check User Permission
-        if: ${{ (github.event_name == 'pull_request_target' || github.triggering_actor != 'github-merge-queue[bot]') && steps.checkAccess.outputs.require-result == 'false' }}
-        run: |
-          echo "${{ github.triggering_actor }} does not have permissions on this repo."
-          echo "Current permission level is ${{ steps.checkAccess.outputs.user-permission }}"
-          echo "Job originally triggered by ${{ github.actor }}"
-          exit 1
+  # permissions-check:
+  #   runs-on: ubuntu-latest
+  #   steps:
+
+  #     # The following two steps (permissions checks) ensure that only users with write access can run this workflow on a PR (except the merge queue bot)
+  #     # PRs from forks we check the permissions of the user that triggered the workflow (github.triggering_actor)
+  #     # This means that if a user without write access opens a PR from a fork, they cannot run this workflow
+  #     # Users with write access can still run this workflow on a PR from a fork
+  #     # For PRs from the same repo, we allow the workflow to run as normal
+  #     - name: Get User Permission
+  #       if: ${{ github.event_name == 'pull_request_target' || github.triggering_actor != 'github-merge-queue[bot]' }}
+  #       id: checkAccess
+  #       uses: actions-cool/check-user-permission@v2
+  #       with:
+  #         require: write
+  #         username: ${{ github.triggering_actor }}
+  #       env:
+  #         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  #     - name: Check User Permission
+  #       if: ${{ (github.event_name == 'pull_request_target' || github.triggering_actor != 'github-merge-queue[bot]') && steps.checkAccess.outputs.require-result == 'false' }}
+  #       run: |
+  #         echo "${{ github.triggering_actor }} does not have permissions on this repo."
+  #         echo "Current permission level is ${{ steps.checkAccess.outputs.user-permission }}"
+  #         echo "Job originally triggered by ${{ github.actor }}"
+  #         exit 1
 
   check:
     strategy:
@@ -127,7 +126,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
-        python-version: ['3.12', '3.13']
+        python-version: ['3.12', '3.13', '3.14']
         exclude:
         - os: windows-latest
           python-version: '3.13'
@@ -158,58 +157,58 @@ jobs:
         run: |
             ./make.bat test
 
-  online-test:
-    needs: permissions-check
-    environment:
-      name: build-pipeline
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest]
-        python-version: ['3.14']
-    runs-on: ${{ matrix.os }}
-    name: ${{ matrix.os }} (py ${{ matrix.python-version }})
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha || github.ref }}
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v7
-
-      - name: Install Dependencies (Linux)
-        if: runner.os == 'Linux'
-        shell: bash
-        run: |
-            make sync
-
-      - name: Install Dependencies (Windows)
-        if: runner.os == 'Windows'
-        run: |
-            make sync
-
-      - name: Login to Azure
-        uses: azure/login@v2.2.0
-        with:
-          client-id: ${{ secrets.AZURE_CLIENTID }}
-          tenant-id: ${{ secrets.AZURE_TENANTID }}
-          subscription-id: ${{ secrets.AZURE_SUBSCRIPTIONID }}
-
-      - name: Get Keys
-        run: |
-            uv run python tools/get_keys.py --vault build-pipeline-kv
-
-      - name: Run Test
-        shell: bash
-        run: |
-            uv run pytest
-
-      - name: Clean up Keys
-        run: |
-          node -e "try{require('fs').unlinkSync('./.env');}catch(e){}"
+  # online-test:
+  #   needs: permissions-check
+  #   environment:
+  #     name: build-pipeline
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       os: [ubuntu-latest]
+  #       python-version: ['3.14']
+  #   runs-on: ${{ matrix.os }}
+  #   name: ${{ matrix.os }} (py ${{ matrix.python-version }})
+
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{ github.event.pull_request.head.sha || github.ref }}
+
+  #     - name: Set up Python
+  #       uses: actions/setup-python@v5
+  #       with:
+  #         python-version: ${{ matrix.python-version }}
+
+  #     - name: Install uv
+  #       uses: astral-sh/setup-uv@v7
+
+  #     - name: Install Dependencies (Linux)
+  #       if: runner.os == 'Linux'
+  #       shell: bash
+  #       run: |
+  #           make sync
+
+  #     - name: Install Dependencies (Windows)
+  #       if: runner.os == 'Windows'
+  #       run: |
+  #           make sync
+
+  #     - name: Login to Azure
+  #       uses: azure/login@v2.2.0
+  #       with:
+  #         client-id: ${{ secrets.AZURE_CLIENTID }}
+  #         tenant-id: ${{ secrets.AZURE_TENANTID }}
+  #         subscription-id: ${{ secrets.AZURE_SUBSCRIPTIONID }}
+
+  #     - name: Get Keys
+  #       run: |
+  #           uv run python tools/get_keys.py --vault build-pipeline-kv
+
+  #     - name: Run Test
+  #       shell: bash
+  #       run: |
+  #           uv run pytest
+
+  #     - name: Clean up Keys
+  #       run: |
+  #         node -e "try{require('fs').unlinkSync('./.env');}catch(e){}"

.github/workflows/release-py.yml

@@ -32,7 +32,7 @@ jobs:
           echo "$HOME/.local/bin" >> "$GITHUB_PATH"
 
       - name: Create .venv and install deps
-        run: uv sync --extra dev
+        run: uv sync
 
       - name: Build sdist + wheel (via make)
         run: make build                   # runs `uv build`, outputs to dist/

.gitignore

@@ -31,13 +31,16 @@ pytest.local.ini
 
 # Evaluations
 /evals
-/tests/testdata/Episode_53_Answer_results.json
-/tests/testdata/Episode_53_Search_results.json
+
+# E-Mail test data
+!/tests/testdata/email-testdata/.gitkeep
+/tests/testdata/email-testdata/*
 
 # Email demo
-/tools/gmail/client_secret.json
-/tools/gmail/token.json
+/tools/mail/client_secret.json
+/tools/mail/token.json
 *_dump/
+fwts/
 
 # Monty Python demo
 /examples/testdata/MP

AGENTS.md

@@ -9,9 +9,14 @@ that make changes to the repository. Not even `git add`**
 
 When moving, copying or deleting files, use the git commands: `git mv`, `git cp`, `git rm`
 
+When I ask to update AGENTS.md (even if maybe) extract a general rule from what I said
+before and update AGENTS.md (unless it's already in there -- maybe reformulate since
+it apparently didn't work). Also, when it looks like I state a general rule, add it to
+AGENTS.md. In all cases show what you added to AGENTS.md.
+
 - Don't use '!' on the command line, it's some bash magic (even inside single quotes)
-- Activate `.venv`: `make venv; source .venv/bin/activate` (run this only once)
-- To get API keys in ad-hoc code, call `typeagent.aitools.utils.load_dotenv()`
+- When running 'make' commands, do not use the venv (the Makefile uses 'uv run')
+- To get API keys in ad-hoc code, call `load_dotenv()`
 - Use `pytest test` to run tests in test/
 - Use `pyright` to check type annotations in src/, tools/,  tests/, examples/
 - Ignore build/, dist/
@@ -20,6 +25,8 @@ When moving, copying or deleting files, use the git commands: `git mv`, `git cp`
 - Use `make test` to run all tests
 - Use `make check test` to run `make check` and if it passes also run `make test`
 - Use `make format` to format all files using `black`. Do this before reporting success.
+- When validating changes, first run `pytest` only on new/modified test files, then run `make format check test` once at the end.
+- Keep ad-hoc and performance benchmarks under `tools/`, not `tests/`, so `make test` does not run them.
 
 ## Package Management with uv
 
@@ -30,7 +37,7 @@ When moving, copying or deleting files, use the git commands: `git mv`, `git cp`
 - uv maintains consistency between `pyproject.toml`, `uv.lock`, and installed packages
 - Trust uv's automatic version resolution and file management
 
-**IMPORTANT! YOU ARE NOT DONE UNTIL `make check test format` PASSES**
+**IMPORTANT! YOU ARE NOT DONE UNTIL `make format check test` PASSES**
 
 # Code generation
 
@@ -93,3 +100,6 @@ please follow these guidelines:
 
 * **Code Validation**: Don't use `py_compile` for syntax checking.
   Use `pyright` or `make check` instead for proper type checking and validation.
+
+* **Deprecations**: Don't deprecate things -- just delete them and fix the usage sites.
+  Don't create backward compatibility APIs or exports or whatever. Fix the usage sites.

CHANGELOG.md

@@ -1,5 +1,71 @@
 # Typeagent Change Log
 
+## 2026
+
+### 0.4.0 (March 3)
+
+Lots of improvements; the highlights are provider-agnostic model
+configuration backed by pydantic_ai, email ingestion, and a major
+infrastructure overhaul.
+
+#### Core typeagent package
+- Fixed a number of bugs that affected the core query algorithms
+- Provider-agnostic chat and embedding model configuration via new
+  `model_adapters` module backed by pydantic_ai (#200):
+  - Use `provider:model` spec strings,
+    e.g. `create_chat_model("openai:gpt-4o")`.
+  - Replace `AsyncEmbeddingModel` with `IEmbedder`/`IEmbeddingModel`
+    protocols and `CachingEmbeddingModel`.
+  - Add `OPENAI_MODEL` and `OPENAI_EMBEDDING_MODEL` envvars
+    to override the default chat and embedding models.
+- Split `interfaces.py` into separate modules
+  (`interfaces_core`, `_indexes`, `_search`, `_serialization`,
+  `_storage`) (Bernhard Merkle, #118).
+- Make remaining storage-provider APIs async
+  (`get/set_conversation_metadata`, `is_source_ingested`, etc.) (#196).
+- Fix listeners/recipients confusion in podcast metadata serialization (#174).
+- Implement `SqliteRelatedTermsIndex.serialize()` (Rajiv Singh, #115).
+
+#### Email
+- New _tools/ingest_email.py_ tool to ingest email
+  into a SQLite-backed conversation database (#111).
+- Add _tools/mail/outlook_dump.py_ to dump Outlook/Microsoft 365 email
+  via the Graph API (Bernhard Merkle, #199).
+- Add _tools/mail/mbox_dump.py_ to convert mbox files for ingestion
+  (Bernhard Merkle, #198).
+- Consolidate mail dump tools under _tools/mail/_ with shared
+  _README.md_ (Bernhard Merkle).
+- Various ergonomic improvements and fixes (#162, #168, #170).
+
+#### Tools
+- Add conversation history to _tools/query.py_ for
+  pronoun/reference resolution across multi-turn queries
+  (Rajiv Singh, #117).
+- Add _tools/load_json.py_ to load JSON-serialized index data
+  into a SQLite database; remove `--podcast` flag from
+  _tools/query.py_ (#164).
+
+#### Docs
+- Improve docs for Azure env vars (#175).
+- Add AgentCon 2026 presentation and demo videos
+  (Bernhard Merkle, #194, #202).
+- VS Code / Pyright plugin setup instructions
+  (Bernhard Merkle, #150).
+
+#### Infrastructure
+- Changes pyproject.toml to use uv more idiomatically.
+  - Local devs write `uv sync` instead of `uv sync --extra dev`.
+  - From PyPI (with uv or pip) you can use `typeagent[dev]`
+    to install the dev dependencies with the package.
+- Move _typeagent/_ to _src/typeagent/_ (Bernhard Merkle, #139).
+- Move tests and test data to _tests/_ directory (Bernhard Merkle, #144).
+- Move ancillary dirs into subdirs (Bernhard Merkle, #145).
+- Introduce `isort` for import sorting.
+- Make pyright error on unused variables and imports (#129).
+- Add readline support on Windows (#152, Bernhard Merkle).
+- Enhance release script to update _uv.lock_ and create release PR
+  (Rajiv Singh, #169).
+
 ## 2025
 
 ### 0.3.3 (Nov 25)