diff --git a/.github/workflows/dispatch_shadow_signal.yml b/.github/workflows/dispatch_shadow_signal.yml
index b4b6bd5..1f09ecb 100644
--- a/.github/workflows/dispatch_shadow_signal.yml
+++ b/.github/workflows/dispatch_shadow_signal.yml
@@ -11,7 +11,7 @@ on:
description: "Bridge provider"
required: false
type: choice
- default: "auto"
+ default: "codex"
options:
- auto
- api
@@ -39,7 +39,7 @@ jobs:
BRIDGE_REPOSITORY: ${{ vars.CODEX_AUDIT_BRIDGE_REPOSITORY || 'QuantStrategyLab/AIAuditBridge' }}
BRIDGE_REF: ${{ vars.CODEX_AUDIT_BRIDGE_REF || 'main' }}
BRIDGE_TASK: long_horizon_signal_shadow
- BRIDGE_PROVIDER: ${{ github.event.inputs.provider || 'auto' }}
+ BRIDGE_PROVIDER: ${{ github.event.inputs.provider || 'codex' }}
SOURCE_REF: ${{ github.event.inputs.source_ref || 'main' }}
SHADOW_SIGNAL_LABEL: long-horizon-shadow
steps:
diff --git a/scripts/build_context_bundle.py b/scripts/build_context_bundle.py
index e72bbfd..ddeccea 100644
--- a/scripts/build_context_bundle.py
+++ b/scripts/build_context_bundle.py
@@ -2,6 +2,7 @@
from __future__ import annotations
import argparse
+import datetime as dt
import json
import sys
from pathlib import Path
@@ -18,6 +19,7 @@
normalize_symbols,
write_context_bundle,
)
+from research_signal_context_pipelines.research_context_adapter import ResearchContextAdapter # noqa: E402
from research_signal_context_pipelines.theme_universe import ( # noqa: E402
build_theme_context,
load_symbol_theme_exposure,
@@ -50,9 +52,13 @@ def main() -> int:
default="data/output/context_bundle/latest_context_bundle.json",
help="Output JSON context bundle path",
)
+ parser.add_argument("--web-research-sources", help="Optional JSON config with whitelist and web research sources")
+ parser.add_argument("--web-research-timeout", type=float, default=10.0, help="Timeout in seconds for web research fetches")
+ parser.add_argument("--web-research-max-entries", type=int, default=8, help="Maximum web research entries to include")
args = parser.parse_args()
symbols = normalize_symbols(args.symbols)
+ generated_at = dt.datetime.now(dt.timezone.utc)
theme_context = None
if not args.no_theme_context:
theme_taxonomy_path = Path(args.theme_taxonomy)
@@ -62,6 +68,14 @@ def main() -> int:
exposures = load_symbol_theme_exposure(theme_exposure_path, known_theme_ids=themes)
theme_context = build_theme_context(symbols=symbols, themes=themes, exposures=exposures)
+ web_research_context = None
+ if args.web_research_sources:
+ web_research_context = ResearchContextAdapter(
+ Path(args.web_research_sources),
+ timeout_seconds=args.web_research_timeout,
+ max_entries=args.web_research_max_entries,
+ ).build_context(pit_timestamp=generated_at)
+
try:
bundle = build_context_from_source(
symbols=symbols,
@@ -70,7 +84,9 @@ def main() -> int:
end_date=args.end_date,
lookback_days=args.lookback_days,
theme_context=theme_context,
+ web_research_context=web_research_context,
allow_partial_downloads=not args.strict_downloads,
+ generated_at=generated_at,
)
except Exception as exc:
if not args.allow_download_errors:
@@ -79,6 +95,8 @@ def main() -> int:
symbols=symbols,
error=f"{type(exc).__name__}: {exc}",
theme_context=theme_context,
+ web_research_context=web_research_context,
+ generated_at=generated_at,
)
output_path = Path(args.output)
write_context_bundle(bundle, output_path)
diff --git a/scripts/post_shadow_signal_request.py b/scripts/post_shadow_signal_request.py
index fd8c074..0527de5 100644
--- a/scripts/post_shadow_signal_request.py
+++ b/scripts/post_shadow_signal_request.py
@@ -172,7 +172,7 @@ def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Create or update a long-horizon shadow signal issue.")
parser.add_argument("--repo", required=True)
parser.add_argument("--source-ref", default="main")
- parser.add_argument("--provider", default="auto")
+ parser.add_argument("--provider", default="codex")
parser.add_argument("--bridge-repository", default="QuantStrategyLab/AIAuditBridge")
parser.add_argument("--as-of-date")
parser.add_argument("--context-file", help="Optional JSON context bundle to embed in the issue body")
diff --git a/src/research_signal_context_pipelines/context_bundle.py b/src/research_signal_context_pipelines/context_bundle.py
index 258c787..ab752bd 100644
--- a/src/research_signal_context_pipelines/context_bundle.py
+++ b/src/research_signal_context_pipelines/context_bundle.py
@@ -276,6 +276,7 @@ def build_context_bundle(
symbols: list[str],
generated_at: dt.datetime | None = None,
theme_context: Mapping[str, Any] | None = None,
+ web_research_context: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
contexts: dict[str, SymbolContext] = {}
warnings: list[str] = []
@@ -337,6 +338,8 @@ def build_context_bundle(
}
if theme_context is not None:
bundle["theme_context"] = dict(theme_context)
+ if web_research_context is not None:
+ bundle["web_research"] = dict(web_research_context)
return bundle
@@ -347,6 +350,7 @@ def build_error_context_bundle(
as_of_date: dt.date | None = None,
generated_at: dt.datetime | None = None,
theme_context: Mapping[str, Any] | None = None,
+ web_research_context: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
as_of = as_of_date or dt.date.today()
timestamp = generated_at or dt.datetime.now(dt.timezone.utc)
@@ -375,6 +379,8 @@ def build_error_context_bundle(
}
if theme_context is not None:
bundle["theme_context"] = dict(theme_context)
+ if web_research_context is not None:
+ bundle["web_research"] = dict(web_research_context)
return bundle
@@ -395,7 +401,9 @@ def build_context_from_source(
lookback_days: int = 420,
fetch_fn: Callable[..., Mapping[str, Any]] | None = None,
theme_context: Mapping[str, Any] | None = None,
+ web_research_context: Mapping[str, Any] | None = None,
allow_partial_downloads: bool = False,
+ generated_at: dt.datetime | None = None,
) -> dict[str, Any]:
end = parse_price_date(end_date) if end_date else dt.date.today()
start = parse_price_date(start_date) if start_date else end - dt.timedelta(days=int(lookback_days))
@@ -409,7 +417,13 @@ def build_context_from_source(
fetch_fn=fetch_fn,
allow_partial=allow_partial_downloads,
)
- return build_context_bundle(rows, symbols=symbols, theme_context=theme_context)
+ return build_context_bundle(
+ rows,
+ symbols=symbols,
+ generated_at=generated_at,
+ theme_context=theme_context,
+ web_research_context=web_research_context,
+ )
def write_context_bundle(bundle: Mapping[str, Any], path: Path) -> None:
diff --git a/src/research_signal_context_pipelines/research_context_adapter.py b/src/research_signal_context_pipelines/research_context_adapter.py
new file mode 100644
index 0000000..e3127fe
--- /dev/null
+++ b/src/research_signal_context_pipelines/research_context_adapter.py
@@ -0,0 +1,306 @@
+from __future__ import annotations
+
+import datetime as dt
+import email.utils
+import json
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from html.parser import HTMLParser
+from pathlib import Path
+from typing import Any
+from urllib.error import URLError
+from urllib.parse import urlsplit
+from urllib.request import Request, urlopen
+from xml.etree import ElementTree as ET
+
+
+DEFAULT_WEB_RESEARCH_TIMEOUT_SECONDS = 10.0
+DEFAULT_WEB_RESEARCH_MAX_ENTRIES = 8
+WEB_RESEARCH_USER_AGENT = "Mozilla/5.0"
+
+
+@dataclass(frozen=True)
+class ResearchSourceConfig:
+ url: str
+ kind: str = "auto"
+
+
+def _clean_text(value: object) -> str:
+ text = " ".join(str(value or "").split())
+ return text.strip()
+
+
+def _isoformat_utc(value: dt.datetime | None) -> str:
+ timestamp = value or dt.datetime.now(dt.timezone.utc)
+ if timestamp.tzinfo is None:
+ timestamp = timestamp.replace(tzinfo=dt.timezone.utc)
+ return timestamp.astimezone(dt.timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def _parse_datetime(value: object) -> str | None:
+ text = _clean_text(value)
+ if not text:
+ return None
+ if text.endswith("Z"):
+ text = f"{text[:-1]}+00:00"
+ try:
+ parsed = dt.datetime.fromisoformat(text)
+ except ValueError:
+ try:
+ parsed = email.utils.parsedate_to_datetime(text)
+ except (TypeError, ValueError):
+ return None
+ if parsed.tzinfo is None:
+ parsed = parsed.replace(tzinfo=dt.timezone.utc)
+ return parsed.astimezone(dt.timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def _load_research_config(path: Path) -> tuple[list[str], list[ResearchSourceConfig]]:
+ payload = json.loads(path.read_text(encoding="utf-8"))
+ if isinstance(payload, list):
+ whitelist: list[str] = []
+ sources_payload = payload
+ elif isinstance(payload, dict):
+ whitelist = [_clean_text(item) for item in payload.get("whitelist", []) if _clean_text(item)]
+ sources_payload = list(payload.get("sources", []))
+ else:
+ raise ValueError("research sources config must be a list or object")
+
+ sources: list[ResearchSourceConfig] = []
+ for raw in sources_payload:
+ if not isinstance(raw, Mapping):
+ raise ValueError("research source entries must be objects")
+ url = _clean_text(raw.get("url"))
+ if not url:
+ raise ValueError("research source entries require url")
+ sources.append(ResearchSourceConfig(url=url, kind=_clean_text(raw.get("kind")) or "auto"))
+
+ return whitelist, sources
+
+
+def _is_whitelisted(url: str, whitelist: Iterable[str]) -> bool:
+ if not whitelist:
+ return False
+ candidate = urlsplit(url.strip())
+ if not candidate.scheme or not candidate.netloc:
+ return False
+ candidate_path = candidate.path or "/"
+ for raw_prefix in whitelist:
+ allowed = urlsplit(raw_prefix)
+ if candidate.scheme != allowed.scheme or candidate.netloc != allowed.netloc:
+ continue
+ allowed_path = allowed.path or "/"
+ if allowed_path == "/":
+ return True
+ normalized_allowed = allowed_path.rstrip("/")
+ if candidate_path == allowed_path or candidate_path.startswith(f"{normalized_allowed}/"):
+ return True
+ return False
+
+
+def _source_kind(source: ResearchSourceConfig, *, content_type: str | None, body: bytes) -> str:
+ if source.kind and source.kind != "auto":
+ return source.kind.lower()
+ if content_type:
+ lowered = content_type.lower()
+ if "rss" in lowered or "atom" in lowered or "xml" in lowered:
+ return "rss"
+ if "html" in lowered:
+ return "http"
+ prefix = body.lstrip()[:200].lower()
+ if prefix.startswith(b" dict[str, Any]:
+ def child_text(*names: str) -> str:
+ for node in item:
+ local_name = node.tag.rsplit("}", 1)[-1].lower()
+ if local_name in names:
+ text = _clean_text("".join(node.itertext()))
+ if text:
+ return text
+ return ""
+
+ link = ""
+ for node in item:
+ local_name = node.tag.rsplit("}", 1)[-1].lower()
+ if local_name == "link":
+ href = _clean_text(node.attrib.get("href"))
+ if href:
+ link = href
+ break
+ text = _clean_text(node.text)
+ if text:
+ link = text
+ break
+
+ published_at = None
+ for field in ("published", "updated", "pubdate", "date"):
+ published_at = _parse_datetime(child_text(field))
+ if published_at:
+ break
+
+ summary = child_text("summary", "description", "encoded")
+ title = child_text("title") or source_url
+ return {
+ "title": title,
+ "summary": summary,
+ "published_at": published_at,
+ "url": link or source_url,
+ "source_url": source_url,
+ "source_type": "rss",
+ "fetched_at": fetched_at,
+ }
+
+
+def _feed_entries(body: bytes, *, source_url: str, fetched_at: str) -> list[dict[str, Any]]:
+ root = ET.fromstring(body)
+ local_root = root.tag.rsplit("}", 1)[-1].lower()
+ items: list[ET.Element] = []
+ if local_root == "rss":
+ channel = next((child for child in root if child.tag.rsplit("}", 1)[-1].lower() == "channel"), root)
+ items = [child for child in channel if child.tag.rsplit("}", 1)[-1].lower() == "item"]
+ else:
+ items = [child for child in root.iter() if child.tag.rsplit("}", 1)[-1].lower() in {"item", "entry"}]
+ return [_entry_from_feed_item(item, source_url=source_url, fetched_at=fetched_at) for item in items]
+
+
+class _NewsHTMLParser(HTMLParser):
+ def __init__(self) -> None:
+ super().__init__()
+ self.title_parts: list[str] = []
+ self._capture_title = False
+ self._capture_paragraph = False
+ self._paragraph_buffer: list[str] = []
+ self.title: str = ""
+ self.summary: str = ""
+ self.published_at: str | None = None
+
+ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+ attributes = {key.lower(): value or "" for key, value in attrs}
+ lowered = tag.lower()
+ if lowered == "title":
+ self._capture_title = True
+ elif lowered == "p" and not self.summary:
+ self._capture_paragraph = True
+ self._paragraph_buffer = []
+ elif lowered == "meta":
+ name = attributes.get("name", "").lower()
+ prop = attributes.get("property", "").lower()
+ content = _clean_text(attributes.get("content"))
+ if not content:
+ return
+ if name == "description" or prop in {"og:description", "twitter:description"}:
+ self.summary = content
+ if prop == "article:published_time" or name in {"pubdate", "date"}:
+ self.published_at = self.published_at or _parse_datetime(content)
+ elif lowered == "time" and not self.published_at:
+ self.published_at = _parse_datetime(attributes.get("datetime"))
+
+ def handle_endtag(self, tag: str) -> None:
+ lowered = tag.lower()
+ if lowered == "title":
+ self._capture_title = False
+ if not self.title:
+ self.title = _clean_text("".join(self.title_parts))
+ elif lowered == "p" and self._capture_paragraph:
+ self._capture_paragraph = False
+ if not self.summary:
+ summary = _clean_text("".join(self._paragraph_buffer))
+ if summary:
+ self.summary = summary
+
+ def handle_data(self, data: str) -> None:
+ if self._capture_title:
+ self.title_parts.append(data)
+ if self._capture_paragraph:
+ self._paragraph_buffer.append(data)
+
+
+def _html_entry(body: bytes, *, source_url: str, fetched_at: str) -> dict[str, Any]:
+ parser = _NewsHTMLParser()
+ parser.feed(body.decode("utf-8", errors="replace"))
+ title = parser.title or source_url
+ summary = parser.summary
+ return {
+ "title": title,
+ "summary": summary,
+ "published_at": parser.published_at,
+ "url": source_url,
+ "source_url": source_url,
+ "source_type": "news",
+ "fetched_at": fetched_at,
+ }
+
+
+class ResearchContextAdapter:
+ def __init__(
+ self,
+ sources_path: Path,
+ *,
+ timeout_seconds: float = DEFAULT_WEB_RESEARCH_TIMEOUT_SECONDS,
+ max_entries: int = DEFAULT_WEB_RESEARCH_MAX_ENTRIES,
+ ) -> None:
+ if timeout_seconds <= 0:
+ raise ValueError("timeout_seconds must be positive")
+ if max_entries <= 0:
+ raise ValueError("max_entries must be positive")
+ self.sources_path = Path(sources_path)
+ self.timeout_seconds = float(timeout_seconds)
+ self.max_entries = int(max_entries)
+
+ def build_context(self, *, pit_timestamp: dt.datetime | None = None) -> dict[str, Any]:
+ fetched_at = _isoformat_utc(pit_timestamp)
+ context: dict[str, Any] = {
+ "pit_timestamp": fetched_at,
+ "research_sources": [],
+ "warnings": [],
+ }
+ if not self.sources_path.exists():
+ context["warnings"].append(f"research sources file not found: {self.sources_path}")
+ return context
+
+ try:
+ whitelist, sources = _load_research_config(self.sources_path)
+ except Exception as exc:
+ context["warnings"].append(f"failed to load research sources config: {type(exc).__name__}: {exc}")
+ return context
+
+ if not whitelist:
+ context["warnings"].append("research sources config has no whitelist")
+ return context
+
+ collected: list[dict[str, Any]] = []
+ warnings = context["warnings"]
+ for source in sources:
+ if len(collected) >= self.max_entries:
+ break
+ if not _is_whitelisted(source.url, whitelist):
+ warnings.append(f"skipped non-whitelisted source: {source.url}")
+ continue
+ try:
+ request = Request(source.url, headers={"User-Agent": WEB_RESEARCH_USER_AGENT, "Accept": "*/*"})
+ with urlopen(request, timeout=self.timeout_seconds) as response: # noqa: S310 - operator-controlled research fetch.
+ body = response.read()
+ content_type = response.headers.get_content_type() if hasattr(response.headers, "get_content_type") else None
+ except (OSError, URLError, TimeoutError, ValueError) as exc:
+ warnings.append(f"failed to fetch research source {source.url}: {type(exc).__name__}: {exc}")
+ continue
+
+ kind = _source_kind(source, content_type=content_type, body=body)
+ try:
+ entries = _feed_entries(body, source_url=source.url, fetched_at=fetched_at) if kind == "rss" else [_html_entry(body, source_url=source.url, fetched_at=fetched_at)]
+ except Exception as exc:
+ warnings.append(f"failed to parse research source {source.url}: {type(exc).__name__}: {exc}")
+ continue
+
+ for entry in entries:
+ if len(collected) >= self.max_entries:
+ break
+ collected.append(entry)
+
+ context["research_sources"] = collected
+ context["source_count"] = len(collected)
+ return context
diff --git a/tests/test_context_bundle.py b/tests/test_context_bundle.py
index 3799b2a..60c1bc3 100644
--- a/tests/test_context_bundle.py
+++ b/tests/test_context_bundle.py
@@ -136,3 +136,35 @@ def fake_fetch(symbol, *, start, end=None):
assert sorted(bundle["price_context"]) == ["QQQ"]
assert "missing price history for BAD" in bundle["data_quality"]["warnings"]
+
+
+def test_build_context_bundle_omits_web_research_by_default() -> None:
+ rows = [
+ PriceRow(date=dt.date(2026, 1, 1) + dt.timedelta(days=idx), symbol="QQQ", close=100 + idx)
+ for idx in range(260)
+ ]
+
+ bundle = build_context_bundle(rows, symbols=["QQQ"], generated_at=dt.datetime(2026, 1, 1, tzinfo=dt.timezone.utc))
+
+ assert "web_research" not in bundle
+
+
+def test_build_context_bundle_embeds_web_research_context() -> None:
+ rows = [
+ PriceRow(date=dt.date(2026, 1, 1) + dt.timedelta(days=idx), symbol="QQQ", close=100 + idx)
+ for idx in range(260)
+ ]
+
+ web_research_context = {
+ "pit_timestamp": "2026-01-01T00:00:00Z",
+ "research_sources": [{"title": "Example", "summary": "Summary", "url": "https://example.com"}],
+ }
+
+ bundle = build_context_bundle(
+ rows,
+ symbols=["QQQ"],
+ generated_at=dt.datetime(2026, 1, 1, tzinfo=dt.timezone.utc),
+ web_research_context=web_research_context,
+ )
+
+ assert bundle["web_research"] == web_research_context
diff --git a/tests/test_dispatch_shadow_signal_workflow.py b/tests/test_dispatch_shadow_signal_workflow.py
index a72078f..7ac19c9 100644
--- a/tests/test_dispatch_shadow_signal_workflow.py
+++ b/tests/test_dispatch_shadow_signal_workflow.py
@@ -7,5 +7,7 @@ def test_dispatch_shadow_signal_pins_bridge_ref_via_variable() -> None:
workflow = Path(".github/workflows/dispatch_shadow_signal.yml").read_text(encoding="utf-8")
assert "BRIDGE_REF: ${{ vars.CODEX_AUDIT_BRIDGE_REF || 'main' }}" in workflow
+ assert "default: \"codex\"" in workflow
+ assert "BRIDGE_PROVIDER: ${{ github.event.inputs.provider || 'codex' }}" in workflow
assert '"ref": os.environ["BRIDGE_REF"]' in workflow
assert "QuantStrategyLab/AIAuditBridge" in workflow
diff --git a/tests/test_post_shadow_signal_request.py b/tests/test_post_shadow_signal_request.py
index 817e8b9..5329436 100644
--- a/tests/test_post_shadow_signal_request.py
+++ b/tests/test_post_shadow_signal_request.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import json
+import sys
from scripts import post_shadow_signal_request as shadow_issue
@@ -27,6 +28,13 @@ def test_resolve_as_of_date_prefers_context_bundle() -> None:
assert shadow_issue.resolve_as_of_date("2026-05-28", {"as_of": "2026-05-29"}) == "2026-05-28"
+def test_parse_args_defaults_provider_to_codex(monkeypatch) -> None:
+ monkeypatch.setattr(sys, "argv", ["post_shadow_signal_request.py", "--repo", "QuantStrategyLab/ResearchSignalContextPipelines"])
+ args = shadow_issue.parse_args()
+
+ assert args.provider == "codex"
+
+
def test_upsert_issue_updates_existing_issue(monkeypatch) -> None:
calls: list[tuple[str, str, dict | None]] = []
diff --git a/tests/test_research_context_adapter.py b/tests/test_research_context_adapter.py
new file mode 100644
index 0000000..e8f7c76
--- /dev/null
+++ b/tests/test_research_context_adapter.py
@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+import datetime as dt
+import json
+from pathlib import Path
+
+from research_signal_context_pipelines.research_context_adapter import ResearchContextAdapter
+
+
+class _FakeHeaders:
+ def __init__(self, content_type: str) -> None:
+ self._content_type = content_type
+
+ def get_content_type(self) -> str:
+ return self._content_type
+
+ def get_content_charset(self, default: str = "utf-8") -> str:
+ return default
+
+
+class _FakeResponse:
+ def __init__(self, body: str, content_type: str) -> None:
+ self._body = body.encode("utf-8")
+ self.headers = _FakeHeaders(content_type)
+
+ def read(self) -> bytes:
+ return self._body
+
+ def __enter__(self) -> "_FakeResponse":
+ return self
+
+ def __exit__(self, exc_type, exc, tb) -> bool:
+ return False
+
+
+def test_research_context_adapter_respects_whitelist_and_max_entries(tmp_path: Path, monkeypatch) -> None:
+ config_path = tmp_path / "web_research.json"
+ config_path.write_text(
+ json.dumps(
+ {
+ "whitelist": ["https://allowed.example"],
+ "sources": [
+ {"url": "https://allowed.example/feed.xml", "kind": "rss"},
+ {"url": "https://blocked.example/news", "kind": "news"},
+ ],
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ rss_body = """
+
Fallback summary
+ + + """ + + def fake_urlopen(request, timeout): + assert timeout == 5.0 + assert request.full_url == "https://news.example/story" + return _FakeResponse(html_body, "text/html; charset=utf-8") + + monkeypatch.setattr("research_signal_context_pipelines.research_context_adapter.urlopen", fake_urlopen) + + context = ResearchContextAdapter(config_path, timeout_seconds=5.0, max_entries=3).build_context( + pit_timestamp=dt.datetime(2026, 1, 1, tzinfo=dt.timezone.utc) + ) + + assert context["source_count"] == 1 + assert context["research_sources"] == [ + { + "title": "Breaking research", + "summary": "Short summary", + "published_at": "2026-01-02T03:04:05Z", + "url": "https://news.example/story", + "source_url": "https://news.example/story", + "source_type": "news", + "fetched_at": "2026-01-01T00:00:00Z", + } + ]