Skip to content

Commit 66d68cb

Browse files
davd-gzlclaude
andcommitted
Add Wikidata railway-station reference build pipeline (CC0)
Adds scripts/build-railways.mjs + a `railways` npm script that pulls mainline railway stations from Wikidata (instance of Q55488) via SPARQL and reshapes them into the app's reference format (id=QID, name, countryIso2 via P17→P297, lat/lon via P625), writing public/reference/railways.json with recorded provenance (source, CC0 licence, retrieval date). Thresholding keeps it to a few-thousand notable mainline stations: require coordinates, drop demolished/disused (P576/P5817), and keep only stations with a Wikipedia sitelink or a UIC/IBNR code (P954); metro/tram are different Wikidata classes and are already excluded. Aggregator-only — reshapes existing openly- licensed facts, invents none (Constitution I). NOTE: query.wikidata.org is blocked by this environment's egress policy (403), so the fetch must run where Wikidata is reachable (a dev machine or CI); the produced railways.json is then bundled like cities/airports. Wiring railway stations as a full place type across the app (search/map/stats/pickers) is a follow-up feature. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014W6tgHRgLEugCsbKC9ccST
1 parent 429a582 commit 66d68cb

2 files changed

Lines changed: 113 additions & 0 deletions

File tree

apps/postcards/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
"test:e2e": "playwright test",
1616
"format": "prettier --write .",
1717
"reference": "node scripts/build-reference.mjs",
18+
"railways": "node scripts/build-railways.mjs",
1819
"cap:sync": "cap sync",
1920
"cap:copy": "cap copy",
2021
"cap:add:android": "cap add android",
Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
// Build the bundled RAILWAY STATION reference data from Wikidata (CC0).
2+
//
3+
// public/reference/railways.json mainline/intercity railway stations
4+
//
5+
// WHY Wikidata (CC0): it is the only global source that maps 1:1 onto this app's
6+
// reference shape — a stable id (QID), a name, a country (P17 → ISO 3166-1 alpha-2
7+
// via P297) and coordinates (P625) — with NO attribution or share-alike burden.
8+
// Aggregator-only (Constitution I): this script RESHAPES existing, openly-licensed
9+
// facts; it invents nothing. Provenance is recorded in the emitted file.
10+
//
11+
// THRESHOLDING to a few-thousand MAINLINE set (not hundreds of thousands of
12+
// tram/metro stops): we take instances of *railway station* (Q55488) EXACTLY — its
13+
// metro/tram/light-rail siblings are DIFFERENT classes, so they're already excluded
14+
// — require a coordinate (P625), drop demolished/disused (P5817/P576), and keep only
15+
// stations notable enough to carry a Wikipedia sitelink OR a UIC/IBNR code (P954).
16+
// That lands in the low-thousands globally and biases toward significant stations.
17+
//
18+
// NOTE: query.wikidata.org must be reachable to run this (it was blocked by the
19+
// egress policy in the authoring sandbox). Run it where Wikidata is reachable:
20+
//
21+
// node scripts/build-railways.mjs (from apps/postcards)
22+
//
23+
// Then the file is bundled under public/reference/ like cities/airports and loaded
24+
// through the same reference-data seam.
25+
26+
import { writeFileSync } from "node:fs";
27+
import { fileURLToPath } from "node:url";
28+
import { dirname, join } from "node:path";
29+
30+
const OUT = join(dirname(fileURLToPath(import.meta.url)), "..", "public", "reference", "railways.json");
31+
const ENDPOINT = "https://query.wikidata.org/sparql";
32+
const PAGE = 5000; // rows per request; paginate with LIMIT/OFFSET to avoid timeouts
33+
const UA = "Postcards-reference-build/1.0 (local-first travel journal; contact via repo)";
34+
35+
// One page of stations. Filters, in order: exactly a railway station (Q55488);
36+
// has coordinates; not marked demolished/disused; has a country; AND (has ≥1
37+
// Wikipedia sitelink OR a UIC/IBNR station code P954) as the notability threshold.
38+
const query = (limit, offset) => `
39+
SELECT ?station ?stationLabel ?iso2 ?lat ?lon WHERE {
40+
?station wdt:P31 wd:Q55488 ;
41+
wdt:P625 ?coord ;
42+
wdt:P17 ?country .
43+
?country wdt:P297 ?iso2 .
44+
FILTER NOT EXISTS { ?station wdt:P576 ?abolished. } # dissolved/abolished date
45+
FILTER NOT EXISTS { ?station wdt:P5817 wd:Q56556915. } # state of use = demolished
46+
FILTER NOT EXISTS { ?station wdt:P5817 wd:Q45382883. } # state of use = disused
47+
FILTER( EXISTS { ?station wdt:P954 ?uic. }
48+
|| EXISTS { ?sl schema:about ?station ; schema:isPartOf ?wiki .
49+
FILTER(CONTAINS(STR(?wiki), "wikipedia.org")) } )
50+
BIND(geof:latitude(?coord) AS ?lat)
51+
BIND(geof:longitude(?coord) AS ?lon)
52+
SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
53+
}
54+
ORDER BY ?station
55+
LIMIT ${limit} OFFSET ${offset}`;
56+
57+
async function fetchPage(limit, offset) {
58+
const url = `${ENDPOINT}?format=json&query=${encodeURIComponent(query(limit, offset))}`;
59+
const res = await fetch(url, { headers: { Accept: "application/sparql-results+json", "User-Agent": UA } });
60+
if (!res.ok) throw new Error(`Wikidata SPARQL ${res.status} ${res.statusText}`);
61+
const json = await res.json();
62+
return json.results.bindings;
63+
}
64+
65+
function main() {
66+
return (async () => {
67+
// De-dupe by QID (occasional split items) and drop unnamed rows — a place must
68+
// carry a label. Coordinates are numbers; ISO2 is upper-cased for the app.
69+
const byId = new Map();
70+
for (let offset = 0; ; offset += PAGE) {
71+
const rows = await fetchPage(PAGE, offset);
72+
if (!rows.length) break;
73+
for (const r of rows) {
74+
const id = r.station.value.replace("http://www.wikidata.org/entity/", "");
75+
const name = r.stationLabel?.value?.trim();
76+
// Wikidata returns the QID as the label when no English label exists — skip those.
77+
if (!name || name === id || byId.has(id)) continue;
78+
const iso2 = r.iso2.value.toUpperCase();
79+
const lat = Number(r.lat.value);
80+
const lon = Number(r.lon.value);
81+
if (!/^[A-Z]{2}$/.test(iso2) || !Number.isFinite(lat) || !Number.isFinite(lon)) continue;
82+
// subdivisionId is left null for v1: railways contribute to per-COUNTRY
83+
// coverage immediately; region (admin-1) assignment via nearest-centroid
84+
// against public/reference/subdivisions.json can be layered on later, the
85+
// way build-reference.mjs matches city regions geographically.
86+
byId.set(id, { id, name, countryIso2: iso2, subdivisionId: null, lat, lon });
87+
}
88+
process.stderr.write(` …${byId.size} stations so far\n`);
89+
if (rows.length < PAGE) break;
90+
}
91+
92+
const stations = [...byId.values()].sort((a, b) => a.name.localeCompare(b.name));
93+
const out = {
94+
// Provenance is part of the file (Constitution: named source + license + date).
95+
_source: {
96+
dataset: "Wikidata railway stations (instance of Q55488)",
97+
url: "https://query.wikidata.org/",
98+
license: "CC0-1.0",
99+
retrieved: new Date().toISOString().slice(0, 10),
100+
note: "Mainline stations with coordinates and a Wikipedia sitelink or UIC code; metro/tram excluded.",
101+
},
102+
stations,
103+
};
104+
writeFileSync(OUT, JSON.stringify(out));
105+
process.stderr.write(`Wrote ${stations.length} railway stations → ${OUT}\n`);
106+
})();
107+
}
108+
109+
main().catch((e) => {
110+
process.stderr.write(String(e?.stack || e) + "\n");
111+
process.exitCode = 1;
112+
});

0 commit comments

Comments
 (0)