diff --git a/src/lib/import-gtfs.ts b/src/lib/import-gtfs.ts index bddb21aa..09af4885 100644 --- a/src/lib/import-gtfs.ts +++ b/src/lib/import-gtfs.ts @@ -22,10 +22,15 @@ import { validateConfigForImport, } from './utils.ts'; -import { Config, ConfigAgency, Model } from '../types/global_interfaces.ts'; +import { + Config, + ConfigAgency, + Model, + TableNames, +} from '../types/global_interfaces.ts'; interface GtfsImportTask { - exclude?: string[]; + exclude?: TableNames[]; url?: string; headers?: Record; realtimeAlerts?: { @@ -198,11 +203,11 @@ const createGtfsTables = (db: Database.Database): void => { if (column.type === 'time') { sqlColumnCreateStatements.push( `${getTimestampColumnName(column.name)} INTEGER GENERATED ALWAYS AS ( - CASE - WHEN ${column.name} IS NULL OR ${column.name} = '' THEN NULL + CASE + WHEN ${column.name} IS NULL OR ${column.name} = '' THEN NULL ELSE CAST( - substr(${column.name}, 1, instr(${column.name}, ':') - 1) * 3600 + - substr(${column.name}, instr(${column.name}, ':') + 1, 2) * 60 + + substr(${column.name}, 1, instr(${column.name}, ':') - 1) * 3600 + + substr(${column.name}, instr(${column.name}, ':') + 1, 2) * 60 + substr(${column.name}, -2) AS INTEGER ) END @@ -536,6 +541,11 @@ const importGtfsFiles = ( }), ); +/** + * Function to import GTFS files into the database + * + * @param initialConfig + */ export async function importGtfs(initialConfig: Config): Promise { // Start timer const startTime = process.hrtime.bigint(); @@ -559,7 +569,6 @@ export async function importGtfs(initialConfig: Config): Promise { const task = { exclude: agency.exclude, - url: agency.url, headers: agency.headers, realtimeAlerts: agency.realtimeAlerts, realtimeTripUpdates: agency.realtimeTripUpdates, @@ -567,7 +576,6 @@ export async function importGtfs(initialConfig: Config): Promise { downloadDir: tempPath, downloadTimeout: config.downloadTimeout, gtfsRealtimeExpirationSeconds: config.gtfsRealtimeExpirationSeconds, - path: agency.path, csvOptions: config.csvOptions || {}, ignoreDuplicates: config.ignoreDuplicates, ignoreErrors: config.ignoreErrors, @@ -579,8 +587,14 @@ export async function importGtfs(initialConfig: Config): Promise { logError: logError(config), }; - if (task.url) { + if ('url' in agency) { + Object.assign(task, { url: agency.url }); + await downloadGtfsFiles(task); + } else { + Object.assign(task, { + path: agency.path, + }); } await extractGtfsFiles(task); diff --git a/src/types/global_interfaces.ts b/src/types/global_interfaces.ts index d523361c..656eee56 100644 --- a/src/types/global_interfaces.ts +++ b/src/types/global_interfaces.ts @@ -3,37 +3,171 @@ import type { Database } from 'better-sqlite3'; export type UnixTimestamp = number; -export interface ConfigAgency { - exclude?: string[]; - url?: string; - path?: string; +export type TableNames = + | 'agency' + | 'stops' + | 'routes' + | 'trips' + | 'stop_times' + | 'calendar' + | 'calendar_dates' + | 'fare_attributes' + | 'fare_rules' + | 'timeframes' + | 'rider_categories' + | 'fare_media' + | 'fare_products' + | 'fare_leg_rules' + | 'fare_leg_join_rules' + | 'fare_transfer_rules' + | 'areas' + | 'stop_areas' + | 'networks' + | 'route_networks' + | 'shapes' + | 'frequencies' + | 'transfers' + | 'pathways' + | 'levels' + | 'location_groups' + | 'location_group_stops' + | 'locations' + | 'booking_rules' + | 'translations' + | 'feed_info' + | 'attributions'; + +interface BaseConfigAgency { + /** + * An array of GTFS file names (without .txt) to exclude when importing + */ + exclude?: TableNames[]; + /** + * An object of HTTP headers in key:value format to use when fetching GTFS from the url specified + */ headers?: Record; + /** + * Settings for fetching GTFS-Realtime alerts + */ realtimeAlerts?: { + /** + * URL for fetching GTFS-Realtime alerts + */ url: string; + /** + * Headers to use when fetching GTFS-Realtime alerts + */ headers?: Record; }; + /** + * Settings for fetching GTFS-Realtime trip updates + */ realtimeTripUpdates?: { + /** + * URL for fetching GTFS-Realtime trip updates + */ url: string; + /** + * Headers to use when fetching GTFS-Realtime trip updates + */ headers?: Record; }; + /** + * Settings for fetching GTFS-Realtime vehicle positions + */ realtimeVehiclePositions?: { + /** + * URL for fetching GTFS-Realtime vehicle positions + */ url: string; + /** + * Headers to use when fetching GTFS-Realtime vehicle positions + */ headers?: Record; }; + /** + * A prefix to be added to every ID field maintain uniqueness when importing multiple GTFS from multiple agencies + */ prefix?: string; } +export type ConfigAgency = BaseConfigAgency & + ( + | { + /** + * The URL to a zipped GTFS file. Required if path not present + */ + url: string; + } + | { + /** + * A path to a zipped GTFS file or a directory of unzipped .txt files. Required if url is not present + */ + path: string; + } + ); + export interface Config { + /** + * An existing database instance to use instead of relying on node-gtfs to connect. + */ db?: Database; + /** + * A path to an SQLite database. Defaults to using an in-memory database. + */ sqlitePath?: string; + /** + * Amount of time in seconds to allow GTFS-Realtime data to be stored in database before allowing to be deleted. + * + * Note: is an integer + * + * @defaultValue 0 + */ gtfsRealtimeExpirationSeconds?: number; + /** + * The number of milliseconds to wait before throwing an error when downloading GTFS. + * + * Note: is an integer + */ downloadTimeout?: number; + /** + * Options passed to `csv-parse` for parsing GTFS CSV files. + */ csvOptions?: Options; + /** + * A path to a directory to put exported GTFS files. + * + * @defaultValue `gtfs-export/` + */ exportPath?: string; + /** + * Whether or not to ignore unique constraints on ids when importing GTFS, such as `trip_id`, `calendar_id`. + * + * @defaultValue false + */ ignoreDuplicates?: boolean; + /** + * Whether or not to ignore errors during the import process. If true, failed files will be skipped while the rest are processed. + * + * @defaultValue false + */ ignoreErrors?: boolean; + /** + * An array of GTFS files to be imported, and which files to exclude. + */ agencies: ConfigAgency[]; + /** + * Whether or not to print output to the console. + * + * @defaulValue true + */ verbose?: boolean; + /** + * An optional custom logger instead of the build in console.log + * + * @param message + * @returns + */ logFunction?: (message: string) => void; } @@ -52,7 +186,7 @@ export interface ModelColumn { } export interface Model { - filenameBase: string; + filenameBase: TableNames; filenameExtension?: string; extension?: string; nonstandard?: boolean;