Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 23 additions & 9 deletions src/lib/import-gtfs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,15 @@ import {
validateConfigForImport,
} from './utils.ts';

import { Config, ConfigAgency, Model } from '../types/global_interfaces.ts';
import {
Config,
ConfigAgency,
Model,
TableNames,
} from '../types/global_interfaces.ts';

interface GtfsImportTask {
exclude?: string[];
exclude?: TableNames[];
url?: string;
headers?: Record<string, string>;
realtimeAlerts?: {
Expand Down Expand Up @@ -198,11 +203,11 @@ const createGtfsTables = (db: Database.Database): void => {
if (column.type === 'time') {
sqlColumnCreateStatements.push(
`${getTimestampColumnName(column.name)} INTEGER GENERATED ALWAYS AS (
CASE
WHEN ${column.name} IS NULL OR ${column.name} = '' THEN NULL
CASE
WHEN ${column.name} IS NULL OR ${column.name} = '' THEN NULL
ELSE CAST(
substr(${column.name}, 1, instr(${column.name}, ':') - 1) * 3600 +
substr(${column.name}, instr(${column.name}, ':') + 1, 2) * 60 +
substr(${column.name}, 1, instr(${column.name}, ':') - 1) * 3600 +
substr(${column.name}, instr(${column.name}, ':') + 1, 2) * 60 +
substr(${column.name}, -2) AS INTEGER
)
END
Expand Down Expand Up @@ -536,6 +541,11 @@ const importGtfsFiles = (
}),
);

/**
* Function to import GTFS files into the database
*
* @param initialConfig
*/
export async function importGtfs(initialConfig: Config): Promise<void> {
// Start timer
const startTime = process.hrtime.bigint();
Expand All @@ -559,15 +569,13 @@ export async function importGtfs(initialConfig: Config): Promise<void> {

const task = {
exclude: agency.exclude,
url: agency.url,
headers: agency.headers,
realtimeAlerts: agency.realtimeAlerts,
realtimeTripUpdates: agency.realtimeTripUpdates,
realtimeVehiclePositions: agency.realtimeVehiclePositions,
downloadDir: tempPath,
downloadTimeout: config.downloadTimeout,
gtfsRealtimeExpirationSeconds: config.gtfsRealtimeExpirationSeconds,
path: agency.path,
csvOptions: config.csvOptions || {},
ignoreDuplicates: config.ignoreDuplicates,
ignoreErrors: config.ignoreErrors,
Expand All @@ -579,8 +587,14 @@ export async function importGtfs(initialConfig: Config): Promise<void> {
logError: logError(config),
};

if (task.url) {
if ('url' in agency) {
Object.assign(task, { url: agency.url });

await downloadGtfsFiles(task);
} else {
Object.assign(task, {
path: agency.path,
});
}

await extractGtfsFiles(task);
Expand Down
144 changes: 139 additions & 5 deletions src/types/global_interfaces.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,37 +3,171 @@ import type { Database } from 'better-sqlite3';

export type UnixTimestamp = number;

export interface ConfigAgency {
exclude?: string[];
url?: string;
path?: string;
export type TableNames =
| 'agency'
| 'stops'
| 'routes'
| 'trips'
| 'stop_times'
| 'calendar'
| 'calendar_dates'
| 'fare_attributes'
| 'fare_rules'
| 'timeframes'
| 'rider_categories'
| 'fare_media'
| 'fare_products'
| 'fare_leg_rules'
| 'fare_leg_join_rules'
| 'fare_transfer_rules'
| 'areas'
| 'stop_areas'
| 'networks'
| 'route_networks'
| 'shapes'
| 'frequencies'
| 'transfers'
| 'pathways'
| 'levels'
| 'location_groups'
| 'location_group_stops'
| 'locations'
| 'booking_rules'
| 'translations'
| 'feed_info'
| 'attributions';

interface BaseConfigAgency {
/**
* An array of GTFS file names (without .txt) to exclude when importing
*/
exclude?: TableNames[];
/**
* An object of HTTP headers in key:value format to use when fetching GTFS from the url specified
*/
headers?: Record<string, string>;
/**
* Settings for fetching GTFS-Realtime alerts
*/
realtimeAlerts?: {
/**
* URL for fetching GTFS-Realtime alerts
*/
url: string;
/**
* Headers to use when fetching GTFS-Realtime alerts
*/
headers?: Record<string, string>;
};
/**
* Settings for fetching GTFS-Realtime trip updates
*/
realtimeTripUpdates?: {
/**
* URL for fetching GTFS-Realtime trip updates
*/
url: string;
/**
* Headers to use when fetching GTFS-Realtime trip updates
*/
headers?: Record<string, string>;
};
/**
* Settings for fetching GTFS-Realtime vehicle positions
*/
realtimeVehiclePositions?: {
/**
* URL for fetching GTFS-Realtime vehicle positions
*/
url: string;
/**
* Headers to use when fetching GTFS-Realtime vehicle positions
*/
headers?: Record<string, string>;
};
/**
* A prefix to be added to every ID field maintain uniqueness when importing multiple GTFS from multiple agencies
*/
prefix?: string;
}

export type ConfigAgency = BaseConfigAgency &
(
| {
/**
* The URL to a zipped GTFS file. Required if path not present
*/
url: string;
}
| {
/**
* A path to a zipped GTFS file or a directory of unzipped .txt files. Required if url is not present
*/
path: string;
}
);

export interface Config {
/**
* An existing database instance to use instead of relying on node-gtfs to connect.
*/
db?: Database;
/**
* A path to an SQLite database. Defaults to using an in-memory database.
*/
sqlitePath?: string;
/**
* Amount of time in seconds to allow GTFS-Realtime data to be stored in database before allowing to be deleted.
*
* Note: is an integer
*
* @defaultValue 0
*/
gtfsRealtimeExpirationSeconds?: number;
/**
* The number of milliseconds to wait before throwing an error when downloading GTFS.
*
* Note: is an integer
*/
downloadTimeout?: number;
/**
* Options passed to `csv-parse` for parsing GTFS CSV files.
*/
csvOptions?: Options;
/**
* A path to a directory to put exported GTFS files.
*
* @defaultValue `gtfs-export/<agency_name>`
*/
exportPath?: string;
/**
* Whether or not to ignore unique constraints on ids when importing GTFS, such as `trip_id`, `calendar_id`.
*
* @defaultValue false
*/
ignoreDuplicates?: boolean;
/**
* Whether or not to ignore errors during the import process. If true, failed files will be skipped while the rest are processed.
*
* @defaultValue false
*/
ignoreErrors?: boolean;
/**
* An array of GTFS files to be imported, and which files to exclude.
*/
agencies: ConfigAgency[];
/**
* Whether or not to print output to the console.
*
* @defaulValue true
*/
verbose?: boolean;
/**
* An optional custom logger instead of the build in console.log
*
* @param message
* @returns
*/
logFunction?: (message: string) => void;
}

Expand All @@ -52,7 +186,7 @@ export interface ModelColumn {
}

export interface Model {
filenameBase: string;
filenameBase: TableNames;
filenameExtension?: string;
extension?: string;
nonstandard?: boolean;
Expand Down