diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json b/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json index 3d1394117..8c9034637 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json @@ -5,7 +5,7 @@ "url": "github.com/firebase/extensions.git", "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker" }, - "version": "1.1.38", + "version": "2.0.0", "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports", "main": "./lib/index.js", "scripts": { diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts index cc2c8d772..d6ebefeef 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import * as admin from "firebase-admin"; import * as bigquery from "@google-cloud/bigquery"; import { DocumentReference } from "firebase-admin/firestore"; import * as traverse from "traverse"; @@ -45,25 +44,9 @@ import { tableRequiresUpdate, viewRequiresUpdate } from "./checkUpdates"; import { parseErrorMessage, waitForInitialization } from "./utils"; export { RawChangelogSchema, RawChangelogViewSchema } from "./schema"; - -export interface FirestoreBigQueryEventHistoryTrackerConfig { - datasetId: string; - tableId: string; - datasetLocation?: string | undefined; - transformFunction?: string | undefined; - timePartitioning?: string | undefined; - timePartitioningField?: string | undefined; - timePartitioningFieldType?: string | undefined; - timePartitioningFirestoreField?: string | undefined; - clustering: string[] | null; - databaseId?: string | undefined; - wildcardIds?: boolean; - bqProjectId?: string | undefined; - backupTableId?: string | undefined; - useNewSnapshotQuerySyntax?: boolean; - skipInit?: boolean; - kmsKeyName?: string | undefined; -} +import type { Config } from "./types"; +import { PartitioningConfig } from "./partitioning/config"; +export type { Config } from "./types"; /** * An FirestoreEventHistoryTracker that exports data to BigQuery. @@ -80,12 +63,15 @@ export class FirestoreBigQueryEventHistoryTracker { bq: bigquery.BigQuery; _initialized: boolean = false; + partitioningConfig: PartitioningConfig; - constructor(public config: FirestoreBigQueryEventHistoryTrackerConfig) { + constructor(public config: Config) { this.bq = new bigquery.BigQuery(); this.bq.projectId = config.bqProjectId || process.env.PROJECT_ID; + this.partitioningConfig = new PartitioningConfig(this.config.partitioning); + if (!this.config.datasetLocation) { this.config.datasetLocation = "us"; } @@ -96,7 +82,7 @@ export class FirestoreBigQueryEventHistoryTracker await this.initialize(); } - const partitionHandler = new Partitioning(this.config); + const partitionHandler = new Partitioning(this.partitioningConfig); const rows = events.map((event) => { const partitionValue = partitionHandler.getPartitionValue(event); @@ -327,7 +313,7 @@ export class FirestoreBigQueryEventHistoryTracker const dataset = this.bigqueryDataset(); const table = dataset.table(changelogName); const [tableExists] = await table.exists(); - const partitioning = new Partitioning(this.config, table); + const partitioning = new Partitioning(this.partitioningConfig, table); const clustering = new Clustering(this.config, table); if (tableExists) { diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning.ts deleted file mode 100644 index 1d3a468fe..000000000 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning.ts +++ /dev/null @@ -1,368 +0,0 @@ -import { FirestoreBigQueryEventHistoryTrackerConfig } from "."; -import { FirestoreDocumentChangeEvent } from ".."; -import * as firebase from "firebase-admin"; - -import * as logs from "../logs"; -import * as bigquery from "@google-cloud/bigquery"; -import * as functions from "firebase-functions"; -import { getNewPartitionField } from "./schema"; -import { BigQuery, TableMetadata } from "@google-cloud/bigquery"; - -import { PartitionFieldType } from "../types"; - -export class Partitioning { - public config: FirestoreBigQueryEventHistoryTrackerConfig; - public table: bigquery.Table; - public schema: object; - - constructor( - config: FirestoreBigQueryEventHistoryTrackerConfig, - table?: bigquery.Table, - schema?: object - ) { - this.config = config; - this.table = table; - this.schema = schema; - } - - private isPartitioningEnabled(): boolean { - const { timePartitioning } = this.config; - - return !!timePartitioning; - } - - private isValidPartitionTypeString(value) { - return typeof value === "string"; - } - - private async metaDataSchemaFields() { - let metadata: TableMetadata; - - try { - [metadata] = await this.table.getMetadata(); - } catch { - console.log("No metadata found"); - return null; - } - - /** Return null if no valid schema on table **/ - if (!metadata.schema) return null; - - return metadata.schema.fields; - } - - private isValidPartitionTypeDate(value) { - /* Check if valid timestamp value from sdk */ - // if (value instanceof firebase.firestore.Timestamp) return true; - if (isTimestampLike(value)) return true; - - /* Check if valid date/timstemap, expedted result from production */ - if (value && value.toDate && value.toDate()) return true; - - /* Check if valid date/time value from the console, expected result from testing locally */ - return Object.prototype.toString.call(value) === "[object Date]"; - } - - private hasHourAndDatePartitionConfig() { - if ( - this.config.timePartitioning === "HOUR" && - this.config.timePartitioningFieldType === "DATE" - ) { - logs.hourAndDatePartitioningWarning(); - return true; - } - - return false; - } - - private hasValidCustomPartitionConfig() { - /* Return false if partition type option has not been set*/ - if (!this.isPartitioningEnabled()) return false; - - const { - timePartitioningField, - timePartitioningFieldType, - timePartitioningFirestoreField, - } = this.config; - - const hasNoCustomOptions = - !timePartitioningField && - !timePartitioningFieldType && - !timePartitioningFirestoreField; - /* No custom config has been set, use partition value option only */ - if (hasNoCustomOptions) return true; - - /* check if all valid combinations have been provided*/ - const hasOnlyTimestamp = - timePartitioningField === "timestamp" && - !timePartitioningFieldType && - !timePartitioningFirestoreField; - return ( - hasOnlyTimestamp || - (!!timePartitioningField && - !!timePartitioningFieldType && - !!timePartitioningFirestoreField) - ); - } - - private hasValidTimePartitionOption() { - const { timePartitioning } = this.config; - - return ["HOUR", "DAY", "MONTH", "YEAR"].includes(timePartitioning); - } - - private hasValidTimePartitionType() { - const { timePartitioningFieldType } = this.config; - - if (!timePartitioningFieldType || timePartitioningFieldType === undefined) - return true; - - return ["TIMESTAMP", "DATE", "DATETIME"].includes( - timePartitioningFieldType - ); - } - - async hasExistingSchema() { - const [metadata] = await this.table.getMetadata(); - return !!metadata.schema; - } - - hasValidTableReference() { - if (!this.table) { - logs.invalidTableReference(); - } - return !!this.table; - } - - private async isTablePartitioned() { - const [tableExists] = await this.table.exists(); - - if (!this.table || !tableExists) return false; - - /* Return true if partition metadata already exists */ - const [metadata] = await this.table.getMetadata(); - if (metadata.timePartitioning) { - logs.cannotPartitionExistingTable(this.table); - return true; - } - - /** Find schema fields **/ - const schemaFields = await this.metaDataSchemaFields(); - - /** Return false if no schema exists */ - if (!schemaFields) return false; - - /* Return false if time partition field not found */ - return schemaFields.some( - (column) => column.name === this.config.timePartitioningField - ); - } - - async isValidPartitionForExistingTable(): Promise { - /** Return false if partition type option has not been set */ - if (!this.isPartitioningEnabled()) return false; - - /* Return false if table is already partitioned */ - const isPartitioned = await this.isTablePartitioned(); - if (isPartitioned) return false; - - return this.hasValidCustomPartitionConfig(); - } - - convertDateValue(fieldValue: Date): string { - const { timePartitioningFieldType } = this.config; - - /* Return as Datetime value */ - if (timePartitioningFieldType === PartitionFieldType.DATETIME) { - return BigQuery.datetime(fieldValue.toISOString()).value; - } - - /* Return as Date value */ - if (timePartitioningFieldType === PartitionFieldType.DATE) { - return BigQuery.date(fieldValue.toISOString().substring(0, 10)).value; - } - - /* Return as Timestamp */ - return BigQuery.timestamp(fieldValue).value; - } - - /* - Extracts a valid Partition field from the Document Change Event. - Matches result based on a pre-defined Firestore field matching the event data object. - Return an empty object if no field name or value provided. - Returns empty object if not a string or timestamp (or result of serializing a timestamp) - Logs warning if not a valid datatype - Delete changes events have no data, return early as cannot partition on empty data. - **/ - getPartitionValue(event: FirestoreDocumentChangeEvent) { - if (!event.data) return {}; - - const firestoreFieldName = this.config.timePartitioningFirestoreField; - const fieldName = this.config.timePartitioningField; - const fieldValue = event.data[firestoreFieldName]; - - if (!fieldName || !fieldValue) { - return {}; - } - - if (this.isValidPartitionTypeString(fieldValue)) { - return { [fieldName]: fieldValue }; - } - - if (this.isValidPartitionTypeDate(fieldValue)) { - /* Return converted console value */ - if (isTimestampLike(fieldValue)) { - const convertedTimestampFieldValue = convertToTimestamp(fieldValue); - return { - [fieldName]: this.convertDateValue( - convertedTimestampFieldValue.toDate() - ), - }; - } - - if (fieldValue.toDate) { - return { [fieldName]: this.convertDateValue(fieldValue.toDate()) }; - } - - /* Return standard date value */ - return { [fieldName]: fieldValue }; - } - - logs.firestoreTimePartitionFieldError( - event.documentName, - fieldName, - firestoreFieldName, - fieldValue - ); - - return {}; - } - - customFieldExists(fields = []) { - /** Extract the time partioning field name */ - const { timePartitioningField } = this.config; - - /** Return based the field already exist */ - return fields.map(($) => $.name).includes(timePartitioningField); - } - - private async shouldAddPartitioningToSchema(fields: string[]): Promise<{ - proceed: boolean; - message: string; - }> { - if (!this.isPartitioningEnabled()) { - return { proceed: false, message: "Partitioning not enabled" }; - } - - if (!this.hasValidTableReference()) { - return { proceed: false, message: "Invalid table reference" }; - } - - if (!this.hasValidCustomPartitionConfig()) { - return { proceed: false, message: "Invalid partition config" }; - } - - if (!this.hasValidTimePartitionType()) { - return { proceed: false, message: "Invalid partition type" }; - } - - if (!this.hasValidTimePartitionOption()) { - return { proceed: false, message: "Invalid partition option" }; - } - - if (this.hasHourAndDatePartitionConfig()) { - return { - proceed: false, - message: "Invalid partitioning and field type combination", - }; - } - - if (this.customFieldExists(fields)) { - return { proceed: false, message: "Field already exists on schema" }; - } - - if (await this.isTablePartitioned()) { - return { proceed: false, message: "Table is already partitioned" }; - } - if (!this.config.timePartitioningField) { - return { proceed: false, message: "Partition field not provided" }; - } - return { proceed: true, message: "" }; - } - - async addPartitioningToSchema(fields = []): Promise { - const { proceed, message } = await this.shouldAddPartitioningToSchema( - fields - ); - - if (!proceed) { - functions.logger.warn(`Did not add partitioning to schema: ${message}`); - return; - } - // Add new partitioning field - fields.push(getNewPartitionField(this.config)); - functions.logger.log( - `Added new partition field: ${this.config.timePartitioningField} to table ID: ${this.table.id}` - ); - } - - async updateTableMetadata(options: bigquery.TableMetadata): Promise { - /** Return if partition type option has not been set */ - if (!this.isPartitioningEnabled()) return; - - /** Return if class has invalid table reference */ - if (!this.hasValidTableReference()) return; - - /** Return if table is already partitioned **/ - if (await this.isTablePartitioned()) return; - - /** Return if an invalid partition type has been requested**/ - if (!this.hasValidCustomPartitionConfig()) return; - - /** Return if an invalid partition type has been requested**/ - if (!this.hasValidTimePartitionType()) return; - - /** Update fields with new schema option ** */ - if (!this.hasValidTimePartitionOption()) return; - - /** Return if invalid partitioning and field type combination */ - if (this.hasHourAndDatePartitionConfig()) return; - - if (this.config.timePartitioning) { - options.timePartitioning = { type: this.config.timePartitioning }; - } - - //TODO: Add check for skipping adding views partition field, this is not a feature that can be added . - - if (this.config.timePartitioningField) { - options.timePartitioning = { - ...options.timePartitioning, - field: this.config.timePartitioningField, - }; - } - } -} - -type TimestampLike = { - _seconds: number; - _nanoseconds: number; -}; - -const isTimestampLike = (value: any): value is TimestampLike => { - if (value instanceof firebase.firestore.Timestamp) return true; - return ( - typeof value === "object" && - value !== null && - "_seconds" in value && - typeof value["_seconds"] === "number" && - "_nanoseconds" in value && - typeof value["_nanoseconds"] === "number" - ); -}; - -const convertToTimestamp = ( - value: TimestampLike -): firebase.firestore.Timestamp => { - if (value instanceof firebase.firestore.Timestamp) return value; - return new firebase.firestore.Timestamp(value._seconds, value._nanoseconds); -}; diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/config.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/config.ts new file mode 100644 index 000000000..84567b31b --- /dev/null +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/config.ts @@ -0,0 +1,163 @@ +export type TimePartitioningGranularity = "HOUR" | "DAY" | "MONTH" | "YEAR"; + +export type PartitioningFieldType = "TIMESTAMP" | "DATE" | "DATETIME"; + +export interface BasePartitioningConfig { + granularity?: TimePartitioningGranularity | "NONE" | null; + bigqueryColumnName?: string; + bigqueryColumnType?: PartitioningFieldType; + firestoreFieldName?: string; +} + +export interface NoPartitioning extends BasePartitioningConfig { + granularity?: "NONE" | null | undefined; + bigqueryColumnName?: undefined; + bigqueryColumnType?: undefined; + firestoreFieldName?: undefined; +} + +export interface IngestionTimePartitioning extends BasePartitioningConfig { + granularity: TimePartitioningGranularity; + bigqueryColumnName?: undefined; + bigqueryColumnType?: undefined; + firestoreFieldName?: undefined; +} + +export interface FirestoreTimestampPartitioning extends BasePartitioningConfig { + granularity: TimePartitioningGranularity; + bigqueryColumnName: "timestamp"; + bigqueryColumnType?: PartitioningFieldType; + firestoreFieldName?: undefined; +} + +export interface FirestoreFieldPartitioning extends BasePartitioningConfig { + granularity: TimePartitioningGranularity; + bigqueryColumnName: string; + bigqueryColumnType: PartitioningFieldType; + firestoreFieldName: string; +} + +export type PartitioningStrategy = + | NoPartitioning + | IngestionTimePartitioning + | FirestoreTimestampPartitioning + | FirestoreFieldPartitioning; + +export enum PartitioningType { + NONE = "NONE", + INGESTION_TIME = "INGESTION_TIME", + FIRESTORE_TIMESTAMP = "FIRESTORE_TIMESTAMP", + FIRESTORE_FIELD = "FIRESTORE_FIELD", +} + +export class PartitioningConfig { + private strategy: PartitioningStrategy; + private type: PartitioningType; + + constructor(config: PartitioningStrategy) { + this.strategy = config; + this.type = this.determineType(config); + } + + private determineType(config: PartitioningStrategy): PartitioningType { + if (!config.granularity || config.granularity === "NONE") { + return PartitioningType.NONE; + } + + if (!config.bigqueryColumnName && !config.firestoreFieldName) { + return PartitioningType.INGESTION_TIME; + } + + if ( + config.bigqueryColumnName === "timestamp" && + !config.firestoreFieldName + ) { + return PartitioningType.FIRESTORE_TIMESTAMP; + } + + if ( + config.bigqueryColumnName && + config.bigqueryColumnType && + config.firestoreFieldName + ) { + return PartitioningType.FIRESTORE_FIELD; + } + + return PartitioningType.NONE; + } + + getType(): PartitioningType { + return this.type; + } + + getStrategy(): PartitioningStrategy { + return this.strategy; + } + + isNoPartitioning(): boolean { + return this.type === PartitioningType.NONE; + } + + isIngestionTimePartitioning(): boolean { + return this.type === PartitioningType.INGESTION_TIME; + } + + isFirestoreTimestampPartitioning(): boolean { + return this.type === PartitioningType.FIRESTORE_TIMESTAMP; + } + + isFirestoreFieldPartitioning(): boolean { + return this.type === PartitioningType.FIRESTORE_FIELD; + } + + getGranularity(): TimePartitioningGranularity | "NONE" | null | undefined { + return this.strategy.granularity; + } + + getBigQueryColumnName(): string | undefined { + return this.strategy.bigqueryColumnName; + } + + getBigQueryColumnType(): PartitioningFieldType | undefined { + return this.strategy.bigqueryColumnType; + } + + getFirestoreFieldName(): string | undefined { + return this.strategy.firestoreFieldName; + } + + static none(): PartitioningConfig { + return new PartitioningConfig({ granularity: "NONE" }); + } + + static ingestionTime( + granularity: TimePartitioningGranularity + ): PartitioningConfig { + return new PartitioningConfig({ granularity }); + } + + static firestoreTimestamp( + granularity: TimePartitioningGranularity, + columnType?: PartitioningFieldType + ): PartitioningConfig { + return new PartitioningConfig({ + granularity, + bigqueryColumnName: "timestamp", + bigqueryColumnType: columnType, + }); + } + + static firestoreField( + granularity: TimePartitioningGranularity, + bigqueryColumnName: string, + bigqueryColumnType: PartitioningFieldType, + firestoreFieldName: string + ): PartitioningConfig { + return new PartitioningConfig({ + granularity, + bigqueryColumnName, + bigqueryColumnType, + firestoreFieldName, + }); + } +} diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts new file mode 100644 index 000000000..c1a6a4849 --- /dev/null +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts @@ -0,0 +1,43 @@ +import * as firebase from "firebase-admin"; +import { BigQuery } from "@google-cloud/bigquery"; + +export class PartitionValueConverter { + constructor(private fieldType: "TIMESTAMP" | "DATE" | "DATETIME") {} + + private isTimestampLike( + value: any + ): value is { _seconds: number; _nanoseconds: number } { + return ( + value !== null && + typeof value === "object" && + typeof value._seconds === "number" && + typeof value._nanoseconds === "number" + ); + } + + convert(value: unknown): string | null { + let date: Date; + + if (value instanceof firebase.firestore.Timestamp) { + date = value.toDate(); + } else if (this.isTimestampLike(value)) { + date = new firebase.firestore.Timestamp( + value._seconds, + value._nanoseconds + ).toDate(); + } else if (value instanceof Date && !isNaN(value.getTime())) { + date = value; + } else { + return null; + } + + switch (this.fieldType) { + case "DATETIME": + return BigQuery.datetime(date.toISOString()).value; + case "DATE": + return BigQuery.date(date.toISOString().substring(0, 10)).value; + case "TIMESTAMP": + return BigQuery.timestamp(date).value; + } + } +} diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/index.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/index.ts new file mode 100644 index 000000000..39cfa06d8 --- /dev/null +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/index.ts @@ -0,0 +1,166 @@ +import * as bigquery from "@google-cloud/bigquery"; +import * as functions from "firebase-functions"; +import { TableField } from "@google-cloud/bigquery"; +import { ChangeType, FirestoreDocumentChangeEvent } from "../.."; +import * as logs from "../../logs"; +import { PartitioningConfig } from "./config"; +import { PartitionValueConverter } from "./converter"; + +export class Partitioning { + public readonly config: PartitioningConfig; + public table?: bigquery.Table; + public schema?: object; + + constructor( + partitioningConfig: PartitioningConfig, + table?: bigquery.Table, + schema?: object + ) { + this.config = partitioningConfig; + this.table = table; + this.schema = schema; + } + + private getNewPartitionField() { + if ( + !this.config.isFirestoreFieldPartitioning() && + !this.config.isFirestoreTimestampPartitioning() + ) { + return null; + } + + const columnName = this.config.getBigQueryColumnName(); + if (!columnName) { + return null; + } + + const columnType = this.config.getBigQueryColumnType() || "TIMESTAMP"; + + return { + name: columnName, + mode: "NULLABLE", + type: columnType, + description: + "The document TimePartition partition field selected by user", + }; + } + + getPartitionValue( + event: FirestoreDocumentChangeEvent + ): Record { + if (!this.config.isFirestoreFieldPartitioning()) { + return {}; + } + + const data = + event.operation === ChangeType.DELETE ? event.oldData : event.data; + if (!data) return {}; + + const fieldName = this.config.getBigQueryColumnName(); + const firestoreFieldName = this.config.getFirestoreFieldName(); + const fieldType = this.config.getBigQueryColumnType(); + + if (!fieldName || !firestoreFieldName || !fieldType) { + return {}; + } + + const fieldValue = data[firestoreFieldName]; + if (fieldValue === undefined) return {}; + + const converter = new PartitionValueConverter(fieldType); + const convertedValue = converter.convert(fieldValue); + + if (convertedValue === null) { + logs.firestoreTimePartitionFieldError( + event.documentName, + fieldName, + firestoreFieldName, + fieldValue + ); + return {}; + } + + return { [fieldName]: convertedValue }; + } + + async isTablePartitioned(): Promise { + if (!this.table) return false; + const [metadata] = await this.table.getMetadata(); + return !!metadata.timePartitioning; + } + + async isValidPartitionForExistingTable(): Promise { + if (this.config.isNoPartitioning()) return false; + + const isPartitioned = await this.isTablePartitioned(); + return !isPartitioned; + } + + async updateTableMetadata(options: bigquery.TableMetadata): Promise { + if (this.config.isNoPartitioning()) return; + if (!this.table) return; + + const [metadata] = await this.table.getMetadata(); + if (metadata.timePartitioning) { + logs.cannotPartitionExistingTable(this.table); + return; + } + + if (this.hasHourAndDatePartitionConfig()) return; + + const granularity = this.config.getGranularity(); + + if (this.config.isIngestionTimePartitioning()) { + if (granularity && granularity !== "NONE") { + options.timePartitioning = { type: granularity }; + } + } else if ( + this.config.isFirestoreTimestampPartitioning() || + this.config.isFirestoreFieldPartitioning() + ) { + const columnName = this.config.getBigQueryColumnName(); + if (granularity && granularity !== "NONE" && columnName) { + options.timePartitioning = { + type: granularity, + field: columnName, + }; + } + } + } + + async addPartitioningToSchema(fields: TableField[]): Promise { + if ( + !this.config.isFirestoreFieldPartitioning() && + !this.config.isFirestoreTimestampPartitioning() + ) { + return; + } + + const newField = this.getNewPartitionField(); + + if (!newField) { + return; + } + + if (fields.some((field) => field.name === newField.name)) { + return; + } + + fields.push(newField); + functions.logger.log( + `Added new partition field: ${newField.name} to schema for table.` + ); + } + + private hasHourAndDatePartitionConfig(): boolean { + if ( + this.config.isFirestoreFieldPartitioning() && + this.config.getGranularity() === "HOUR" && + this.config.getBigQueryColumnType() === "DATE" + ) { + logs.hourAndDatePartitioningWarning(); + return true; + } + return false; + } +} diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/schema.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/schema.ts index ea45aa4a8..f62ef7785 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/schema.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/schema.ts @@ -14,8 +14,6 @@ * limitations under the License. */ -import { FirestoreBigQueryEventHistoryTrackerConfig } from "."; - export type BigQueryFieldMode = "NULLABLE" | "REPEATED" | "REQUIRED"; export type BigQueryFieldType = | "BOOLEAN" @@ -182,17 +180,3 @@ export const RawChangelogSchema = { documentIdField, ], }; - -// Helper function for Partitioned Changelogs field -export const getNewPartitionField = ( - config: FirestoreBigQueryEventHistoryTrackerConfig -) => { - const { timePartitioningField, timePartitioningFieldType } = config; - - return { - name: timePartitioningField, - mode: "NULLABLE", - type: timePartitioningFieldType, - description: "The document TimePartition partition field selected by user", - }; -}; diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/types.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/types.ts new file mode 100644 index 000000000..9a21aef6c --- /dev/null +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/types.ts @@ -0,0 +1,27 @@ +import { PartitioningStrategy } from "./partitioning/config"; + +/** + * Base configuration for all variants. Includes all parameters + * that are not dependent on the partitioning strategy. + */ +export interface Config { + datasetId: string; + tableId: string; + firestoreInstanceId?: string; + datasetLocation?: string; + transformFunction?: string; + partitioning?: PartitioningStrategy; + clustering?: string[] | null; + databaseId?: string; + wildcardIds?: boolean; + bqProjectId?: string; + backupTableId?: string; + useNewSnapshotQuerySyntax?: boolean; + skipInit?: boolean; + kmsKeyName?: string; + useMaterializedView?: boolean; + useIncrementalMaterializedView?: boolean; + maxStaleness?: string; + refreshIntervalMinutes?: number; + logLevel?: "debug" | "info" | "warn" | "error" | "silent"; +}