feat: transform pipeline

src/backend/common/AbstractComponent.ts

@@ -9,23 +9,25 @@ import { buildTrackString, truncateStringToLength } from "../../core/StringUtils
 
 import {
     configPartsToStrongParts, countRegexes,
+    isUserStage,
     transformPlayUsingParts
 } from "../utils/PlayTransformUtils.js";
 import { hasNodeNetworkException } from "./errors/NodeErrors.js";
 import { hasUpstreamError } from "./errors/UpstreamError.js";
-import {
-    ConditionalSearchAndReplaceRegExp,
-    PlayTransformParts, PlayTransformPartsArray,
-    PlayTransformRules,
-    TRANSFORM_HOOK,
-    TransformHook
-} from "./infrastructure/Atomic.js";
 import { CommonClientConfig } from "./infrastructure/config/client/index.js";
 import { CommonSourceConfig } from "./infrastructure/config/source/index.js";
 import play = Simulate.play;
 import { WebhookPayload } from "./infrastructure/config/health/webhooks.js";
 import { AuthCheckError, BuildDataError, ConnectionCheckError, ParseCacheError, PostInitError, TransformRulesError } from "./errors/MSErrors.js";
 import { messageWithCauses, messageWithCausesTruncatedDefault } from "../utils/ErrorUtils.js";
+import {
+    ConditionalSearchAndReplaceRegExp,
+    PlayTransformParts,
+    PlayTransformPartsArray,
+    PlayTransformRules,
+    TRANSFORM_HOOK,
+    TransformHook
+} from "./infrastructure/Transform.js";
 
 export default abstract class AbstractComponent {
     requiresAuth: boolean = false;
@@ -387,17 +389,19 @@ export default abstract class AbstractComponent {
             let transformedPlay: PlayObject = play;
             const transformDetails: string[] = [];
             for(const hookItem of hook) {
-                const newTransformedPlay = transformPlayUsingParts(transformedPlay, hookItem, {
-                    logger: getLogger,
-                    regex: {
-                        searchAndReplace: this.regexCache.searchAndReplace,
-                        testMaybeRegex: this.regexCache.testMaybeRegex,
+                if(hookItem.type === 'user') {
+                    const newTransformedPlay = transformPlayUsingParts(transformedPlay, hookItem, {
+                        logger: getLogger,
+                        regex: {
+                            searchAndReplace: this.regexCache.searchAndReplace,
+                            testMaybeRegex: this.regexCache.testMaybeRegex,
+                        }
+                    });
+                    if(!deepEqual(newTransformedPlay, transformedPlay)) {
+                        transformDetails.push(buildTrackString(transformedPlay, {include: ['artist', 'track', 'album']}));
                     }
-                });
-                if(!deepEqual(newTransformedPlay, transformedPlay)) {
-                    transformDetails.push(buildTrackString(transformedPlay, {include: ['artist', 'track', 'album']}));
+                    transformedPlay = newTransformedPlay;
                 }
-                transformedPlay = newTransformedPlay;
             }
 
             if(transformDetails.length > 0) {

src/backend/common/infrastructure/Atomic.ts

@@ -1,5 +1,4 @@
 import { Logger } from '@foxxmd/logging';
-import { SearchAndReplaceRegExp } from "@foxxmd/regex-buddy-core";
 import { Dayjs } from "dayjs";
 import { Request, Response } from "express";
 import { NextFunction, ParamsDictionary, Query } from "express-serve-static-core";
@@ -299,61 +298,6 @@ export type AbstractApiOptions = Record<any, any> & { logger: Logger }
 
 export type keyOmit<T, U extends keyof any> = T & { [P in U]?: never }
 
-export interface ConditionalSearchAndReplaceRegExp extends SearchAndReplaceRegExp{
-    when?: WhenConditionsConfig
-}
-
-export type ConditionalSearchAndReplaceTerm = Omit<ConditionalSearchAndReplaceRegExp, 'test'>
-
-export type SearchAndReplaceTerm = string | ConditionalSearchAndReplaceTerm;
-
-export type PlayTransformParts<T> = PlayTransformPartsAtomic<T[]> & { when?: WhenConditionsConfig };
-
-export type PlayTransformPartsArray<T> = PlayTransformParts<T>[];
-
-export type PlayTransformPartsConfig<T> = PlayTransformPartsArray<T> | PlayTransformParts<T>;
-
-export interface PlayTransformPartsAtomic<T> {
-    title?: T
-    artists?: T
-    album?: T
-}
-
-export interface PlayTransformHooksConfig<T> {
-    preCompare?: PlayTransformPartsConfig<T>
-    compare?: {
-        candidate?: PlayTransformPartsConfig<T>
-        existing?: PlayTransformPartsConfig<T>
-    }
-    postCompare?: PlayTransformPartsConfig<T>
-}
-
-export interface PlayTransformHooks<T> extends PlayTransformHooksConfig<T> {
-    preCompare?: PlayTransformPartsArray<T>
-    compare?: {
-        candidate?: PlayTransformPartsArray<T>
-        existing?: PlayTransformPartsArray<T>
-    }
-    postCompare?: PlayTransformPartsArray<T>
-}
-
-export type PlayTransformRules = PlayTransformHooks<ConditionalSearchAndReplaceRegExp>
-
-export type TransformHook = 'preCompare' | 'compare' | 'candidate' | 'existing' | 'postCompare';
-export const TRANSFORM_HOOK = {
-    preCompare: 'preCompare' as TransformHook,
-    candidate: 'candidate' as TransformHook,
-    existing: 'existing' as TransformHook,
-    postCompare: 'postCompare' as TransformHook,
-}
-export type PlayTransformConfig = PlayTransformHooksConfig<SearchAndReplaceTerm>;
-export type PlayTransformOptions = PlayTransformConfig & { log?: boolean | 'all' }
-
-export type WhenParts<T> = PlayTransformPartsAtomic<T>;
-
-export type WhenConditions<T> = WhenParts<T>[];
-export type WhenConditionsConfig = WhenConditions<string>;
-
 export type WithRequiredProperty<Type, Key extends keyof Type> = Type & {
   [Property in Key]-?: Type[Property];
 };

src/backend/common/infrastructure/Transform.ts

@@ -0,0 +1,90 @@
+import { SearchAndReplaceRegExp } from "@foxxmd/regex-buddy-core";
+
+export interface ConditionalSearchAndReplaceRegExp extends SearchAndReplaceRegExp {
+    when?: WhenConditionsConfig
+}
+
+export type ConditionalSearchAndReplaceTerm = Omit<ConditionalSearchAndReplaceRegExp, 'test'>
+export type SearchAndReplaceTerm = string | ConditionalSearchAndReplaceTerm;
+export type ExternalMetadataTerm = boolean | undefined | { when: WhenConditionsConfig };
+
+export type PlayTransformParts<T, Y = MaybeStageTyped> = Extract<PlayTransformStage<T[]>, Y> & { when?: WhenConditionsConfig };
+export type PlayTransformUserParts<T> = PlayTransformUserStage<T[]> & { when?: WhenConditionsConfig };
+export type PlayTransformPartsArray<T, Y = MaybeStageTyped> = PlayTransformParts<T, Y>[];
+
+/** Represents the weakly-defined user config. May be an array of parts or one parts object */
+export type PlayTransformPartsConfig<T, Y = MaybeStageTyped> = PlayTransformPartsArray<T, Y> | PlayTransformParts<T, Y>;
+
+export interface PlayTransformPartsAtomic<T> {
+    title?: T
+    artists?: T
+    album?: T
+}
+
+export type StageTypeMetadata = 'spotify' | 'listenbrainz' | 'native';
+export type StageTypeUser = 'user';
+export type StageType = StageTypeMetadata | StageTypeUser;
+export const STAGE_TYPES_USER: StageTypeUser[] = ['user'];
+export const STAGE_TYPES_METADATA: StageTypeMetadata[] = ['spotify','listenbrainz','native'];
+export const STAGE_TYPES: StageType[] = [...STAGE_TYPES_METADATA, ...STAGE_TYPES_USER];
+
+export interface StageTyped {
+    type: StageType
+}
+
+export interface NotStageTyped {
+    type?: never
+}
+
+export type MaybeStageTyped = StageTyped | NotStageTyped;
+
+export interface PlayTransformStageTyped<T> extends PlayTransformPartsAtomic<T> {
+    type: StageType
+}
+
+export interface PlayTransformMetadataStage extends PlayTransformStageTyped<ExternalMetadataTerm> {
+    score?: number
+//    all?: ExternalMetadataTerm
+    type: StageTypeMetadata
+}
+
+export interface PlayTransformUserStage<T> extends PlayTransformStageTyped<T> {
+    type: StageTypeUser
+}
+export type UntypedPlayTransformUserStage<T> = Omit<PlayTransformUserStage<T>, 'type'> & {type?: never};
+
+export type PlayTransformStage<T> = PlayTransformMetadataStage | PlayTransformUserStage<T> | UntypedPlayTransformUserStage<T>;
+
+/** Represents the plain json user-configured structure (input) */
+export interface PlayTransformHooksConfig<T> {
+    preCompare?: PlayTransformPartsConfig<T>
+    compare?: {
+        candidate?: PlayTransformPartsConfig<T>
+        existing?: PlayTransformPartsConfig<T>
+    }
+    postCompare?: PlayTransformPartsConfig<T>
+}
+
+/** Represents the final, strongly-typed transform configuration used during runtime */
+export interface PlayTransformHooks<T> extends PlayTransformHooksConfig<T> {
+    preCompare?: PlayTransformPartsArray<T, StageTyped>
+    compare?: {
+        candidate?: PlayTransformPartsArray<T, StageTyped>
+        existing?: PlayTransformPartsArray<T, StageTyped>
+    }
+    postCompare?: PlayTransformPartsArray<T, StageTyped>
+}
+
+export type PlayTransformRules = PlayTransformHooks<ConditionalSearchAndReplaceRegExp>
+export type TransformHook = 'preCompare' | 'compare' | 'candidate' | 'existing' | 'postCompare';
+export const TRANSFORM_HOOK = {
+    preCompare: 'preCompare' as TransformHook,
+    candidate: 'candidate' as TransformHook,
+    existing: 'existing' as TransformHook,
+    postCompare: 'postCompare' as TransformHook,
+}
+export type PlayTransformConfig = PlayTransformHooksConfig<SearchAndReplaceTerm>;
+export type PlayTransformOptions = PlayTransformConfig & { log?: boolean | 'all' }
+export type WhenParts<T> = PlayTransformPartsAtomic<T>;
+export type WhenConditions<T> = WhenParts<T>[];
+export type WhenConditionsConfig = WhenConditions<string>;

src/backend/common/infrastructure/config/client/index.ts

@@ -1,4 +1,4 @@
-import { PlayTransformConfig, PlayTransformOptions } from "../../Atomic.js";
+import { PlayTransformConfig, PlayTransformOptions } from "../../Transform.js";
 import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js";
 
 /**

src/backend/common/infrastructure/config/source/index.ts

@@ -1,5 +1,6 @@
 import { FileLogOptions, LogLevel } from "@foxxmd/logging";
-import { PlayTransformConfig, PlayTransformOptions } from "../../Atomic.js";
+
+import { PlayTransformConfig, PlayTransformOptions } from "../../Transform.js";
 import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js";
 
 export interface SourceRetryOptions extends RequestRetryOptions {

src/backend/scrobblers/AbstractScrobbleClient.ts

@@ -25,9 +25,10 @@ import {
     ScrobbledPlayObject,
     SourceIdentifier,
     TIME_WEIGHT,
-    TITLE_WEIGHT, TRANSFORM_HOOK,
+    TITLE_WEIGHT,
 } from "../common/infrastructure/Atomic.js";
 import { CommonClientConfig, NowPlayingOptions, UpstreamRefreshOptions } from "../common/infrastructure/config/client/index.js";
+import { TRANSFORM_HOOK } from "../common/infrastructure/Transform.js";
 import { Notifiers } from "../notifier/Notifiers.js";
 import {
     comparingMultipleArtists,

src/backend/sources/AbstractSource.ts

@@ -18,9 +18,10 @@ import {
     PlayUserId,
     ProgressAwarePlayObject,
     SINGLE_USER_PLATFORM_ID,
-    SourceType, TRANSFORM_HOOK,
+    SourceType,
 } from "../common/infrastructure/Atomic.js";
 import { SourceConfig } from "../common/infrastructure/config/source/sources.js";
+import { TRANSFORM_HOOK } from "../common/infrastructure/Transform.js";
 import TupleMap from "../common/TupleMap.js";
 import {
     difference,

src/backend/sources/DeezerInternalSource.ts

@@ -2,8 +2,9 @@ import dayjs from "dayjs";
 import EventEmitter from "events";
 import request, { Request, Response, SuperAgent } from 'superagent';
 import { PlayObject, SOURCE_SOT, TA_CLOSE, TA_DURING, TA_EXACT, TA_FUZZY, TemporalAccuracy } from "../../core/Atomic.js";
-import { DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfig, TRANSFORM_HOOK } from "../common/infrastructure/Atomic.js";
+import { DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic.js";
 import { DeezerInternalSourceConfig, DeezerInternalTrackData, DeezerSourceConfig } from "../common/infrastructure/config/source/deezer.js";
+import { TRANSFORM_HOOK } from "../common/infrastructure/Transform.js";
 import { parseRetryAfterSecsFromObj, playObjDataMatch, readJson, sleep, sortByOldestPlayDate, writeFile, } from "../utils.js";
 import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource.js";
 import { CookieJar, Cookie } from 'tough-cookie';
@@ -278,4 +279,4 @@ const buildInternalUrl = (method: string, token: string = ''): URL => {
     const u = new URL(`https://www.deezer.com/ajax/gw-light.php?${params.toString()}`);
 
     return u;
-}
+}