From 47ba554f55709defea67f47992813370ce853ef2 Mon Sep 17 00:00:00 2001 From: POOJA DONODE Date: Wed, 28 May 2025 17:23:30 +0530 Subject: [PATCH] RTDEV-577 - Archive-old-artifacts-worker-sample-for-deprecated-plugin --- .../archive-old-artifact/README.md | 44 +++ .../archive-old-artifact/manifest.json | 9 + .../archive-old-artifact/package.json | 40 ++ .../archive-old-artifact/payload-example.json | 14 + .../archive-old-artifact/tsconfig.json | 15 + .../archive-old-artifact/types.ts | 37 ++ .../archive-old-artifact/worker.spec.ts | 63 +++ .../archive-old-artifact/worker.ts | 373 ++++++++++++++++++ 8 files changed, 595 insertions(+) create mode 100644 samples/artifactory/GENERIC_EVENT/archive-old-artifact/README.md create mode 100644 samples/artifactory/GENERIC_EVENT/archive-old-artifact/manifest.json create mode 100644 samples/artifactory/GENERIC_EVENT/archive-old-artifact/package.json create mode 100644 samples/artifactory/GENERIC_EVENT/archive-old-artifact/payload-example.json create mode 100644 samples/artifactory/GENERIC_EVENT/archive-old-artifact/tsconfig.json create mode 100644 samples/artifactory/GENERIC_EVENT/archive-old-artifact/types.ts create mode 100644 samples/artifactory/GENERIC_EVENT/archive-old-artifact/worker.spec.ts create mode 100644 samples/artifactory/GENERIC_EVENT/archive-old-artifact/worker.ts diff --git a/samples/artifactory/GENERIC_EVENT/archive-old-artifact/README.md b/samples/artifactory/GENERIC_EVENT/archive-old-artifact/README.md new file mode 100644 index 0000000..6d7efa2 --- /dev/null +++ b/samples/artifactory/GENERIC_EVENT/archive-old-artifact/README.md @@ -0,0 +1,44 @@ +Artifactory Archive Old Artifacts User Worker +This worker is used to archive artifacts from a given source repository in Artifactory to a given destination repository. The artifacts are chosen based on a mixture of available parameters. + +Note that this worker will delete your artifacts. The archive process is designed to preserve the name, path, and properties of an artifact, but save disk space by deleting the file contents. This worker is to be used for build artifacts that are no longer needed, when it's still useful to keep the build around for auditing or history purposes. + + +Features +Re-deploys artifacts that are to be archived, to save disk space. +Archived artifacts are moved to an archive repository, to be separate from non-archived artifacts. +Archived artifacts retain all properties that were set, and are also tagged with the archival timestamp. +Input Parameters +filePattern - the file pattern to match against in the source repository +srcRepo - the source repository to scan for artifacts to be archived +archiveRepo - the repository where matching artifacts are archived to +archiveProperty - the name of the property to use when tagging the archived artifact with the archive timestamp +Available 'time period' archive policies: +lastModified - the last time the artifact was modified +lastUpdated - the last time the artifact was updated +created - the creation date of the artifact +lastDownloaded - the last time the artifact was downloaded +age - the age of the artifact +NOTE: the time period archive policies are all specified in number of days + +Available 'property' archive policies: +includePropertySet - the artifact will be archived if it possesses all of the passed in properties +excludePropertySet - the artifact will not be archived if it possesses all of the passed in properties +NOTE: property set format ⇒ prop[:value1[;prop2[:value2]......[;propN[:valueN]]]) + +A property key must be provided, but a corresponding value is not necessary. If a property is set without a value, then a check is made for just the key. + +Available artifact keep policy: +numKeepArtifacts - the number of artifacts to keep per directory +NOTE: This allows one to keep X number of artifacts (based on natural directory sort per directory). So, if your artifacts are laid out in a flat directory structure, you can keep the last X artifacts in each directory with this setting. + +One can set any number of 'time period' archive policies as well as any number of include and exclude attribute sets. It is up to the caller to decide how best to archive artifacts. If no archive policy parameters are sent in, the plugin aborts in order to not allow default deleting of every artifact. + +Archive Process +The 'archive' process performs the following: + +Grabs all of the currently set properties on the artifact +Does a deploy over top of the artifact, to conserve space +Adds all of the previously held attributes to the newly deployed artifact +Moves the artifact from the source repository to the destination repository specified +Adds a property containing the archive timestamp to the artifact \ No newline at end of file diff --git a/samples/artifactory/GENERIC_EVENT/archive-old-artifact/manifest.json b/samples/artifactory/GENERIC_EVENT/archive-old-artifact/manifest.json new file mode 100644 index 0000000..bfc40bf --- /dev/null +++ b/samples/artifactory/GENERIC_EVENT/archive-old-artifact/manifest.json @@ -0,0 +1,9 @@ +{ + "name": "archive-old-artifact", + "description": "This worker script is designed to archive old artifacts in Artifactory based on configurable criteria. It is useful for managing storage and maintaining repository hygiene.", + "secrets": {}, + "sourceCodePath": "./worker.ts", + "action": "GENERIC_EVENT", + "enabled": false, + "debug": true +} \ No newline at end of file diff --git a/samples/artifactory/GENERIC_EVENT/archive-old-artifact/package.json b/samples/artifactory/GENERIC_EVENT/archive-old-artifact/package.json new file mode 100644 index 0000000..188d9cb --- /dev/null +++ b/samples/artifactory/GENERIC_EVENT/archive-old-artifact/package.json @@ -0,0 +1,40 @@ +{ + "name": "archive-old-artifact", + "description": "Run a script on GENERIC_EVENT", + "version": "1.0.1", + "scripts": { + "deploy": "jf worker deploy", + "undeploy": "jf worker rm \"archive-old-artifact\"", + "test": "jest" + }, + "license": "ISC", + "devDependencies": { + "jfrog-workers": "^0.4.0", + "@golevelup/ts-jest": "^0.4.0", + "@types/jest": "^29.5.12", + "jest": "^29.7.0", + "jest-jasmine2": "^29.7.0", + "ts-jest": "^29.1.2" + }, + "jest": { + "moduleFileExtensions": [ + "ts", + "js" + ], + "rootDir": ".", + "testEnvironment": "node", + "clearMocks": true, + "maxConcurrency": 1, + "testRegex": "\\.spec\\.ts$", + "moduleDirectories": ["node_modules"], + "collectCoverageFrom": [ + "**/*.ts" + ], + "coverageDirectory": "../coverage", + "transform": { + "^.+\\.(t|j)s$": "ts-jest" + }, + "testRunner": "jest-jasmine2", + "verbose": true + } +} diff --git a/samples/artifactory/GENERIC_EVENT/archive-old-artifact/payload-example.json b/samples/artifactory/GENERIC_EVENT/archive-old-artifact/payload-example.json new file mode 100644 index 0000000..5929f0f --- /dev/null +++ b/samples/artifactory/GENERIC_EVENT/archive-old-artifact/payload-example.json @@ -0,0 +1,14 @@ +{ + "filePattern": "example.txt", + "srcRepo": "libs-release-local", + "archiveRepo": "archive-repo", + "lastModifiedDays": 30, + "lastUpdatedDays": 0, + "createdDays": 1, + "lastDownloadedDays": 60, + "ageDays": 30, + "excludePropertySet": "keeper:true", + "includePropertySet": "archive:true", + "archiveProperty": "archived.timestamp", + "numKeepArtifacts": 5 +} diff --git a/samples/artifactory/GENERIC_EVENT/archive-old-artifact/tsconfig.json b/samples/artifactory/GENERIC_EVENT/archive-old-artifact/tsconfig.json new file mode 100644 index 0000000..81d7a59 --- /dev/null +++ b/samples/artifactory/GENERIC_EVENT/archive-old-artifact/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "module": "commonjs", + "declaration": true, + "target": "es2017", + "skipLibCheck": true, + "forceConsistentCasingInFileNames": false, + "noFallthroughCasesInSwitch": false, + "allowJs": true + }, + "include": [ + "**/*.ts", + "node_modules/@types/**/*.d.ts" + ] +} diff --git a/samples/artifactory/GENERIC_EVENT/archive-old-artifact/types.ts b/samples/artifactory/GENERIC_EVENT/archive-old-artifact/types.ts new file mode 100644 index 0000000..7ff8453 --- /dev/null +++ b/samples/artifactory/GENERIC_EVENT/archive-old-artifact/types.ts @@ -0,0 +1,37 @@ +export interface PropertyMap { + [key: string]: string; +} + +export interface Checksums { + sha1: string; + md5: string; + sha256: string; +} + +export interface Artifact { + repo: string; + path: string; + created: string; + createdBy: string; + lastModified: string; + modifiedBy: string; + lastUpdated: string; + downloadUri: string; + mimeType: string; + size: string; + checksums: Checksums; + originalChecksums: Checksums; + uri: string; +} + +export interface ArtifactResponse { + artifact: Artifact; +} + +export interface ArtifactStatistics { + uri: string; + downloadCount: number; + lastDownloaded: number; + remoteDownloadCount: number; + remoteLastDownloaded: number; +} diff --git a/samples/artifactory/GENERIC_EVENT/archive-old-artifact/worker.spec.ts b/samples/artifactory/GENERIC_EVENT/archive-old-artifact/worker.spec.ts new file mode 100644 index 0000000..87055ef --- /dev/null +++ b/samples/artifactory/GENERIC_EVENT/archive-old-artifact/worker.spec.ts @@ -0,0 +1,63 @@ +import { PlatformContext } from 'jfrog-workers'; +import archiveOldArtifacts from './worker'; + +describe('archiveOldArtifacts', () => { + let context: PlatformContext; + + beforeEach(() => { + context = { + clients: { + platformHttp: { + get: jest.fn(), + post: jest.fn(), + put: jest.fn(), + }, + }, + } as unknown as PlatformContext; + }); + + + it('should throw an error when archiveRepo is missing', async () => { + const params = { + filePattern:"example.txt", + srcRepo: "source-repo", + archiveRepo:"", + lastModifiedDays:30, + lastUpdatedDays:80, + createdDays:102, + lastDownloadedDays:20, + ageDays:102, + excludePropertySet: "keeper:true", + includePropertySet: "archive:true", + archiveProperty: "archived.timestamp", + numKeepArtifacts: 5 + }; + + await expect(() => archiveOldArtifacts(context, params)).rejects.toThrow( + 'Both srcRepo and archiveRepo must be defined, srcRepo: source-repo, archiveRepo: ' + ); + }); + + it('should throw an error when any of the day is missing', async () => { + const params = { + filePattern: "example.txt", + srcRepo: "source-repo", + archiveRepo: "archive-repo", + lastModifiedDays: 0, + lastUpdatedDays: 0, + createdDays: 0, + lastDownloadedDays: 0, + ageDays: 0, + excludePropertySet: "", + includePropertySet: "", + archiveProperty: "archived.timestamp", + numKeepArtifacts: 5 + }; + + await expect(archiveOldArtifacts(context, params)).rejects.toThrow( + 'No selection criteria specified!' + ); + }); + + +}); \ No newline at end of file diff --git a/samples/artifactory/GENERIC_EVENT/archive-old-artifact/worker.ts b/samples/artifactory/GENERIC_EVENT/archive-old-artifact/worker.ts new file mode 100644 index 0000000..45814b6 --- /dev/null +++ b/samples/artifactory/GENERIC_EVENT/archive-old-artifact/worker.ts @@ -0,0 +1,373 @@ +import {PlatformContext} from 'jfrog-workers'; +import {ArtifactResponse, ArtifactStatistics, PropertyMap} from './types'; + +const ArchiveConstants = { + DAYS_TO_MILLIS: 24 * 60 * 60 * 1000, // 24 hours * 60 minutes * 60 seconds * 1000 milliseconds +}; + + +export default async function archiveOldArtifacts(context: PlatformContext, params: { + filePattern?: string; + srcRepo: string; + archiveRepo: string; + lastModifiedDays?: number; + lastUpdatedDays?: number; + createdDays?: number; + lastDownloadedDays?: number; + ageDays?: number; + excludePropertySet?: string; + includePropertySet?: string; + archiveProperty?: string; + numKeepArtifacts?: number; +}): Promise { + const { + filePattern = '*', + srcRepo, + archiveRepo, + lastModifiedDays = 0, + lastUpdatedDays = 0, + createdDays = 0, + lastDownloadedDays = 0, + ageDays = 0, + excludePropertySet = '', + includePropertySet = '', + archiveProperty = 'archived.timestamp', + numKeepArtifacts = 0, + } = params; + if (lastModifiedDays === 0 && + lastUpdatedDays === 0 && + createdDays === 0 && + lastDownloadedDays === 0 && + ageDays === 0 && + excludePropertySet === '' && + includePropertySet === '') { + console.error('No selection criteria specified, exiting now!'); + throw new Error('No selection criteria specified!'); + } + + if (!archiveRepo || !srcRepo) { + const errmsg = `Both srcRepo and archiveRepo must be defined, srcRepo: ${srcRepo}, archiveRepo: ${archiveRepo}`; + console.error(errmsg); + throw new Error(errmsg); + } + + try { + const searchResults = await searchArtifacts(context, filePattern, srcRepo); + for (const artifact of searchResults) { + console.info(`Search found artifact: ${filePattern}`); + + const storageIndex = artifact.uri.indexOf('/storage/') + '/storage/'.length; + const filePath = artifact.uri.substring(storageIndex); + const todayTime = Date.now(); + + let archiveTiming: boolean = true; + let archiveExcludeProperties: boolean = true; + let archiveIncludeProperties: boolean = true; + let artifactsArchived: number = 0; + + const itemInfo = await getItemInfo(context, srcRepo, filePath); + console.log(`Artifact: ${filePath} , item info: ${JSON.stringify(itemInfo)}`); + + if (lastModifiedDays !== 0 || + lastUpdatedDays !== 0 || + createdDays !== 0 || + lastDownloadedDays !== 0 || + ageDays !== 0) { + console.info('We are going to perform a timing policies check...'); + + const archiveTiming = checkArchiveTimingPolicies( + context, + itemInfo, + lastModifiedDays, + lastUpdatedDays, + createdDays, + lastDownloadedDays, + ageDays, + todayTime, + srcRepo, + filePath + ); + } + if (excludePropertySet !== '') { + console.info('We are going to exclude artifacts based on attributes...'); + const excludeMap = translatePropertiesString(excludePropertySet); + console.info('About to call verify properties for false'); + archiveExcludeProperties = await verifyProperties(context, srcRepo, filePath, excludeMap, false); + } + if (includePropertySet !== '') { + console.info('We are going to include artifacts based on attributes...'); + const includeMap = translatePropertiesString(includePropertySet); + console.info('About to call verify properties for true'); + archiveIncludeProperties = await verifyProperties(context, srcRepo, filePath, includeMap, true); + } + + if (archiveTiming && archiveExcludeProperties && archiveIncludeProperties) { + const properties = await getArtifactProperties(context, srcRepo, filePath); + const deployFileResponse = await deployArtifact(context, srcRepo, filePath, filePattern); + const propertiesJsonString = JSON.stringify(properties.data.properties); + const propertiesInfo = JSON.parse(propertiesJsonString); + const deployFileJsonString = JSON.stringify(deployFileResponse); + const deployFileJsonInfo = JSON.parse(deployFileJsonString); + for (const key in propertiesInfo) { + if (propertiesInfo.hasOwnProperty(key)) { + await setArtifactProperties(context, srcRepo, deployFileJsonInfo.data.repo + deployFileJsonInfo.data.path, {[key]: propertiesInfo[key]}); + + + } + } + await moveArtifact(context, deployFileJsonInfo.data.repo + deployFileJsonInfo.data.path, archiveRepo); + await setArtifactProperties(context, archiveRepo, archiveRepo + "/" + filePattern, {["archiveProperty"]: [archiveProperty]}); + artifactsArchived++; + } else { + console.log('Not archiving artifact: ' + filePath) + console.log('Timing archive policy status: ' + archiveTiming) + console.log('Exclude properties policy status: ' + archiveExcludeProperties) + console.log('Include properties policy status: ' + archiveIncludeProperties) + } + console.log('Process archived ' + artifactsArchived + ' total artifact(s)'); + } + + } catch (Error) { + console.log("Artifactory archieved failed"); + } +} + +export async function searchArtifacts(context: PlatformContext, filePattern: string, srcRepo: string): Promise { + const apiUrl = `artifactory/api/search/artifact?name=${filePattern}&repos=${srcRepo}`; + const response = await context.clients.platformHttp.get(apiUrl); + + if (response.status !== 200) { + throw new Error(`Error fetching artifacts: ${response.data}`); + } + return response.data.results; +} + +export async function getItemInfo(context: PlatformContext, repoKey: string, filePath: string): Promise { + const apiUrl = `artifactory/api/storage/${filePath}`; + const response = await context.clients.platformHttp.get(apiUrl); + if (response.status !== 200) { + throw new Error(`Error fetching file information: ${response.data}`); + } + return response.data; +} + +function getCompareDays(todayTime: number, policyTime: number): number { + return (todayTime - policyTime) / ArchiveConstants.DAYS_TO_MILLIS; +} + +function checkTimingPolicy(compareDays: number, days: number, artifact: string, policyName: string): boolean { + if (compareDays >= days) { + console.log(artifact + ' passed the ' + policyName + ' policy check (' + days + ' days)'); + return true; + } + console.log(artifact + ' did not pass the ' + policyName + ' policy check (' + days + ' days)'); + return false; +} + +export function checkArchiveTimingPolicies( + context: PlatformContext, + itemInfo: object, + lastModifiedDays: number, + lastUpdatedDays: number, + createdDays: number, + lastDownloadedDays: number, + ageDays: number, + todayTime: number, + srcRepo: string, + filePath: string +): boolean { + let compareDays: number; + const itemInfoJsonString = JSON.stringify(itemInfo, null, 2); // Pretty print with 2-space indentation + const parsedItemInfo = JSON.parse(itemInfoJsonString); + if (lastModifiedDays !== 0) { + const lastModifiedTime = new Date(parsedItemInfo.lastModified); + console.log(`${filePath} was last modified: ${lastModifiedTime}`); + compareDays = getCompareDays(todayTime, lastModifiedTime.getTime()); + console.log(`${filePath} days since last modified: ${compareDays}`); + if (!checkTimingPolicy(compareDays, lastModifiedDays, filePath, 'last modified')) { + return false; + } + } + if (lastUpdatedDays !== 0) { + const lastUpdatedTime = new Date(parsedItemInfo.lastUpdated); + console.log(filePath + ' was last updated: ' + lastUpdatedTime); + compareDays = getCompareDays(todayTime, lastUpdatedTime.getTime()); + if (!checkTimingPolicy(compareDays, createdDays, filePath, 'last updated')) { + return false; + } + } + if (createdDays !== 0) { + const createdTime = new Date(parsedItemInfo.created).getTime(); // Convert to milliseconds + console.log(filePath + ' was created: ' + createdTime); + const todayTime = new Date().getTime(); // Get current time in milliseconds + const compareDays = getCompareDays(todayTime, createdTime); + console.log(filePath + ' days since created: ' + compareDays); + if (!checkTimingPolicy(compareDays, createdDays, filePath, 'created')) { + return false; + } + } + + if (lastDownloadedDays !== 0) { + const statsInfo = getFileStatistics(context, srcRepo, filePath); + if (statsInfo == null) { + const statsInfoJsonString = JSON.stringify(statsInfo, null, 2); // Pretty print with 2-space indentation + const parsedStatsInfo = JSON.parse(statsInfoJsonString); + console.log('Artifact ' + filePath + ' stats info: ' + statsInfo); + const lastDownloadedTime = new Date(parsedStatsInfo.lastDownloaded); + compareDays = getCompareDays(todayTime, lastDownloadedTime.getTime()); + if (!checkTimingPolicy(compareDays, lastDownloadedDays, filePath, 'last downloaded')) { + return false; + } + + } + } + if (ageDays !== 0) { + const compareTime = new Date(); + const fileCreationDate = new Date(parsedItemInfo.created); + const fileAge = compareTime.getTime() - fileCreationDate.getTime(); + compareDays = fileAge / ArchiveConstants.DAYS_TO_MILLIS; + if (!checkTimingPolicy(compareDays, ageDays, filePath, 'age')) { + return false; + } + } + return true; +} + +export async function getFileStatistics(context: PlatformContext, repoKey: string, artifactPath: string): Promise { + const subPath = artifactPath.split('/')[1]?.split('/').slice(0, 2).join('/') || ''; + const apiUrl = `artifactory/api/storage/${repoKey}/${subPath}?stats`; + const response = await context.clients.platformHttp.get(apiUrl); + if (response.status !== 200) { + throw new Error(`Error fetching file statistics: ${response.data}`); + } + console.log(`Fetched statistics for file: ${subPath}`); + return response.data; +} + +async function verifyProperties( + context: PlatformContext, + srcRepo: string, + filePath: string, + propertyMap: PropertyMap, + inclusive: boolean +): Promise { + const response = await getArtifactProperties(context, srcRepo, filePath); + const propertiesInfoJsonString = JSON.stringify(response.data.properties); + const propertiesInfo = JSON.parse(propertiesInfoJsonString); + if (response.status !== 200) { + throw new Error(`Error fetching properties for artifact: ${propertiesInfo}`); + } + console.log("Got properties for artifact: " + propertiesInfoJsonString); + + for (const key in propertyMap) { + if (propertiesInfo != null && propertiesInfo[key]) { + const value = propertyMap[key]; + if (value !== undefined) { + const valueSet = propertiesInfo[key]; + + if (valueSet.includes(value)) { + console.log('Both have key:' + key + ' value:' + value); + } else { + console.log('Both have key:' + key + ' but values differ. Value checked:' + value); + return !inclusive; + } + } else { + console.log('We were not given a value for the provided key: ' + key + ' this is a match since the key matches.'); + } + } else { + console.log("The artifact did not contain the key: " + key); + return !inclusive; + } + } + return true; +} + +function translatePropertiesString(properties: string): PropertyMap { + const regex = /(\w.+)(:\w.)*(;(\w.+)(:\w.)*)*/; + + if (regex.test(properties)) { + console.log(`Properties are of the proper format! Properties: ${properties}`); + } else { + console.error(`Properties are not of the proper format: ${properties}. Exiting now!`); + throw new Error('Incorrect format for properties!'); + } + const map: PropertyMap = {}; + + const propertySets = properties.split(';'); // Use split instead of tokenize + propertySets.forEach(set => { + const [key, value] = set.split(':'); + map[key] = value !== undefined ? value : ''; + }); + + return map; +} + +export async function getArtifactProperties( + context: PlatformContext, + srcRepo: string, + filePath: string, +): Promise { + const apiUrl = `artifactory/api/storage/${filePath}?properties`; + const response = await context.clients.platformHttp.get(apiUrl); + + if (response.status !== 200) { + throw new Error(`Error fetching artifact properties: ${response.data}`); + } + return response; +} + +export async function setArtifactProperties( + context: PlatformContext, + repoKey: string, + itemPath: string, + properties: Record, + recursive: boolean = true +): Promise { + const key = Object.keys(properties)[0]; + const value = properties[key][0]; + const propertiesString = key + "=" + value + const apiUrl = 'artifactory/api/storage/' + itemPath + '?properties=' + propertiesString; + try { + const response = await context.clients.platformHttp.put(apiUrl); + if (response.status !== 204) { + throw new Error('Error setting properties'); + } + } catch (error) { + console.error('Error during setArtifactProperties:') + } + +} + +export async function deployArtifact( + context: PlatformContext, + repoKey: string, + itemPath: string, + filePattern: string +): Promise { + const apiUrl = "artifactory/" + repoKey + "/archived/" + filePattern; + try { + const response = await context.clients.platformHttp.put(apiUrl); + if (response.status !== 201) { + throw new Error("Error deploying artifact:" + itemPath); + } + console.log("Artifact deployed successfully to:" + itemPath); + return response; + } catch (error) { + console.error("Error during artifact deployment:"); + throw error; + } + +} + +export async function moveArtifact( + context: PlatformContext, + src: string, + destination: string, +): Promise { + const apiUrl = "artifactory/api/move/" + src + "?to=/" + destination; + try { + const response = await context.clients.platformHttp.post(apiUrl); + } catch (error) { + throw error; + } +} \ No newline at end of file