refactor: use newest NodeSecure scanner features

Author: fraxken

README.md

@@ -19,11 +19,10 @@ I personally created this project to analyze npm packages by various criteria (p
 
 - Pull packages from the npm registry by divers criteria.
 - Offers you various methods to read and extract information from the npm tarball.
-- Include [js-x-ray](https://github.com/fraxken/js-x-ray) by default.
 - Functionalities can be extended
 
 ## Requirements
-- [Node.js](https://nodejs.org/en/) v14 or higher
+- [Node.js](https://nodejs.org/en/) v16 or higher
 
 ## Getting Started
 

bin/index.ts

@@ -1,6 +1,5 @@
 #!/usr/bin/env node
 
-import "make-promises-safe";
 import dotenv from "dotenv";
 dotenv.config();
 
@@ -11,7 +10,7 @@ import sade from "sade";
 // Import Internal Dependencies
 import * as commands from "./commands/index.js";
 
-const prog = sade("npm-security-fetcher").version("1.0.0");
+const prog = sade("npm-security-fetcher").version("2.0.0");
 console.log(
   colors.gray(
     `\n > executing npm-security-fetch at: ${colors.yellow(process.cwd())}\n`

example/js-x-ray.ts

@@ -1,125 +1,43 @@
-/* eslint-disable max-params */
-
-// Node.js
+// Import Node.js Dependencies
 import fs from "node:fs/promises";
 import path from "node:path";
 
-// Third-party
-import filenamify from "filenamify";
-
-// Internal
-import { analyzeJavaScriptFile, Utils } from "../index";
-
-// CONSTANTS
-const kJavaScriptExtensions = new Set([".js", ".mjs", ".cjs"]);
-const kJSONSpace = 2;
-const kDefaultReplacement = "#";
+// Import Third-party Dependencies
+import { tarball } from "@nodesecure/scanner";
 
-let count = 0;
+// Import Internal Dependencies
+import { IRunOptions } from "../index.js";
 
-export async function init() {
-  const baseDir = path.join(process.cwd(), "results");
+type Context = {
+  outdir: string;
+  count: number;
+};
 
-  const analysisDir = path.join(baseDir, "packages");
-  const errorDir = path.join(baseDir, "parsing-errors");
-  await fs.mkdir(analysisDir, { recursive: true });
-  await fs.mkdir(errorDir, { recursive: true });
+export async function init(): Promise<Context> {
+  const outdir = path.join(process.cwd(), "nsf-results");
+  await fs.mkdir(outdir, { recursive: true });
 
-  return { analysisDir, errorDir };
+  return { outdir, count: 0 };
 }
 
 export async function close() {
   console.log("close triggered");
 }
 
-type RunOptions = {
-  name: string;
-  location: string;
-  root: string;
-};
+export async function run(ctx: Context, options: IRunOptions) {
+  const { name, location } = options;
 
-export async function run(ctx: Ctx, { name, location, root }: RunOptions) {
   try {
-    console.log(`handle package name: ${name}, count: ${count++}`);
-    const { files } = await Utils.getTarballComposition(location);
-    const jsFiles = files.filter((name) => kJavaScriptExtensions.has(path.extname(name)));
-    const toWait: any[] = [];
-
-    for (const file of jsFiles) {
-      const cleanName = filenamify(
-        path.relative(root, file).slice(name.length),
-        { replacement: kDefaultReplacement }
-      );
-      toWait.push(runASTAnalysis(ctx, cleanName, file, name));
-    }
+    console.log(`handle package name: ${name}, count: ${ctx.count++}`);
+    const result = await tarball.scanPackage(location, name);
 
-    const results = (await Promise.allSettled(toWait))
-      .filter(({ status }) => status === "fulfilled")
-      .map((p) => (p as PromiseFulfilledResult<unknown>).value);
-
-    const content = JSON.stringify(
-      Object.assign({}, ...results),
-      null,
-      kJSONSpace
+    const fileName = path.join(ctx.outdir, name.replace(/\//g, "__")) + ".json";
+    await fs.writeFile(
+      fileName,
+      JSON.stringify(result, null, 2)
     );
-
-    const fileName =
-      path.join(ctx.analysisDir, name.replace(/\//g, "__")) + ".json";
-    await fs.writeFile(fileName, content);
   }
   finally {
     await fs.rmdir(location, { recursive: true });
   }
 }
-
-async function dumpParsingError(
-  ctx: Ctx,
-  error: { code: string; message: string; stack: string } | string,
-  cleanName: string,
-  pkgName: string
-) {
-  try {
-    const stack = typeof error === "string" ? "" : error.stack.split("\n");
-    const dumpStr = JSON.stringify(
-      {
-        pkgName,
-        code: typeof error === "string" ? null : error.code || null,
-        message: typeof error === "string" ? error : error.message || "",
-        stack
-      },
-      null,
-      kJSONSpace
-    );
-
-    await fs.writeFile(path.join(ctx.errorDir, `${cleanName}.json`), dumpStr);
-  }
-  catch {
-    // ignore
-  }
-}
-
-type Ctx = { analysisDir: string; errorDir: string };
-
-async function runASTAnalysis(
-  ctx: Ctx,
-  cleanName: string,
-  location: string,
-  pkgName: string
-) {
-  try {
-    const ASTAnalysis = await analyzeJavaScriptFile(location);
-    // @ts-ignore
-    const deps = [...ASTAnalysis.dependencies];
-    const warnings = ASTAnalysis.warnings;
-
-    return { [cleanName]: { warnings, deps } };
-  }
-  catch (error: any) {
-    if (error.name === "SyntaxError") {
-      return {};
-    }
-    await dumpParsingError(ctx, error, cleanName, pkgName);
-
-    return {};
-  }
-}

index.ts

@@ -10,13 +10,9 @@ import { search } from "@nodesecure/npm-registry-sdk";
 import { klona } from "klona/json";
 import is from "@slimio/is";
 import Locker from "@slimio/lock";
-import * as JSXRay from "@nodesecure/js-x-ray";
-
-// @ts-ignore
-import isMinified from "is-minified-code";
 
 // Import Internal Dependencies
-import { fetchPackage, getTarballComposition } from "./src/utils.js";
+import { fetchPackage } from "./src/utils.js";
 
 // CONSTANTS
 const kRegSearchLimit = 10;
@@ -27,15 +23,21 @@ const kDefaultLimit = 500;
 const kDefaultFetcher = (raw: { package: { name: string; version: number } }) => `${raw.package.name}@${raw.package.version}`;
 const kMaximumConcurrentDownload = 5;
 
-type searchPackagesByCriteriaOptions = {
+export interface IRunOptions {
+  name: string;
+  location: string;
+  root: string;
+}
+
+export interface ISearchPackagesByCriteriaOptions {
   limit?: string;
   delay?: string;
   dataFetcher?: any;
   criteria?: any;
-};
+}
 
 export async function* searchPackagesByCriteria(
-  options: searchPackagesByCriteriaOptions = {}
+  options: ISearchPackagesByCriteriaOptions = {}
 ) {
   const limit = Number(options.limit) || kDefaultLimit;
   const delay = Number(options.delay) || 0;
@@ -102,13 +104,13 @@ export async function downloadFromSource(
   }
 }
 
-type downloadPackageOnRegistryOptions = {
+export type IDownloadPackageOnRegistryOptions = {
   maxConcurrent?: string;
 };
 
 export async function* downloadPackageOnRegistry(
   source: AsyncGenerator<any, void, any>,
-  options: downloadPackageOnRegistryOptions = {}
+  options: IDownloadPackageOnRegistryOptions = {}
 ) {
   const maxConcurrent =
     Number(options.maxConcurrent) || kMaximumConcurrentDownload;
@@ -137,14 +139,3 @@ export async function* downloadPackageOnRegistry(
     await fs.rm(tmpLocation, { force: true, recursive: true });
   }
 }
-
-export async function analyzeJavaScriptFile(fileLocation: string) {
-  const str = await fs.readFile(fileLocation, "utf-8");
-  const isMin = path.basename(fileLocation).includes(".min") || isMinified(str);
-
-  return JSXRay.runASTAnalysis(str, { isMinified: isMin });
-}
-
-export const Utils = {
-  getTarballComposition
-};