Skip to content

Commit af964b8

Browse files
max-leuthaeusermax-leuthaeuser
authored
Added --exclude-regex and --exclude-file (#24)
1 parent ddbd431 commit af964b8

File tree

7 files changed

+175
-19
lines changed

7 files changed

+175
-19
lines changed

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@joernio/astgen",
3-
"version": "3.20.0",
3+
"version": "3.21.0",
44
"description": "Generate JS/TS AST in json format with Babel",
55
"exports": "./index.js",
66
"keywords": [

‎src/Defaults.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@ export const IGNORE_DIRS: string[] = [
3131
"build",
3232
];
3333

34-
export const IGNORE_FILE_PATTERN: RegExp = new RegExp("(conf|test|spec|[.-]min|\\.d)\\.(js|ts|jsx|tsx)$", "i");
34+
export const IGNORE_FILE_PATTERN: RegExp =
35+
new RegExp("(chunk-vendors|app~|mock|e2e|conf|test|spec|[.-]min|\\.d)\\.(js|ts|jsx|tsx)$", "i");
36+
3537
export const MAX_LOC_IN_FILE: number = 50000;
3638

3739
export const BABEL_PARSER_OPTIONS: babelParser.ParserOptions = {

‎src/FileUtils.ts‎

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1+
import Options from "./Options";
12
import * as Defaults from "./Defaults";
23

34
import {readdirpPromise} from 'readdirp';
45
import * as fs from "node:fs";
56
import nReadlines from "n-readlines";
7+
import * as path from "node:path"
68

79
function countFileLines(filePath: string): number {
810
const broadbandLines = new nReadlines(filePath);
@@ -13,9 +15,29 @@ function countFileLines(filePath: string): number {
1315
return lineNumber
1416
}
1517

16-
function ignoreDirectory(dirName: string): boolean {
18+
function dirIsInIgnorePath(options: Options, fullPath: string, ignorePath: string): boolean {
19+
if (path.isAbsolute(ignorePath)) {
20+
return fullPath.startsWith(ignorePath)
21+
} else {
22+
const absIgnorePath = path.join(options.src, ignorePath)
23+
return fullPath.startsWith(absIgnorePath)
24+
}
25+
}
26+
27+
function fileIsInIgnorePath(options: Options, fullPath: string, ignorePath: string): boolean {
28+
if (path.isAbsolute(ignorePath)) {
29+
return fullPath == ignorePath
30+
} else {
31+
const absIgnorePath = path.join(options.src, ignorePath)
32+
return fullPath == absIgnorePath
33+
}
34+
}
35+
36+
function ignoreDirectory(options: Options, dirName: string, fullPath: string): boolean {
1737
return dirName.startsWith(".") ||
1838
dirName.startsWith("__") ||
39+
options["exclude-file"].some((e: string) => dirIsInIgnorePath(options, fullPath, e)) ||
40+
options["exclude-regex"]?.test(fullPath) ||
1941
Defaults.IGNORE_DIRS.includes(dirName.toLowerCase())
2042
}
2143

@@ -35,19 +57,22 @@ function isTooLarge(fileWithDir: string): boolean {
3557
return false;
3658
}
3759

38-
function ignoreFile(fileName: string, fileWithDir: string, extensions: string[]): boolean {
60+
function ignoreFile(options: Options, fileName: string, fullPath: string, extensions: string[]): boolean {
3961
return !extensions.some((e: string) => fileName.endsWith(e)) ||
4062
fileName.startsWith(".") ||
4163
fileName.startsWith("__") ||
4264
Defaults.IGNORE_FILE_PATTERN.test(fileName) ||
43-
isEmscripten(fileWithDir) ||
44-
isTooLarge(fileWithDir)
65+
options["exclude-file"].some((e: string) => fileIsInIgnorePath(options, fullPath, e)) ||
66+
options["exclude-regex"]?.test(fullPath) ||
67+
isEmscripten(fullPath) ||
68+
isTooLarge(fullPath)
4569
}
4670

47-
export async function filesWithExtensions(dir: string, extensions: string[]): Promise<string[]> {
71+
export async function filesWithExtensions(options: Options, extensions: string[]): Promise<string[]> {
72+
const dir = options.src
4873
const files = await readdirpPromise(dir, {
49-
fileFilter: (f) => !ignoreFile(f.basename, f.fullPath, extensions),
50-
directoryFilter: (d) => !ignoreDirectory(d.basename),
74+
fileFilter: (f) => !ignoreFile(options, f.basename, f.fullPath, extensions),
75+
directoryFilter: (d) => !ignoreDirectory(options, d.basename, d.fullPath),
5176
lstat: true
5277
});
5378
// @ts-ignore

‎src/Options.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,7 @@ export default interface Options {
33
output: string,
44
type?: string,
55
recurse: boolean,
6-
tsTypes: boolean
6+
tsTypes: boolean,
7+
"exclude-file": string[],
8+
"exclude-regex"?: RegExp
79
}

‎src/astgen.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ async function main(argv: string[]) {
1212
.option("src", {
1313
alias: "i",
1414
default: ".",
15+
coerce: (arg: any): string => {
16+
return path.resolve(arg.toString())
17+
},
1518
description: "Source directory",
1619
})
1720
.option("output", {
@@ -35,6 +38,22 @@ async function main(argv: string[]) {
3538
type: "boolean",
3639
description: "Generate type mappings using the Typescript Compiler API",
3740
})
41+
.option("exclude-file", {
42+
default: [],
43+
type: "string",
44+
array: true,
45+
description: "Exclude this file. Can be specified multiple times. Default is empty."
46+
})
47+
.option("exclude-regex", {
48+
coerce: (arg: any): RegExp | undefined => {
49+
try {
50+
return new RegExp(arg.toString(), "i")
51+
} catch (err) {
52+
return undefined;
53+
}
54+
},
55+
description: "Exclude files matching this regex (matches the absolute path)."
56+
})
3857
.version()
3958
.help("h").parseSync();
4059

‎src/index.ts‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ function createTsc(srcFiles: string[]): TscUtils.TscResult | undefined {
5858
*/
5959
async function createJSAst(options: Options) {
6060
try {
61-
const srcFiles: string[] = await FileUtils.filesWithExtensions(options.src, Defaults.JS_EXTENSIONS);
61+
const srcFiles: string[] = await FileUtils.filesWithExtensions(options, Defaults.JS_EXTENSIONS);
6262
let ts: TscUtils.TscResult | undefined;
6363
if (options.tsTypes) {
6464
ts = createTsc(srcFiles);
@@ -95,7 +95,7 @@ async function createJSAst(options: Options) {
9595
* Generate AST for .vue files
9696
*/
9797
async function createVueAst(options: Options) {
98-
const srcFiles: string[] = await FileUtils.filesWithExtensions(options.src, [".vue"]);
98+
const srcFiles: string[] = await FileUtils.filesWithExtensions(options, [".vue"]);
9999
for (const file of srcFiles) {
100100
try {
101101
const ast = toVueAst(file);
@@ -138,20 +138,20 @@ function writeTypesFile(file: string, seenTypes: Map<number, string>, options: O
138138
}
139139

140140
async function createXAst(options: Options) {
141-
const src_dir = options.src;
141+
const srcDir = options.src;
142142
try {
143-
fs.accessSync(src_dir, fs.constants.R_OK);
143+
fs.accessSync(srcDir, fs.constants.R_OK);
144144
} catch (err) {
145-
console.error(src_dir, "is invalid");
145+
console.error(srcDir, "is invalid");
146146
process.exit(1);
147147
}
148148
if (
149-
fs.existsSync(path.join(src_dir, "package.json")) ||
150-
fs.existsSync(path.join(src_dir, "rush.json"))
149+
fs.existsSync(path.join(srcDir, "package.json")) ||
150+
fs.existsSync(path.join(srcDir, "rush.json"))
151151
) {
152152
return await createJSAst(options);
153153
}
154-
console.error(src_dir, "unknown project type");
154+
console.error(srcDir, "unknown project type");
155155
process.exit(1);
156156
}
157157

‎test/astgen.test.ts‎

Lines changed: 109 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ describe('astgen basic functionality', () => {
1414
type: "js",
1515
output: path.join(tmpDir, "ast_out"),
1616
recurse: true,
17-
tsTypes: true
17+
tsTypes: true,
18+
"exclude-file": []
1819
});
1920
const resultAst = fs.readFileSync(path.join(tmpDir, "ast_out", "main.js.json")).toString();
2021
expect(resultAst).toContain("\"fullName\":\"" + testFile.replaceAll("\\", "\\\\") + "\"");
@@ -24,4 +25,111 @@ describe('astgen basic functionality', () => {
2425

2526
fs.rmSync(tmpDir, {recursive: true});
2627
});
28+
29+
it('should exclude files by relative file path correctly', async () => {
30+
const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests"));
31+
const testFile = path.join(tmpDir, "main.js");
32+
fs.writeFileSync(testFile, "console.log(\"Hello, world!\");");
33+
await start({
34+
src: tmpDir,
35+
type: "js",
36+
output: path.join(tmpDir, "ast_out"),
37+
recurse: true,
38+
tsTypes: false,
39+
"exclude-file": ["main.js"]
40+
});
41+
expect(fs.existsSync(path.join(tmpDir, "ast_out", "main.js.json"))).toBeFalsy()
42+
43+
fs.rmSync(tmpDir, {recursive: true});
44+
});
45+
46+
it('should exclude files by absolute file path correctly', async () => {
47+
const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests"));
48+
const testFile = path.join(tmpDir, "main.js");
49+
fs.writeFileSync(testFile, "console.log(\"Hello, world!\");");
50+
await start({
51+
src: tmpDir,
52+
type: "js",
53+
output: path.join(tmpDir, "ast_out"),
54+
recurse: true,
55+
tsTypes: false,
56+
"exclude-file": [testFile]
57+
});
58+
expect(fs.existsSync(path.join(tmpDir, "ast_out", "main.js.json"))).toBeFalsy()
59+
60+
fs.rmSync(tmpDir, {recursive: true});
61+
});
62+
63+
it('should exclude files by relative file path with dir correctly', async () => {
64+
const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests"));
65+
const testFile = path.join(tmpDir, "src", "main.js");
66+
fs.mkdirSync(path.join(tmpDir, "src"))
67+
fs.writeFileSync(testFile, "console.log(\"Hello, world!\");");
68+
await start({
69+
src: tmpDir,
70+
type: "js",
71+
output: path.join(tmpDir, "ast_out"),
72+
recurse: true,
73+
tsTypes: false,
74+
"exclude-file": [path.join("src", "main.js")]
75+
});
76+
expect(fs.existsSync(path.join(tmpDir, "ast_out", "src", "main.js.json"))).toBeFalsy()
77+
78+
fs.rmSync(tmpDir, {recursive: true});
79+
});
80+
81+
it('should exclude files by relative dir path correctly', async () => {
82+
const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests"));
83+
const testFile = path.join(tmpDir, "src", "main.js");
84+
fs.mkdirSync(path.join(tmpDir, "src"))
85+
fs.writeFileSync(testFile, "console.log(\"Hello, world!\");");
86+
await start({
87+
src: tmpDir,
88+
type: "js",
89+
output: path.join(tmpDir, "ast_out"),
90+
recurse: true,
91+
tsTypes: false,
92+
"exclude-file": ["src"]
93+
});
94+
expect(fs.existsSync(path.join(tmpDir, "ast_out", "src", "main.js.json"))).toBeFalsy()
95+
96+
fs.rmSync(tmpDir, {recursive: true});
97+
});
98+
99+
it('should exclude files by absolute dir path correctly', async () => {
100+
const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests"));
101+
const testFile = path.join(tmpDir, "src", "main.js");
102+
fs.mkdirSync(path.join(tmpDir, "src"))
103+
fs.writeFileSync(testFile, "console.log(\"Hello, world!\");");
104+
await start({
105+
src: tmpDir,
106+
type: "js",
107+
output: path.join(tmpDir, "ast_out"),
108+
recurse: true,
109+
tsTypes: false,
110+
"exclude-file": [path.join(tmpDir, "src")]
111+
});
112+
expect(fs.existsSync(path.join(tmpDir, "ast_out", "main.js.json"))).toBeFalsy()
113+
114+
fs.rmSync(tmpDir, {recursive: true});
115+
});
116+
117+
it('should exclude files by regex correctly', async () => {
118+
const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests"));
119+
const testFile = path.join(tmpDir, "main.js");
120+
fs.writeFileSync(testFile, "console.log(\"Hello, world!\");");
121+
await start({
122+
src: tmpDir,
123+
type: "js",
124+
output: path.join(tmpDir, "ast_out"),
125+
recurse: true,
126+
tsTypes: false,
127+
"exclude-file": [],
128+
"exclude-regex": new RegExp(".*main.*", "i")
129+
});
130+
expect(fs.existsSync(path.join(tmpDir, "ast_out", "main.js.json"))).toBeFalsy()
131+
132+
fs.rmSync(tmpDir, {recursive: true});
133+
});
134+
27135
});

0 commit comments

Comments
 (0)