[x2cpg] Use SourceFiles.filterFile logic to also filter config files (#5641)

johannescoetzee · web-flow · commit 049c2ffd67d8 · 2025-09-05T15:39:14.000+02:00
* Use SourceFiles.filterFile logic to also filter config files
diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/JavaSrc2Cpg.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/JavaSrc2Cpg.scala
@@ -27,7 +27,7 @@ class JavaSrc2Cpg extends X2CpgFrontend {
       astCreationPass.sourceParser.cleanupDelombokOutput()
       astCreationPass.clearJavaParserCaches()
       new OuterClassRefPass(cpg).createAndApply()
-      JavaConfigFileCreationPass(cpg).createAndApply()
+      JavaConfigFileCreationPass(cpg, config = config).createAndApply()
       if (!config.skipTypeInfPass) {
         TypeNodePass.withRegisteredTypes(astCreationPass.global.usedTypes.keys().asScala.toList, cpg).createAndApply()
         new TypeInferencePass(cpg).createAndApply()
diff --git a/joern-cli/frontends/javasrc2cpg/src/test/resources/config_tests/excluded/excluded.properties b/joern-cli/frontends/javasrc2cpg/src/test/resources/config_tests/excluded/excluded.properties
@@ -0,0 +1 @@
+basic excluded properties
diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/passes/ConfigFileCreationPassTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/passes/ConfigFileCreationPassTests.scala
@@ -1,6 +1,7 @@
 package io.joern.javasrc2cpg.passes
 
 import flatgraph.misc.TestUtils.*
+import io.joern.javasrc2cpg.Config
 import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture
 import io.joern.x2cpg.passes.frontend.JavaConfigFileCreationPass
 import io.shiftleft.semanticcpg.utils.FileUtil.*
@@ -17,9 +18,10 @@ class ConfigFileCreationPassTests extends JavaSrcCode2CpgFixture {
     ProjectRoot.relativise("joern-cli/frontends/javasrc2cpg/src/test/resources/config_tests")
 
   "it should find the correct config files" in {
-    val cpg = Cpg.from(_.addNode(NewMetaData().root(testConfigDir)))
+    val cpg    = Cpg.from(_.addNode(NewMetaData().root(testConfigDir)))
+    val config = Config().withIgnoredFilesRegex(".*excluded.*").withDefaultIgnoredFilesRegex(Nil)
 
-    val foundFiles        = new JavaConfigFileCreationPass(cpg).generateParts().map(_.absolutePathAsString)
+    val foundFiles = new JavaConfigFileCreationPass(cpg, config = config).generateParts().map(_.absolutePathAsString)
     val absoluteConfigDir = Paths.get(testConfigDir).absolutePathAsString
     foundFiles should contain theSameElementsAs Array(
       Paths.get(absoluteConfigDir, "application.conf").toString,
diff --git a/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/Jimple2Cpg.scala b/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/Jimple2Cpg.scala
@@ -129,7 +129,7 @@ class Jimple2Cpg extends X2CpgFrontend {
       .withRegisteredTypes(global.usedTypes.keys().asScala.toList, cpg)
       .createAndApply()
     DeclarationRefPass(cpg).createAndApply()
-    JavaConfigFileCreationPass(cpg, Option(tmpDir.toString)).createAndApply()
+    JavaConfigFileCreationPass(cpg, Option(tmpDir.toString), config).createAndApply()
   }
 
   private def decompileClassFiles(classFiles: List[ClassFile], decompileJava: Boolean): Unit = {
diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/ConfigFileCreationPass.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/ConfigFileCreationPass.scala
@@ -5,8 +5,8 @@ import io.shiftleft.codepropertygraph.generated.Cpg
 
 import java.nio.file.Path
 
-class ConfigFileCreationPass(cpg: Cpg, requirementsTxt: String = "requirement.txt")
-    extends XConfigFileCreationPass(cpg) {
+class ConfigFileCreationPass(cpg: Cpg, requirementsTxt: String = "requirement.txt", config: Py2CpgOnFileSystemConfig)
+    extends XConfigFileCreationPass(cpg, config = config) {
 
   override val configFileFilters: List[Path => Boolean] = List(
     // TOML files
diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2Cpg.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2Cpg.scala
@@ -25,29 +25,22 @@ object Py2Cpg {
   * @param schemaValidationMode
   *   The boolean switch for enabling or disabling early schema checking during AST creation.
   */
-class Py2Cpg(
-  inputProviders: Iterable[Py2Cpg.InputProvider],
-  outputCpg: Cpg,
-  inputPath: String,
-  requirementsTxt: String = "requirements.txt",
-  schemaValidationMode: ValidationMode,
-  enableFileContent: Boolean
-) {
+class Py2Cpg(inputProviders: Iterable[Py2Cpg.InputProvider], outputCpg: Cpg, config: Py2CpgOnFileSystemConfig) {
   private val diffGraph   = Cpg.newDiffGraphBuilder
   private val nodeBuilder = new NodeBuilder(diffGraph)
   private val edgeBuilder = new EdgeBuilder(diffGraph)
 
   def buildCpg(): Unit = {
-    nodeBuilder.metaNode(Languages.PYTHONSRC, version = "").root(inputPath + java.io.File.separator)
+    nodeBuilder.metaNode(Languages.PYTHONSRC, version = "").root(config.inputPath + java.io.File.separator)
     val globalNamespaceBlock =
       nodeBuilder.namespaceBlockNode(Constants.GLOBAL_NAMESPACE, Constants.GLOBAL_NAMESPACE, "N/A")
     nodeBuilder.typeNode(Constants.ANY, Constants.ANY)
     val anyTypeDecl =
       nodeBuilder.typeDeclNode(Constants.ANY, Constants.ANY, "N/A", Nil, LineAndColumn(1, 1, 1, 1, 1, 1))
     edgeBuilder.astEdge(anyTypeDecl, globalNamespaceBlock, 0)
     DiffGraphApplier.applyDiff(outputCpg.graph, diffGraph)
-    new CodeToCpg(outputCpg, inputProviders, schemaValidationMode, enableFileContent).createAndApply()
-    new ConfigFileCreationPass(outputCpg, requirementsTxt).createAndApply()
+    new CodeToCpg(outputCpg, inputProviders, config.schemaValidation, !config.disableFileContent).createAndApply()
+    new ConfigFileCreationPass(outputCpg, config.requirementsTxt, config).createAndApply()
     new DependenciesFromRequirementsTxtPass(outputCpg).createAndApply()
   }
 }
diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2CpgOnFileSystem.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2CpgOnFileSystem.scala
@@ -97,14 +97,7 @@ class Py2CpgOnFileSystem extends X2CpgFrontend {
           Py2Cpg.InputPair(content, inputPath.relativize(inputFile).toString)
         }
       }
-      val py2Cpg = new Py2Cpg(
-        inputProviders,
-        cpg,
-        config.inputPath,
-        config.requirementsTxt,
-        config.schemaValidation,
-        !config.disableFileContent
-      )
+      val py2Cpg = new Py2Cpg(inputProviders, cpg, config)
       py2Cpg.buildCpg()
     }
   }
diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/testfixtures/Py2CpgTestContext.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/testfixtures/Py2CpgTestContext.scala
@@ -1,10 +1,11 @@
 package io.joern.pysrc2cpg.testfixtures
 
-import io.joern.pysrc2cpg.Py2Cpg
+import io.joern.pysrc2cpg.{Py2Cpg, Py2CpgOnFileSystemConfig}
 import io.joern.x2cpg.ValidationMode
 import io.joern.x2cpg.X2Cpg.defaultOverlayCreators
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.semanticcpg.layers.LayerCreatorContext
+import io.shiftleft.semanticcpg.utils.FileUtil
 
 import scala.collection.mutable
 
@@ -25,7 +26,7 @@ class Py2CpgTestContext {
 
   private val codeAndFile       = mutable.ArrayBuffer.empty[Py2Cpg.InputPair]
   private var buildResult       = Option.empty[Cpg]
-  private val absTestFilePath   = "<absoluteTestPath>/"
+  private val absTestFilePath   = "absoluteTestPath/"
   private var enableFileContent = true
 
   def withEnabledFileContent(value: Boolean): Py2CpgTestContext = {
@@ -45,16 +46,17 @@ class Py2CpgTestContext {
   }
 
   def buildCpg: Cpg = {
+    val cpgOutFile = FileUtil.newTemporaryFile(suffix = "cpg.bin")
+    FileUtil.deleteOnExit(cpgOutFile)
+    val config = Py2CpgOnFileSystemConfig()
+      .withInputPath(absTestFilePath)
+      .withOutputPath(cpgOutFile.toString)
+      .withSchemaValidation(ValidationMode.Enabled)
+      .withDisableFileContent(!enableFileContent)
     if (buildResult.isEmpty) {
       val cpg = new Cpg()
       val py2Cpg =
-        new Py2Cpg(
-          codeAndFile.map(inputPair => () => inputPair),
-          cpg,
-          absTestFilePath,
-          schemaValidationMode = ValidationMode.Enabled,
-          enableFileContent = enableFileContent
-        )
+        new Py2Cpg(codeAndFile.map(inputPair => () => inputPair), cpg, config)
       py2Cpg.buildCpg()
 
       val context = new LayerCreatorContext(cpg)
diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala
@@ -40,7 +40,7 @@ class RubySrc2Cpg extends X2CpgFrontend {
   override def createCpg(config: Config): Try[Cpg] = {
     withNewEmptyCpg(config.outputPath, config: Config) { (cpg, config) =>
       new MetaDataPass(cpg, Languages.RUBYSRC, config.inputPath).createAndApply()
-      new ConfigFileCreationPass(cpg).createAndApply()
+      new ConfigFileCreationPass(cpg, config).createAndApply()
       new DependencyPass(cpg).createAndApply()
       createCpgAction(cpg, config)
     }
diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/ConfigFileCreationPass.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/ConfigFileCreationPass.scala
@@ -1,18 +1,18 @@
 package io.joern.rubysrc2cpg.passes
 
+import io.joern.rubysrc2cpg.Config
 import io.joern.x2cpg.passes.frontend.XConfigFileCreationPass
 import io.shiftleft.semanticcpg.utils.FileUtil.*
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.utils.FileUtil
 
 import java.nio.file.{Path, Paths}
-
 import scala.util.Try
 
 /** Creates the CONFIGURATION layer from any existing `Gemfile` or `Gemfile.lock` files found at root level.
   */
-class ConfigFileCreationPass(cpg: Cpg) extends XConfigFileCreationPass(cpg) {
+class ConfigFileCreationPass(cpg: Cpg, config: Config) extends XConfigFileCreationPass(cpg, config = config) {
 
   private val validGemfilePaths = Try(Paths.get(cpg.metaData.root.headOption.getOrElse(""))).toOption match {
     case Some(rootPath) => Seq("Gemfile", "Gemfile.lock").map(rootPath / _)
diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/SwiftSrc2Cpg.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/SwiftSrc2Cpg.scala
@@ -35,7 +35,7 @@ class SwiftSrc2Cpg extends X2CpgFrontend {
 
         new ExtensionInheritancePass(cpg).createAndApply()
         new MetaDataPass(cpg, hash, config.inputPath).createAndApply()
-        new ConfigFileCreationPass(cpg).createAndApply()
+        new ConfigFileCreationPass(cpg, config).createAndApply()
         new DependenciesPass(cpg).createAndApply()
         new FullNameUniquenessPass(cpg).createAndApply()
 
diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/ConfigFileCreationPass.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/ConfigFileCreationPass.scala
@@ -1,11 +1,12 @@
 package io.joern.swiftsrc2cpg.passes
 
+import io.joern.swiftsrc2cpg.Config
 import io.joern.x2cpg.passes.frontend.XConfigFileCreationPass
 import io.shiftleft.codepropertygraph.generated.Cpg
 
 import java.nio.file.Path
 
-class ConfigFileCreationPass(cpg: Cpg) extends XConfigFileCreationPass(cpg) {
+class ConfigFileCreationPass(cpg: Cpg, config: Config) extends XConfigFileCreationPass(cpg, config = config) {
 
   override val configFileFilters: List[Path => Boolean] = List(extensionFilter(".plist"), extensionFilter(".xib"))
 
diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XConfigFileCreationPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XConfigFileCreationPass.scala
@@ -1,5 +1,6 @@
 package io.joern.x2cpg.passes.frontend
 
+import io.joern.x2cpg.{SourceFiles, X2CpgConfig}
 import io.shiftleft.semanticcpg.utils.FileUtil.*
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.NewConfigFile
@@ -15,7 +16,7 @@ import scala.util.{Failure, Success, Try}
 /** Scans for and inserts configuration files into the CPG. Relies on the MetaData's `ROOT` property to provide the path
   * to scan, but alternatively one can specify a directory on the `rootDir` parameter.
   */
-abstract class XConfigFileCreationPass(cpg: Cpg, private val rootDir: Option[String] = None)
+abstract class XConfigFileCreationPass(cpg: Cpg, private val rootDir: Option[String] = None, config: X2CpgConfig[_])
     extends ForkJoinParallelCpgPass[Path](cpg) {
 
   private val logger = LoggerFactory.getLogger(this.getClass)
@@ -32,6 +33,7 @@ abstract class XConfigFileCreationPass(cpg: Cpg, private val rootDir: Option[Str
               .walk()
               .filterNot(_ == file)
               .filter(isConfigFile)
+              .filter(isAccepted(root, _))
               .toArray
 
           case Success(file) if isConfigFile(file) => Array(file)
@@ -71,6 +73,16 @@ abstract class XConfigFileCreationPass(cpg: Cpg, private val rootDir: Option[Str
     file.absolutePathAsString.endsWith(pathEnd)
   }
 
+  protected def isAccepted(rootPath: String, filePath: Path): Boolean = {
+    SourceFiles.filterFile(
+      filePath.toAbsolutePath.toString,
+      rootPath,
+      Option(config.defaultIgnoredFilesRegex),
+      Option(config.ignoredFilesRegex),
+      Option(config.ignoredFiles)
+    )
+  }
+
   private def isConfigFile(file: Path): Boolean = {
     configFileFilters.exists(predicate => predicate(file))
   }
@@ -79,8 +91,8 @@ abstract class XConfigFileCreationPass(cpg: Cpg, private val rootDir: Option[Str
 /** Parses common Java related configuration files. Multiple frontends cover JVM languages so this is in a single shared
   * spot.
   */
-class JavaConfigFileCreationPass(cpg: Cpg, rootDir: Option[String] = None)
-    extends XConfigFileCreationPass(cpg, rootDir) {
+class JavaConfigFileCreationPass(cpg: Cpg, rootDir: Option[String] = None, config: X2CpgConfig[_])
+    extends XConfigFileCreationPass(cpg, rootDir, config) {
 
   override val configFileFilters: List[Path => Boolean] = List(
     // JAVA_INTERNAL

Original file line number	Diff line number	Diff line change
`@@ -129,7 +129,7 @@ class Jimple2Cpg extends X2CpgFrontend {`
`129`	`129`	`.withRegisteredTypes(global.usedTypes.keys().asScala.toList, cpg)`
`130`	`130`	`.createAndApply()`
`131`	`131`	`DeclarationRefPass(cpg).createAndApply()`
`132`		`- JavaConfigFileCreationPass(cpg, Option(tmpDir.toString)).createAndApply()`
	`132`	`+ JavaConfigFileCreationPass(cpg, Option(tmpDir.toString), config).createAndApply()`
`133`	`133`	`}`
`134`	`134`
`135`	`135`	`private def decompileClassFiles(classFiles: List[ClassFile], decompileJava: Boolean): Unit = {`
Original file line number	Diff line number	Diff line change
`@@ -97,14 +97,7 @@ class Py2CpgOnFileSystem extends X2CpgFrontend {`
`97`	`97`	`Py2Cpg.InputPair(content, inputPath.relativize(inputFile).toString)`
`98`	`98`	`}`
`99`	`99`	`}`
`100`		`- val py2Cpg = new Py2Cpg(`
`101`		`- inputProviders,`
`102`		`- cpg,`
`103`		`- config.inputPath,`
`104`		`- config.requirementsTxt,`
`105`		`- config.schemaValidation,`
`106`		`- !config.disableFileContent`
`107`		`- )`
	`100`	`+ val py2Cpg = new Py2Cpg(inputProviders, cpg, config)`
`108`	`101`	`py2Cpg.buildCpg()`
`109`	`102`	`}`
`110`	`103`	`}`
Original file line number	Diff line number	Diff line change
`@@ -40,7 +40,7 @@ class RubySrc2Cpg extends X2CpgFrontend {`
`40`	`40`	`override def createCpg(config: Config): Try[Cpg] = {`
`41`	`41`	`withNewEmptyCpg(config.outputPath, config: Config) { (cpg, config) =>`
`42`	`42`	`new MetaDataPass(cpg, Languages.RUBYSRC, config.inputPath).createAndApply()`
`43`		`- new ConfigFileCreationPass(cpg).createAndApply()`
	`43`	`+ new ConfigFileCreationPass(cpg, config).createAndApply()`
`44`	`44`	`new DependencyPass(cpg).createAndApply()`
`45`	`45`	`createCpgAction(cpg, config)`
`46`	`46`	`}`