[ruby] Add Support for ERB files (#5447)

* [ruby] Add support for basic ERB files (#5404)

* Added tests for ERB processing

* update test expectation

* Add isLineFeed func

* removed erb config file test

* Fixed new line issues with ERB files, updated test expectations

* [ruby] rework erb support. Added templateOutRaw and templateOutEscape operators. Included explicit return for ERB file processing

* [ruby] Add .html.erb files as config files for ruby

* review comments

* Fixed ERB lowering implementation. Updated test expectations

* Updated comments

* remove test

* upgrade ruby-ast-gen version

* upgrade ruby-ast-gen version, remove unnecessary whitespace

* [ruby] update code string and tests for joern__buffer changes to self.joern__buffer

* [ruby] update ERB tests to reflect no line numbers being set from ruby_ast_gen

* [ruby] remove self parameter from lambda, capture from outside instead

* [ruby] Add empty expression for if statement with no expressions in body specifically in ERB files

* [ruby] Only look for the closest self in the outerscope in closure body

* [ruby] capture self variable

* fix: resolve multiple self identifier refOuts bug

* chore: add return type

* refactor: scalafmt run

* fix: resolve incorrect closure binding refOut bug

This also fixes a bug where a lambda param IDENTIFIER was not referencing the param

* feat: use joern__buffer_append

* chore: scalafmt

* fix: recognise joern_inner_buffer appends as erb calls

* fix: retain joern__buffer_append args' receiver ast

* improvement: turn lookupSelfInOuterScope into wrapper

* improvement: override span instead of using 2 parameter lists

* refactor: remove debug lines in test

* improvement: move typeFullName to outer scope

* improvement: use buffer_append constant

* improvement: use Option.when

* improvement: remove case for self

* improvement: remove ERB call special case

* improvement: use camelCase for ERB buffer symbols and methods

* refactor: scalafmt

* cleanup: remove unused variable

* improvement: rename isErbCall

* improvement: avoid unnecessary call

* refactor: refactor block

* fix: don't generate receiver AST if isStatic

* fix: add tmp assignments to tests

* Revert "fix: add tmp assignments to tests"

This reverts commit f0fdf28.

* Revert "fix: don't generate receiver AST if isStatic"

This reverts commit 39edae8.

* chore: use Option.unless

---------

Co-authored-by: Tebogo Selahle <[email protected]>

Author: AndreiDreyer

joern-cli/frontends/rubysrc2cpg/src/main/resources/application.conf

@@ -1,4 +1,4 @@
 rubysrc2cpg {
-    ruby_ast_gen_version: "0.33.0"
+    ruby_ast_gen_version: "0.58.0"
     joern_type_stubs_version: "0.6.0"
-}
+}

joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala

@@ -67,7 +67,7 @@ class RubySrc2Cpg extends X2CpgFrontend {
         }
         .filter { x =>
           if x.fileContent.isBlank then logger.info(s"File content empty, skipping - ${x.fileName}")
-          !x.fileContent.isBlank
+          !x.fileContent.isBlank || x.fileName.endsWith(".html.erb")
         }
 
       val internalProgramSummary = ConcurrentTaskUtil
@@ -122,7 +122,7 @@ object RubySrc2Cpg {
   ): Iterator[() => AstCreator] = {
     astFiles.map { fileName => () =>
       val parserResult   = RubyJsonParser.readFile(Paths.get(fileName))
-      val rubyProgram    = new RubyJsonToNodeCreator().visitProgram(parserResult.json)
+      val rubyProgram    = new RubyJsonToNodeCreator(fileName = fileName).visitProgram(parserResult.json)
       val sourceFileName = parserResult.fullPath
       val fileContent    = new String(Files.readAllBytes(Paths.get(sourceFileName)), Charset.defaultCharset())
       new AstCreator(

joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala

@@ -21,8 +21,10 @@ import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.{
 }
 import io.joern.rubysrc2cpg.datastructures.{BlockScope, FieldDecl}
 import io.joern.rubysrc2cpg.passes.Defines
+import io.joern.rubysrc2cpg.passes.Defines.RubyOperators
 import io.joern.rubysrc2cpg.passes.GlobalTypes
 import io.joern.rubysrc2cpg.passes.GlobalTypes.{kernelFunctions, kernelPrefix}
+import io.joern.x2cpg.frontendspecific.rubysrc2cpg.Constants
 import io.joern.x2cpg.{Ast, ValidationMode}
 import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, Operators}
@@ -252,6 +254,14 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
     StatementList(tmpAssignment :: ifStmt :: Nil)(originSpan)
   }
 
+  protected def isErbCall(callName: String): Boolean = ErbTemplateCallNames.contains(callName)
+
+  protected val ErbTemplateCallNames: Map[String, String] = Map(
+    Constants.joernErbTemplateOutRawName    -> RubyOperators.templateOutRaw,
+    Constants.joernErbTemplateOutEscapeName -> RubyOperators.templateOutEscape,
+    Constants.joernErbBufferAppend          -> RubyOperators.bufferAppend
+  )
+
   protected val UnaryOperatorNames: Map[String, String] = Map(
     "!"   -> Operators.logicalNot,
     "not" -> Operators.logicalNot,

joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala

@@ -2,20 +2,12 @@ package io.joern.rubysrc2cpg.astcreation
 
 import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.{Unknown, Block as RubyBlock, *}
 import io.joern.rubysrc2cpg.datastructures.BlockScope
-import io.joern.rubysrc2cpg.parser.RubyJsonHelpers
-import io.joern.rubysrc2cpg.passes.Defines
-import io.joern.rubysrc2cpg.passes.GlobalTypes
-import io.joern.rubysrc2cpg.passes.Defines.{RubyOperators, prefixAsKernelDefined}
+import io.joern.rubysrc2cpg.passes.Defines.RubyOperators
+import io.joern.rubysrc2cpg.passes.{Defines, GlobalTypes}
+import io.joern.x2cpg.frontendspecific.rubysrc2cpg.Constants
 import io.joern.x2cpg.{Ast, ValidationMode, Defines as XDefines}
 import io.shiftleft.codepropertygraph.generated.nodes.*
-import io.shiftleft.codepropertygraph.generated.{
-  ControlStructureTypes,
-  DispatchTypes,
-  EdgeTypes,
-  NodeTypes,
-  Operators,
-  PropertyNames
-}
+import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, EdgeTypes, NodeTypes, Operators}
 
 import scala.collection.mutable
 
@@ -42,6 +34,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
     case node: TypeIdentifier                   => astForTypeIdentifier(node)
     case node: RubyIdentifier                   => astForSimpleIdentifier(node)
     case node: SimpleCall                       => astForSimpleCall(node)
+    case node: ErbTemplateCall                  => astForErbTemplateCall(node)
     case node: RequireCall                      => astForRequireCall(node)
     case node: IncludeCall                      => astForIncludeCall(node)
     case node: RaiseCall                        => astForRaiseCall(node)
@@ -64,6 +57,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
     case node: ArrayPattern                     => astForArrayPattern(node)
     case node: DummyNode                        => Ast(node.node)
     case node: DummyAst                         => node.ast
+    case node: EmptyExpression                  => Ast()
     case node: Unknown                          => astForUnknown(node)
     case x =>
       logger.warn(s"Unhandled expression of type ${x.getClass.getSimpleName}")
@@ -162,7 +156,12 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
           case None => x
         }
         astForFieldAccess(node.copy(target = newTarget)(node.span))
-      case _ => astForFieldAccess(node)
+      case _ =>
+        if (Constants.joernErbBuffers.contains(node.memberName)) {
+          astForFieldAccess(node, typeFullName = Constants.stringPrefix)
+        } else {
+          astForFieldAccess(node)
+        }
     }
   }
 
@@ -191,26 +190,35 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
   protected def astForMemberCall(node: RubyCallWithBase, isStatic: Boolean = false): Ast = {
 
     def createMemberCall(n: RubyCallWithBase): Ast = {
-      val receiverAst = astForFieldAccess(MemberAccess(n.target, ".", n.methodName)(n.span), stripLeadingAt = true)
+      val isErbBufferAppendCall = n.methodName == Constants.joernErbBufferAppend
+
+      val receiverAstOpt = Option.unless(isErbBufferAppendCall)(
+        astForFieldAccess(MemberAccess(n.target, ".", n.methodName)(n.span), stripLeadingAt = true)
+      )
+
       val (baseAst, baseCode) = astForMemberAccessTarget(n.target)
       val builtinType = n.target match {
         case MemberAccess(_: SelfIdentifier, _, memberName) if isBundledClass(memberName) =>
           Option(prefixAsCoreType(memberName))
         case x: TypeIdentifier if x.isBuiltin => Option(x.typeFullName)
         case _                                => None
       }
-      val methodFullName = receiverAst.nodes
-        .collectFirst {
-          case _ if builtinType.isDefined => s"${builtinType.get}.${n.methodName}"
-          case x: NewMethodRef            => x.methodFullName
-          case _ =>
-            (n.target match {
-              case ma: MemberAccess => scope.tryResolveTypeReference(ma.memberName).map(_.name)
-              case _                => typeFromCallTarget(n.target)
-            }).map(x => s"$x.${n.methodName}")
-              .getOrElse(XDefines.DynamicCallUnknownFullName)
+      val methodFullName = if (isErbBufferAppendCall) {
+        RubyOperators.bufferAppend
+      } else {
+        val fullNameOpt = receiverAstOpt.flatMap { ast =>
+          ast.nodes.headOption.flatMap {
+            case _ if builtinType.isDefined => builtinType.map(t => s"$t.${n.methodName}")
+            case x: NewMethodRef            => Some(x.methodFullName)
+            case _ =>
+              (n.target match {
+                case ma: MemberAccess => scope.tryResolveTypeReference(ma.memberName).map(_.name)
+                case _                => typeFromCallTarget(n.target)
+              }).map(x => s"$x.${n.methodName}")
+          }
         }
-        .getOrElse(XDefines.DynamicCallUnknownFullName)
+        fullNameOpt.getOrElse(XDefines.DynamicCallUnknownFullName)
+      }
       val argumentAsts = n.arguments.map(astForMethodCallArgument)
       val dispatchType = if (isStatic) DispatchTypes.STATIC_DISPATCH else DispatchTypes.DYNAMIC_DISPATCH
 
@@ -223,12 +231,21 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
         code(n)
       }
 
-      val call = callNode(n, callCode, n.methodName, XDefines.DynamicCallUnknownFullName, dispatchType)
-      if methodFullName != XDefines.DynamicCallUnknownFullName then call.possibleTypes(Seq(methodFullName))
-      if (isStatic) {
-        callAst(call, argumentAsts, base = Option(baseAst)).copy(receiverEdges = Nil)
+      val call = if (isErbBufferAppendCall) {
+        operatorCallNode(n, callCode, methodFullName, Some(Constants.stringPrefix))
+      } else {
+        val call = callNode(n, callCode, n.methodName, XDefines.DynamicCallUnknownFullName, dispatchType)
+        if methodFullName != XDefines.DynamicCallUnknownFullName then call.possibleTypes(Seq(methodFullName))
+        call
+      }
+      if (isErbBufferAppendCall) {
+        callAst(call, argumentAsts)
       } else {
-        callAst(call, argumentAsts, base = Option(baseAst), receiver = Option(receiverAst))
+        if (isStatic) {
+          callAst(call, argumentAsts, base = Option(baseAst)).copy(receiverEdges = Nil)
+        } else {
+          callAst(call, argumentAsts, base = Option(baseAst), receiver = receiverAstOpt)
+        }
       }
     }
 
@@ -278,7 +295,11 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
     }
   }
 
-  protected def astForFieldAccess(node: MemberAccess, stripLeadingAt: Boolean = false): Ast = {
+  protected def astForFieldAccess(
+    node: MemberAccess,
+    stripLeadingAt: Boolean = false,
+    typeFullName: String = Defines.Any
+  ): Ast = {
     val (memberName, memberCode) = node.target match {
       case _ if node.memberName == Defines.Initialize => Defines.Initialize -> Defines.Initialize
       case _ if stripLeadingAt                        => node.memberName    -> node.memberName.stripPrefix("@")
@@ -307,7 +328,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
       Operators.fieldAccess,
       DispatchTypes.STATIC_DISPATCH,
       signature = None,
-      typeFullName = Option(Defines.Any)
+      typeFullName = Option(typeFullName)
     ).possibleTypes(IndexedSeq(memberType.get))
     callAst(fieldAccess, Seq(targetAst, fieldIdentifierAst))
   }
@@ -662,6 +683,13 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
         astForUnknown(targetNode)
   }
 
+  protected def astForErbTemplateCall(node: ErbTemplateCall): Ast = {
+    val argAsts = node.arguments.map(astForExpression)
+    val opName  = ErbTemplateCallNames(node.target.text)
+    val opNode  = callNode(node, node.span.text, opName, opName, DispatchTypes.STATIC_DISPATCH)
+    callAst(opNode, argAsts)
+  }
+
   protected def astForRequireCall(node: RequireCall): Ast = {
     val pathOpt = node.argument match {
       case arg: StaticLiteral if arg.isString => Option(arg.innerText)
@@ -970,7 +998,8 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
   }
 
   private def astForMethodCallWithoutBlock(node: SimpleCall, methodIdentifier: SimpleIdentifier): Ast = {
-    val methodName         = methodIdentifier.text
+    val methodName =
+      if (isErbCall(methodIdentifier.text)) ErbTemplateCallNames(methodIdentifier.text) else methodIdentifier.text
     lazy val defaultResult = Defines.Any -> XDefines.DynamicCallUnknownFullName
 
     val (receiverType, methodFullNameHint) =
@@ -1087,4 +1116,6 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
   private def getUnaryOperatorName(op: String): Option[String] = UnaryOperatorNames.get(op)
 
   private def getAssignmentOperatorName(op: String): Option[String] = AssignmentOperatorNames.get(op)
+
+  private def isLineFeed(text: String): Boolean = text == "\n" || text == "\r" || text == "\r\n"
 }

joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala

@@ -68,22 +68,26 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th
     if (isConstructor) scope.pushNewScope(ConstructorScope(fullName, scope.getNewProcParam))
     else scope.pushNewScope(MethodScope(fullName, scope.getNewProcParam))
 
-    val thisParameterNode = parameterInNode(
-      node,
-      name = Defines.Self,
-      code = Defines.Self,
-      index = 0,
-      isVariadic = false,
-      typeFullName = Option(scope.surroundingTypeFullName.getOrElse(Defines.Any)),
-      evaluationStrategy = EvaluationStrategies.BY_SHARING
-    )
-    val thisParameterAst = Ast(thisParameterNode)
-    scope.addToScope(Defines.Self, thisParameterNode)
-    val parameterAsts = thisParameterAst :: astForParameters(node.parameters)
+    val parameterAsts =
+      if (isClosure) {
+        astForParameters(node.parameters)
+      } else {
+        val thisParameterNode = parameterInNode(
+          node,
+          name = Defines.Self,
+          code = Defines.Self,
+          index = 0,
+          isVariadic = false,
+          typeFullName = Option(scope.surroundingTypeFullName.getOrElse(Defines.Any)),
+          evaluationStrategy = EvaluationStrategies.BY_SHARING
+        )
 
-    val optionalStatementList = statementListForOptionalParams(node.parameters)
+        val thisParameterAst = Ast(thisParameterNode)
+        scope.addToScope(Defines.Self, thisParameterNode)
+        thisParameterAst :: astForParameters(node.parameters)
+      }
 
-    val methodReturn = methodReturnNode(node, Defines.Any)
+    val optionalStatementList = statementListForOptionalParams(node.parameters)
 
     val refs = {
       val typeRef =
@@ -92,8 +96,21 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th
       List(typeRef, methodRefNode(node, methodName, fullName, fullName)).map(Ast.apply)
     }
 
+    val methodReturn = methodReturnNode(node, Defines.Any)
+
     // Consider which variables are captured from the outer scope
     val stmtBlockAst = if (isClosure || isSingletonObjectMethod) {
+      // create closure `self` local used for capturing
+      scope.lookupSelfInOuterScope
+        .collect {
+          case local: NewLocal             => local.name
+          case param: NewMethodParameterIn => param.name
+        }
+        .foreach { name =>
+          val capturingLocal =
+            localNode(node.body, name, name, Defines.Any, closureBindingId = Option(s"$fullName.$name"))
+          scope.addToScope(capturingLocal.name, capturingLocal)
+        }
       val baseStmtBlockAst = astForMethodBody(node.body, optionalStatementList)
       transformAsClosureBody(node.body, refs, baseStmtBlockAst)
     } else {
@@ -205,22 +222,27 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th
     methodAst
   }
 
-  private def transformAsClosureBody(originNode: RubyExpression, refs: List[Ast], baseStmtBlockAst: Ast) = {
+  private def transformAsClosureBody(originNode: RubyExpression, refs: List[Ast], baseStmtBlockAst: Ast): Ast = {
     // Determine which locals are captured
     val capturedLocalNodes = baseStmtBlockAst.nodes
-      .collect { case x: NewIdentifier if x.name != Defines.Self => x } // Self identifiers are handled separately
+      .collect { case x: NewIdentifier if x.name != Defines.Self => x }
       .distinctBy(_.name)
       .map(i => scope.lookupVariableInOuterScope(i.name))
-      .filter(_.nonEmpty)
+      .filter(_.iterator.nonEmpty)
       .flatten
       .toSet
 
+    val selfLocal     = scope.lookupSelfInOuterScope.toSet
+    val capturedNodes = capturedLocalNodes ++ selfLocal
+
     val capturedIdentifiers = baseStmtBlockAst.nodes.collect {
-      case i: NewIdentifier if capturedLocalNodes.map(_.name).contains(i.name) => i
+      case i: NewIdentifier if capturedNodes.map(_.name).contains(i.name) => i
     }
     // Copy AST block detaching the REF nodes between parent locals/params and identifiers, with the closures' one
     val capturedBlockAst = baseStmtBlockAst.copy(refEdges = baseStmtBlockAst.refEdges.filterNot {
-      case AstEdge(_: NewIdentifier, dst: DeclarationNew) => capturedLocalNodes.contains(dst)
+      case AstEdge(_: NewIdentifier, dst: NewLocal) if dst.name == Defines.Self =>
+        capturedNodes.map(_.name).contains(dst.name)
+      case AstEdge(_: NewIdentifier, dst: DeclarationNew) => capturedNodes.contains(dst)
       case _                                              => false
     })
 
@@ -229,18 +251,13 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th
     val astChildren  = mutable.Buffer.empty[NewNode]
     val refEdges     = mutable.Buffer.empty[(NewNode, NewNode)]
     val captureEdges = mutable.Buffer.empty[(NewNode, NewNode)]
-    capturedLocalNodes
-      .collect {
-        case local: NewLocal =>
-          val closureBindingId = scope.variableScopeFullName(local.name).map(x => s"$x.${local.name}")
-          (local, local.name, local.code, closureBindingId)
-        case param: NewMethodParameterIn =>
-          val closureBindingId = scope.variableScopeFullName(param.name).map(x => s"$x.${param.name}")
-          (param, param.name, param.code, closureBindingId)
-      }
+
+    createClosureBindingInformation(capturedNodes)
       .collect { case (capturedLocal, name, code, Some(closureBindingId)) =>
-        val capturingLocal =
+        val selfCapturingLocal = Option.when(name == Defines.Self)(scope.lookupSelfInCurrentScope).flatten
+        val capturingLocal = selfCapturingLocal.getOrElse(
           localNode(originNode, name, name, Defines.Any, closureBindingId = Option(closureBindingId))
+        )
 
         val closureBinding = closureBindingNode(
           closureBindingId = closureBindingId,
@@ -614,4 +631,18 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th
       case _                      => false
     }
   }
+
+  private def createClosureBindingInformation(
+    capturedNodes: Set[DeclarationNew]
+  ): Set[(DeclarationNew, String, String, Option[String])] = {
+    capturedNodes
+      .collect {
+        case local: NewLocal =>
+          val closureBindingId = scope.variableScopeFullName(local.name).map(x => s"$x.${local.name}")
+          (local, local.name, local.code, closureBindingId)
+        case param: NewMethodParameterIn =>
+          val closureBindingId = scope.variableScopeFullName(param.name).map(x => s"$x.${param.name}")
+          (param, param.name, param.code, closureBindingId)
+      }
+  }
 }

joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala

@@ -508,6 +508,10 @@ object RubyIntermediateAst {
       extends RubyExpression(span)
       with RubyCall
 
+  final case class ErbTemplateCall(target: RubyExpression, arguments: List[RubyExpression])(span: TextSpan)
+      extends RubyExpression(span)
+      with RubyCall
+
   sealed trait AccessModifier extends AllowedTypeDeclarationChild {
     def toSimpleIdentifier: SimpleIdentifier
   }
@@ -685,4 +689,5 @@ object RubyIntermediateAst {
       extends RubyExpression(span)
       with AllowedTypeDeclarationChild
 
+  final case class EmptyExpression(override val span: TextSpan) extends RubyExpression(span)
 }