Encode UTF-8 stuff with lossy conversion (#10)

Author: max-leuthaeuser

.github/workflows/pr.yml

@@ -7,24 +7,23 @@ jobs:
   test:
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           fetch-depth: '0'
       - name: Install sbt
-        if: matrix.os == 'macos-latest'
-        run: brew install sbt
+        uses: sbt/setup-sbt@v1
       - name: Install Swift
         uses: SwiftyLab/setup-swift@latest
         with:
           check-latest: true
           development: true
           swift-version: "5.10"
       - name: Run Linux Build
-        if: matrix.os == 'ubuntu-20.04'
+        if: matrix.os == 'ubuntu-latest'
         run: |
           swift build
           mv ./.build/debug/SwiftAstGen SwiftAstGen-linux

.github/workflows/release.yml

@@ -14,7 +14,7 @@ jobs:
   build:
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout
@@ -28,7 +28,7 @@ jobs:
           development: true
           swift-version: "5.10"
       - name: Run Linux Build
-        if: matrix.os == 'ubuntu-20.04'
+        if: matrix.os == 'ubuntu-latest'
         run: |
           swift build -c release --arch arm64 --arch x86_64 --static-swift-stdlib
           mv ./.build/release/SwiftAstGen SwiftAstGen-linux

Sources/SwiftAstGenLib/SyntaxParser.swift

@@ -50,15 +50,21 @@ extension SyntaxProtocol {
 
 struct SyntaxParser {
 
+	static func encode(_ s: String) -> String {
+    	let data = s.data(using: .ascii, allowLossyConversion: true)!
+    	return String(data: data, encoding: .utf8)!
+	}
+
 	static func parse(
 		srcDir: URL,
 		fileUrl: URL,
 		relativeFilePath: String,
 		prettyPrint: Bool
 	) throws -> String {
 		let code = try String(contentsOf: fileUrl)
+		let content = encode(code)
 		let opPrecedence = OperatorTable.standardOperators
-		let ast = Parser.parse(source: code)
+		let ast = Parser.parse(source: content)
 		let folded = try opPrecedence.foldAll(ast)
 
 		let locationConverter = SourceLocationConverter(fileName: fileUrl.path, tree: folded)
@@ -67,6 +73,7 @@ struct SyntaxParser {
 		treeNode.projectFullPath = srcDir.standardized.resolvingSymlinksInPath().path
 		treeNode.fullFilePath = fileUrl.standardized.resolvingSymlinksInPath().path
 		treeNode.relativeFilePath = relativeFilePath
+		treeNode.content = content
 
 		let encoder = JSONEncoder()
 		if prettyPrint { encoder.outputFormatting = .prettyPrinted }

Sources/SwiftAstGenLib/TreeNode.swift

@@ -3,6 +3,7 @@ final class TreeNode: Codable {
 	var projectFullPath: String?
 	var relativeFilePath: String?
 	var fullFilePath: String?
+	var content: String?
 
 	var index: Int
 	var name: String