0.23.2 - Add 'classSignature.comments' parsing and AST output.

TeamworkGuy2 · TeamworkGuy2 · commit 16de37b3016a · 2021-08-21T18:14:59.000Z
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,7 +4,13 @@ This project does its best to adhere to [Semantic Versioning](http://semver.org/
 
 
 --------
-### [0.23.1](N/A) - 2021-08-14
+### [0.23.2](N/A) - 2021-08-21
+#### Added
+* `classSignature.comments` parsing and AST output added
+
+
+--------
+### [0.23.1](https://github.com/TeamworkGuy2/JParseCode/commit/8dcac1d6a2dfe305dcd286a6d97068120f4e022e) - 2021-08-14
 #### Changed
 * Expanded C# and Java field parsing tests
 
diff --git a/README.md b/README.md
@@ -76,7 +76,8 @@ JSON Result (printed to System.out):
     "classSignature": {
         "access": "PUBLIC",
         "name": "ParserExamples.Samples.SimpleCs",
-        "declarationType": "class"
+        "declarationType": "class",
+        "comments": [" <summary>\n", " A simple class to test parsing.\n", " </summary>\n"]
     },
     "blockType": "CLASS",
     "using": [],
diff --git a/bin/jparse_code-with-tests.jar b/bin/jparse_code-with-tests.jar
diff --git a/bin/jparse_code.jar b/bin/jparse_code.jar
diff --git a/package-lib.json b/package-lib.json
@@ -1,5 +1,5 @@
 {
-	"version" : "0.23.1",
+	"version" : "0.23.2",
 	"name" : "jparse-code",
 	"description" : "An in-progress suite of parsing/transpilation tools for C#, Java, and TypeScript code.  Generates simple JSON ASTs.",
 	"homepage" : "https://github.com/TeamworkGuy2/JParseCode",
diff --git a/src/twg2/ast/interm/classes/ClassSigResolved.java b/src/twg2/ast/interm/classes/ClassSigResolved.java
@@ -26,13 +26,14 @@ public class ClassSigResolved implements ClassSig {
 	private final @Getter Keyword accessModifier;
 	/** The block's annotations */
 	private final @Getter List<AnnotationSig> annotations;
+	private final @Getter List<String> comments;
 	/** The block's type (i.e. 'interface', 'class', 'enum', etc.) */
 	private final @Getter String declarationType;
 	private final @Getter TypeSig.TypeSigResolved extendClass;
 	private final @Getter List<TypeSig.TypeSigResolved> implementInterfaces;
 
 
-	public ClassSigResolved(List<String> fullName, List<? extends TypeSigResolved> params, Keyword accessModifier, List<? extends AnnotationSig> annotations,
+	public ClassSigResolved(List<String> fullName, List<? extends TypeSigResolved> params, Keyword accessModifier, List<? extends AnnotationSig> annotations, List<String> comments,
 			String declarationType, TypeSigResolved extendClass, List<? extends TypeSigResolved> implementInterfaces) {
 		@SuppressWarnings("unchecked")
 		var paramsCast = (List<TypeSigResolved>)params;
@@ -45,6 +46,7 @@ public ClassSigResolved(List<String> fullName, List<? extends TypeSigResolved> p
 		this.params = paramsCast;
 		this.accessModifier = accessModifier;
 		this.annotations = annotationsCast;
+		this.comments = comments;
 		this.declarationType = declarationType;
 		this.extendClass = extendClass;
 		this.implementInterfaces = implementInterfacesCast;
@@ -91,6 +93,11 @@ public void toJson(Appendable dst, WriteSettings st) throws IOException {
 				.toArrayConsume(annotations, dst, (a) -> a.toJson(dst, st));
 		}
 
+		if(comments.size() > 0) {
+			json.comma(dst).propName("comments", dst)
+				.toStringArray(comments, dst);
+		}
+
 		dst.append(" }");
 	}
 
diff --git a/src/twg2/ast/interm/classes/ClassSigSimple.java b/src/twg2/ast/interm/classes/ClassSigSimple.java
@@ -25,10 +25,11 @@ public class ClassSigSimple implements ClassSig {
 	/** The block's type (i.e. 'interface', 'class', 'enum', etc.) */
 	private final @Getter String declarationType;
 	private final @Getter List<String> extendImplementSimpleNames;
+	private final @Getter List<String> comments;
 
 
 
-	public ClassSigSimple(List<String> fullName, List<? extends TypeSigSimple> params, Keyword accessModifier, List<? extends AnnotationSig> annotations, String declarationType, List<String> extendImplementSimpleNames) {
+	public ClassSigSimple(List<String> fullName, List<? extends TypeSigSimple> params, Keyword accessModifier, List<? extends AnnotationSig> annotations, List<String> comments, String declarationType, List<String> extendImplementSimpleNames) {
 		@SuppressWarnings("unchecked")
 		var paramsCast = (List<TypeSigSimple>)params;
 		@SuppressWarnings("unchecked")
@@ -38,6 +39,7 @@ public ClassSigSimple(List<String> fullName, List<? extends TypeSigSimple> param
 		this.params = paramsCast;
 		this.accessModifier = accessModifier;
 		this.annotations = annotationsCast;
+		this.comments = comments;
 		this.declarationType = declarationType;
 		this.extendImplementSimpleNames = extendImplementSimpleNames;
 	}
@@ -82,6 +84,11 @@ public void toJson(Appendable dst, WriteSettings st) throws IOException {
 				.toArrayConsume(annotations, dst, (a) -> a.toJson(dst, st));
 		}
 
+		if(comments.size() > 0) {
+			json.comma(dst).propName("comments", dst)
+				.toStringArray(comments, dst);
+		}
+
 		dst.append(" }");
 	}
 
diff --git a/src/twg2/parser/codeParser/csharp/CsBlockParser.java b/src/twg2/parser/codeParser/csharp/CsBlockParser.java
@@ -67,7 +67,7 @@ public AstParser<List<AnnotationSig>> createAnnotationParser(BlockAst<CsBlock> b
 	@Override
 	public AstParser<List<String>> createCommentParser(BlockAst<CsBlock> block) {
 		var lang = CodeLanguageOptions.C_SHARP;
-		return new CommentBlockExtractor(lang.displayName(), block);
+		return new CommentBlockExtractor(lang.displayName());
 	}
 
 
@@ -97,7 +97,8 @@ public List<Entry<SimpleTree<CodeToken>, ClassAst.SimpleImpl<CsBlock>>> extractC
 	public List<BlockAst<CsBlock>> extractBlocks(List<String> nameScope, SimpleTree<CodeToken> astTree, BlockAst<CsBlock> parentScope) {
 		List<BlockAst<CsBlock>> blocks = new ArrayList<>();
 		var annotationExtractor = new CsAnnotationExtractor();
-		extractBlocksFromTree(nameScope, astTree, 0, null, parentScope, annotationExtractor, blocks);
+		var commentExtractor = createCommentParser(parentScope);
+		extractBlocksFromTree(nameScope, astTree, 0, null, parentScope, annotationExtractor, commentExtractor, blocks);
 		return blocks;
 	}
 
@@ -108,8 +109,8 @@ public List<BlockAst<CsBlock>> extractBlocks(List<String> nameScope, SimpleTree<
 	 * @param depth the current blockTree's depth within the tree (0=root node, 1=child of root, etc.)
 	 * @param parentNode the current blockTree's parent node or null if the parent is null (only possible if blockTree is a child of a tree with a null root or blockTree is the root and has no parent)
 	 */
-	public static void extractBlocksFromTree(List<String> nameScope, SimpleTree<CodeToken> blockTree,
-			int depth, SimpleTree<CodeToken> parentNode, BlockAst<CsBlock> parentScope, CsAnnotationExtractor annotationExtractor, List<BlockAst<CsBlock>> blocks) {
+	public static void extractBlocksFromTree(List<String> nameScope, SimpleTree<CodeToken> blockTree, int depth, SimpleTree<CodeToken> parentNode, BlockAst<CsBlock> parentScope,
+			AstParser<List<AnnotationSig>> annotationExtractor, AstParser<List<String>> commentExtractor, List<BlockAst<CsBlock>> blocks) {
 		var lang = CodeLanguageOptions.C_SHARP;
 		var keywordUtil = lang.getKeywordUtil();
 		var children = blockTree.getChildren();
@@ -124,6 +125,7 @@ public static void extractBlocksFromTree(List<String> nameScope, SimpleTree<Code
 			var token = child.getData();
 
 			boolean annotAccepted = annotationExtractor.acceptNext(child);
+			boolean commentAccepted = commentExtractor.acceptNext(child) || annotAccepted; // keep comments if annotation was accepted so that comments before an annotation carry over if there is a block following the annotation(s)
 
 			int addBlockCount = 0;
 			BlockAst<CsBlock> newestBlock = null;
@@ -152,8 +154,9 @@ public static void extractBlocksFromTree(List<String> nameScope, SimpleTree<Code
 						var blockTypes = blockSig.isGeneric() ? blockSig.getParams() : Collections.<TypeSig.TypeSigSimple>emptyList();
 						var blockFqName = NameUtil.splitFqName(blockSig.getTypeName());
 						var annotations = new ArrayList<>(annotationExtractor.getParserResult());
+						var comments = new ArrayList<>(commentExtractor.getParserResult());
 
-						newestBlock = new BlockAst<>(new ClassSigSimple(blockFqName, blockTypes, access, annotations, blockTypeStr, nameCompoundRes.getValue()), child, blockType);
+						newestBlock = new BlockAst<>(new ClassSigSimple(blockFqName, blockTypes, access, annotations, comments, blockTypeStr, nameCompoundRes.getValue()), child, blockType);
 						blocks.add(newestBlock);
 					}
 
@@ -164,8 +167,11 @@ public static void extractBlocksFromTree(List<String> nameScope, SimpleTree<Code
 			// a valid block must have 2 or more children: a 'name' and a '{...}' block
 			// create a separate annotation extractor when extracting blocks within an annotation (not sure if this could ever happen)
 			if(child.size() > 1) {
-				extractBlocksFromTree(nameScope, child, depth + 1, blockTree, newestBlock != null ? newestBlock : parentScope, (annotAccepted ? annotationExtractor.copy() : annotationExtractor.recycle()), blocks);
+				var childAnnotExtractor = annotAccepted ? annotationExtractor.copy() : annotationExtractor.recycle();
+				var childCommentExtractor = commentAccepted ? commentExtractor.copy() : commentExtractor.recycle();
+				extractBlocksFromTree(nameScope, child, depth + 1, blockTree, newestBlock != null ? newestBlock : parentScope, childAnnotExtractor, childCommentExtractor, blocks);
 				if(!annotAccepted) { annotationExtractor.recycle(); }
+				if(!commentAccepted) { commentExtractor.recycle(); }
 			}
 
 			while(addBlockCount > 0) {
diff --git a/src/twg2/parser/codeParser/extractors/CommentBlockExtractor.java b/src/twg2/parser/codeParser/extractors/CommentBlockExtractor.java
@@ -3,8 +3,6 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import twg2.ast.interm.block.BlockAst;
-import twg2.parser.codeParser.BlockType;
 import twg2.parser.fragment.CodeToken;
 import twg2.parser.fragment.CodeTokenType;
 import twg2.parser.stateMachine.AstParserReusableBase;
@@ -25,16 +23,14 @@ static enum State {
 	}
 
 
-	BlockAst<? extends BlockType> parentBlock;
 	List<String> comments = new ArrayList<>();
 	boolean multiLine;
 	String langName;
 
 
-	public CommentBlockExtractor(String langName, BlockAst<? extends BlockType> parentBlock) {
+	public CommentBlockExtractor(String langName) {
 		super(langName + " field", State.COMPLETE, State.FAILED);
 		this.langName = langName;
-		this.parentBlock = parentBlock;
 		this.state = State.INIT;
 	}
 
@@ -121,7 +117,7 @@ public CommentBlockExtractor recycle() {
 
 	@Override
 	public CommentBlockExtractor copy() {
-		return new CommentBlockExtractor(this.langName, this.parentBlock);
+		return new CommentBlockExtractor(this.langName);
 	}
 
 
diff --git a/src/twg2/parser/codeParser/java/JavaBlockParser.java b/src/twg2/parser/codeParser/java/JavaBlockParser.java
@@ -65,7 +65,7 @@ public AstParser<List<AnnotationSig>> createAnnotationParser(BlockAst<JavaBlock>
 	@Override
 	public AstParser<List<String>> createCommentParser(BlockAst<JavaBlock> block) {
 		var lang = CodeLanguageOptions.JAVA;
-		return new CommentBlockExtractor(lang.displayName(), block);
+		return new CommentBlockExtractor(lang.displayName());
 	}
 
 
@@ -97,14 +97,15 @@ public List<Entry<SimpleTree<CodeToken>, ClassAst.SimpleImpl<JavaBlock>>> extrac
 	public List<BlockAst<JavaBlock>> extractBlocks(List<String> nameScope, SimpleTree<CodeToken> astTree, BlockAst<JavaBlock> parentScope) {
 		List<BlockAst<JavaBlock>> blocks = new ArrayList<>();
 		var annotationExtractor = new JavaAnnotationExtractor();
+		var commentExtractor = createCommentParser(parentScope);
 		// parse package name and push it into the name scope
 		String pkgName = parsePackageDeclaration(astTree);
 		if(pkgName == null) {
 			pkgName = "(default package)";
 		}
 		nameScope.add(pkgName);
 
-		extractBlocksFromTree(nameScope, astTree, 0, null, parentScope, annotationExtractor, blocks);
+		extractBlocksFromTree(nameScope, astTree, 0, null, parentScope, annotationExtractor, commentExtractor, blocks);
 		return blocks;
 	}
 
@@ -115,8 +116,8 @@ public List<BlockAst<JavaBlock>> extractBlocks(List<String> nameScope, SimpleTre
 	 * @param depth the current blockTree's depth within the tree (0=root node, 1=child of root, etc.)
 	 * @param parentNode the current blockTree's parent node or null if the parent is null (only possible if blockTree is a child of a tree with a null root or blockTree is the root and has no parent)
 	 */
-	public static void extractBlocksFromTree(List<String> nameScope, SimpleTree<CodeToken> blockTree,
-			int depth, SimpleTree<CodeToken> parentNode, BlockAst<JavaBlock> parentScope, JavaAnnotationExtractor annotationExtractor, List<BlockAst<JavaBlock>> blocks) {
+	public static void extractBlocksFromTree(List<String> nameScope, SimpleTree<CodeToken> blockTree, int depth, SimpleTree<CodeToken> parentNode, BlockAst<JavaBlock> parentScope,
+			AstParser<List<AnnotationSig>> annotationExtractor, AstParser<List<String>> commentExtractor, List<BlockAst<JavaBlock>> blocks) {
 		var lang = CodeLanguageOptions.JAVA;
 		var keywordUtil = lang.getKeywordUtil();
 		var children = blockTree.getChildren();
@@ -129,6 +130,7 @@ public static void extractBlocksFromTree(List<String> nameScope, SimpleTree<Code
 			var token = child.getData();
 
 			boolean annotAccepted = annotationExtractor.acceptNext(child);
+			boolean commentAccepted = commentExtractor.acceptNext(child) || annotAccepted; // keep comments if annotation was accepted so that comments before an annotation carry over if there is a block following the annotation(s)
 
 			int addBlockCount = 0;
 			BlockAst<JavaBlock> newestBlock = null;
@@ -157,8 +159,9 @@ public static void extractBlocksFromTree(List<String> nameScope, SimpleTree<Code
 						var blockTypes = blockSig.isGeneric() ? blockSig.getParams() : Collections.<TypeSig.TypeSigSimple>emptyList();
 						var blockFqName = NameUtil.splitFqName(blockSig.getTypeName());
 						var annotations = new ArrayList<>(annotationExtractor.getParserResult());
+						var comments = new ArrayList<>(commentExtractor.getParserResult());
 
-						newestBlock = new BlockAst<>(new ClassSigSimple(blockFqName, blockTypes, access, annotations, blockTypeStr, nameCompoundRes.getValue()), child, blockType);
+						newestBlock = new BlockAst<>(new ClassSigSimple(blockFqName, blockTypes, access, annotations, comments, blockTypeStr, nameCompoundRes.getValue()), child, blockType);
 						blocks.add(newestBlock);
 					}
 
@@ -168,8 +171,11 @@ public static void extractBlocksFromTree(List<String> nameScope, SimpleTree<Code
 
 			// a valid block must have 2 or more children: a 'name' and a '{...}' block
 			if(child.size() > 1) {
-				extractBlocksFromTree(nameScope, child, depth + 1, blockTree, newestBlock != null ? newestBlock : parentScope, (annotAccepted ? annotationExtractor.copy() : annotationExtractor.recycle()), blocks);
+				var childAnnotExtractor = annotAccepted ? annotationExtractor.copy() : annotationExtractor.recycle();
+				var childCommentExtractor = commentAccepted ? commentExtractor.copy() : commentExtractor.recycle();
+				extractBlocksFromTree(nameScope, child, depth + 1, blockTree, newestBlock != null ? newestBlock : parentScope, childAnnotExtractor, childCommentExtractor, blocks);
 				if(!annotAccepted) { annotationExtractor.recycle(); }
+				if(!commentAccepted) { commentExtractor.recycle(); }
 			}
 
 			while(addBlockCount > 0) {
diff --git a/src/twg2/parser/resolver/ClassSigResolver.java b/src/twg2/parser/resolver/ClassSigResolver.java
@@ -98,7 +98,7 @@ public static <T_ID, T_SIG extends ClassSigSimple, T_BLOCK extends BlockType> Cl
 
 		var classFqName = classSig.getFullName();
 
-		return new ClassSigResolved(classFqName, resolvedClassParams, classSig.getAccessModifier(), classSig.getAnnotations(),
+		return new ClassSigResolved(classFqName, resolvedClassParams, classSig.getAccessModifier(), classSig.getAnnotations(), classSig.getComments(),
 				classSig.getDeclarationType(), extendClassType, implementInterfaceTypes);
 	}
 
diff --git a/test/twg2/parser/codeParser/test/CsAnnotationParseTest.java b/test/twg2/parser/codeParser/test/CsAnnotationParseTest.java
@@ -107,6 +107,22 @@ public class CsAnnotationParseTest {
 		"}"
 	);
 
+	private static List<String> src2Lines = ls(
+		"namespace ParserExamples.Samples {",
+		"  /// <summary>",
+		"  /// First comment for annotation and comment parsing.",
+		"  /// Line two.",
+		"  /// </summary>",
+		"  [PluralAnnotation(1, \"st\")]",
+		"  /// <summary>Second comment.</summary>",
+		"  [PluralAnnotation(2, \"nd\")]",
+		"  public class CsAnnotationComments {",
+		"",
+		"  }",
+		"",
+		"}"
+	);
+
 
 	@Test
 	public void annotationsParseTest() {
@@ -204,4 +220,28 @@ public void annotationsParseTest() {
 		assertParameter(ps, 1, "states", "string[]", false, null, ls(CsKeyword.PARAMS), null);
 	}
 
+
+	@Test
+	public void annotationCommentsParseTest() {
+		CodeFileAndAst<CsBlock> csAnnotations = CodeFileAndAst.<CsBlock>parse(CodeLanguageOptions.C_SHARP, "CsAnnotationComments.cs", "ParserExamples.Samples.CsAnnotationComments", true, src2Lines);
+		List<CodeFileParsed.Simple<CsBlock>> blocks = csAnnotations.parsedBlocks;
+
+		String fullClassName = csAnnotations.fullClassName;
+		Assert.assertEquals(1, blocks.size());
+		ClassAst.SimpleImpl<CsBlock> clas = blocks.get(0).parsedClass;
+		Assert.assertEquals(0, clas.getUsingStatements().size());
+		Assert.assertEquals(0, clas.getFields().size());
+		Assert.assertEquals(0, clas.getMethods().size());
+		Assert.assertEquals(ls(" <summary>\n",
+				" First comment for annotation and comment parsing.\n",
+				" Line two.\n",
+				" </summary>\n",
+				" <summary>Second comment.</summary>\n"), clas.getSignature().getComments());
+
+		Assert.assertEquals(fullClassName, NameUtil.joinFqName(clas.getSignature().getFullName()));
+		Assert.assertEquals(AccessModifierEnum.PUBLIC, clas.getSignature().getAccessModifier());
+		Assert.assertEquals("class", clas.getSignature().getDeclarationType());
+		AnnotationAssert.assertAnnotation(clas.getSignature().getAnnotations(), 0, "PluralAnnotation", new String[] { "arg1", "arg2" }, "1", "st");
+		AnnotationAssert.assertAnnotation(clas.getSignature().getAnnotations(), 1, "PluralAnnotation", new String[] { "arg1", "arg2" }, "2", "nd");
+	}
 }
diff --git a/test/twg2/parser/codeParser/test/CsClassParseTest.java b/test/twg2/parser/codeParser/test/CsClassParseTest.java
@@ -35,6 +35,7 @@ public class CsClassParseTest {
 		"",
 		"  /// <summary>",
 		"  /// A simple class to test parsing.",
+		"  /// Since when.",
 		"  /// </summary>",
 		"  [StringAnnotation(\"-SimpleCs-\")]",
 		"  public class SimpleCs {",
@@ -111,6 +112,10 @@ public void simpleCsParseTest() {
 		Assert.assertEquals(AccessModifierEnum.PUBLIC, clas.getSignature().getAccessModifier());
 		Assert.assertEquals("class", clas.getSignature().getDeclarationType());
 		AnnotationAssert.assertAnnotation(clas.getSignature().getAnnotations(), 0, "StringAnnotation", new String[] { "value" }, "-SimpleCs-");
+		Assert.assertEquals(ls(" <summary>\n",
+				" A simple class to test parsing.\n",
+				" Since when.\n",
+				" </summary>\n"), clas.getSignature().getComments());
 
 		var fields = clas.getFields();
 		assertField(fields, 0, fullClassName + ".mod", "int");
diff --git a/test/twg2/parser/codeParser/test/CsFieldsParseTest.java b/test/twg2/parser/codeParser/test/CsFieldsParseTest.java
@@ -84,6 +84,9 @@ public void simpleCsParseTest() {
 		Assert.assertEquals(fullClassName, NameUtil.joinFqName(clas.getSignature().getFullName()));
 		Assert.assertEquals(AccessModifierEnum.PUBLIC, clas.getSignature().getAccessModifier());
 		Assert.assertEquals("class", clas.getSignature().getDeclarationType());
+		Assert.assertEquals(ls(" <summary>\n",
+				" A simple class to test field parsing.\n",
+				" </summary>\n"), clas.getSignature().getComments());
 
 		List<FieldDef> fields = clas.getFields();
 		assertField(fields, 0, fullClassName + ".mod", "int", "-1");
diff --git a/test/twg2/parser/codeParser/test/CsModelParseTest.java b/test/twg2/parser/codeParser/test/CsModelParseTest.java
@@ -89,6 +89,9 @@ public void model1CsParseTest() {
 		Assert.assertEquals(fullClassName, NameUtil.joinFqName(clas.getSignature().getFullName()));
 		Assert.assertEquals(AccessModifierEnum.INHERITANCE_LOCAL, clas.getSignature().getAccessModifier());
 		Assert.assertEquals("class", clas.getSignature().getDeclarationType());
+		Assert.assertEquals(ls(" <summary>\n",
+				" A simple class to test parsing.\n",
+				" </summary>\n"), clas.getSignature().getComments());
 
 		assertField(fs, 0, fullClassName + ".mod", "int");
 		Assert.assertEquals(ls(" <value>The modification count.</value>\n"), fs.get(0).getComments());
diff --git a/test/twg2/parser/codeParser/test/JavaAnnotationParseTest.java b/test/twg2/parser/codeParser/test/JavaAnnotationParseTest.java
diff --git a/test/twg2/parser/codeParser/test/JavaClassParseTest.java b/test/twg2/parser/codeParser/test/JavaClassParseTest.java
diff --git a/test/twg2/parser/codeParser/test/JavaFieldsParseTest.java b/test/twg2/parser/codeParser/test/JavaFieldsParseTest.java
diff --git a/test/twg2/parser/codeParser/test/JavaModelParseTest.java b/test/twg2/parser/codeParser/test/JavaModelParseTest.java

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,5 @@`
`1`	`1`	`{`
`2`		`- "version" : "0.23.1",`
	`2`	`+ "version" : "0.23.2",`
`3`	`3`	`"name" : "jparse-code",`
`4`	`4`	`"description" : "An in-progress suite of parsing/transpilation tools for C#, Java, and TypeScript code. Generates simple JSON ASTs.",`
`5`	`5`	`"homepage" : "https://github.com/TeamworkGuy2/JParseCode",`