Skip to content

Commit 4ed630e

Browse files
elegoffelegoff
authored
Hardening grammar (#18)
Improve the parsing capabilities of external Rust projects
1 parent d0ee36d commit 4ed630e

32 files changed

+1166
-281
lines changed

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@
5454
<sonar.version>7.9</sonar.version>
5555
<java.version>8</java.version>
5656
<sslr.version>1.24.0.633</sslr.version>
57-
<sslr-squid-bridge.version>2.7.0.377</sslr-squid-bridge.version>
57+
<sslr-squid-bridge.version>2.7.1.392</sslr-squid-bridge.version>
5858
<mockito.version>3.6.28</mockito.version>
5959
<assertj-core.version>3.19.0</assertj-core.version>
6060
<jacoco.version>0.8.6</jacoco.version>

‎rust-frontend/src/main/java/org/sonar/rust/RustGrammar.java‎

Lines changed: 346 additions & 146 deletions
Large diffs are not rendered by default.

‎rust-frontend/src/test/java/org/sonar/rust/RustLexerTest.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,15 @@
2020
package org.sonar.rust;
2121

2222
import com.google.common.base.Charsets;
23+
import com.sonar.sslr.api.AstNode;
2324
import com.sonar.sslr.api.Token;
25+
import com.sonar.sslr.impl.ast.AstXmlPrinter;
2426
import org.junit.Test;
27+
import org.sonar.sslr.parser.LexerlessGrammar;
28+
import org.sonar.sslr.parser.ParserAdapter;
2529
import org.sonar.sslr.tests.Assertions;
2630

31+
import java.nio.charset.StandardCharsets;
2732
import java.util.List;
2833

2934
import static org.fest.assertions.Assertions.assertThat;
@@ -59,4 +64,23 @@ public void testTokens() {
5964

6065
;
6166
}
67+
68+
@Test
69+
public void testParsing() {
70+
71+
72+
String sexpr = "#![feature(const_fn_fn_ptr_basics)]";
73+
74+
75+
//Print out Ast node content for debugging purpose
76+
77+
ParserAdapter<LexerlessGrammar> parser = new ParserAdapter<>(StandardCharsets.UTF_8, RustGrammar.create().build());
78+
AstNode rootNode = parser.parse(sexpr);
79+
org.fest.assertions.Assertions.assertThat(rootNode.getType()).isSameAs(RustGrammar.COMPILATION_UNIT);
80+
AstNode astNode = rootNode;
81+
//org.fest.assertions.Assertions.assertThat(astNode.getNumberOfChildren()).isEqualTo(4);
82+
System.out.println(AstXmlPrinter.print(astNode));
83+
84+
85+
}
6286
}

‎rust-frontend/src/test/java/org/sonar/rust/parser/attributes/AttributeTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ public void testAttribute() {
4343
public void testInnerAttribute() {
4444
assertThat(RustGrammar.create().build().rule(RustGrammar.INNER_ATTRIBUTE))
4545
.matches("#![crate_type = \"lib\"]")
46+
.matches("#![feature(const_fn_fn_ptr_basics)]")
4647
;
4748
}
4849

‎rust-frontend/src/test/java/org/sonar/rust/parser/expressions/BlockExpressionTest.java‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -40,18 +40,7 @@ public class BlockExpressionTest {
4040
*/
4141

4242

43-
@Test
44-
public void testStatements() {
45-
assertThat(RustGrammar.create().build().rule(RustGrammar.STATEMENTS))
46-
.matches("println!(\"hi there\");")
47-
.matches("println!(\"hi there\");\n" +
48-
"println!(\"how are you today ?\");")
4943

50-
.matches("j.set(i.get()); false")
51-
.matches("j.set(i.get() + 1); false")
52-
.matches("node_fetch::create_http_client(user_agent.clone(), my_data.clone()).unwrap()")
53-
;
54-
}
5544

5645
@Test
5746
public void testAsyncBlockExpression() {
@@ -65,6 +54,9 @@ public void testAsyncBlockExpression() {
6554
public void testBlockExpression() {
6655
assertThat(RustGrammar.create().build().rule(RustGrammar.BLOCK_EXPRESSION))
6756
.matches("{}")
57+
.matches("{\n" +
58+
" // comment\n" +
59+
"}")
6860
.matches("{let y=42;}")
6961
.matches("{println!(\"hi there\");}")
7062
.matches("{abc()}")
@@ -93,11 +85,9 @@ public void testBlockExpression() {
9385
.matches("{\n" +
9486
" self.len() as u32\n" +
9587
" }")
96-
/* FIXME
9788
.matches("{\n" +
9889
" &[b' ', b' ', b' '][0..(4 - (len & 3)) & 3]\n" +
9990
"}")
100-
*/
10191

10292
.matches("{ Box::new(move |state : Rc<RefCell<OpState>>, bufs: BufVec| -> Op {\n" +
10393
" let mut b = 42;\n" +
@@ -106,7 +96,7 @@ public void testBlockExpression() {
10696
.matches("{\n" +
10797
" PathBuf::from(\"/demo_dir/\")\n" +
10898
" }")
109-
.matches("{PathBuf::from(r\"C:\\demo_dir\\\")}")
99+
.matches("{PathBuf::from(r\"C:\\demodir\\\")}")
110100
.matches("{\n" +
111101
" if check {\n" +
112102
" check_source_files(config, paths).res1;\n" +
@@ -115,6 +105,12 @@ public void testBlockExpression() {
115105
" }\n" +
116106
" Ok(())\n" +
117107
" }")
108+
.matches("{\n" +
109+
" JsError {} \n" +
110+
"}")
111+
.matches("{\n" +
112+
" continue 'outer;\n" +
113+
"}")
118114

119115

120116

‎rust-frontend/src/test/java/org/sonar/rust/parser/expressions/CallExpressionTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ public class CallExpressionTest {
2929
@Test
3030
public void testCallParams() {
3131
assertThat(RustGrammar.create().build().rule(RustGrammar.CALL_PARAMS))
32-
//.matches("")
32+
.matches("..")
3333
.matches("1i32")
3434
.matches("{let y=42;}")
3535
.matches("{;}")

‎rust-frontend/src/test/java/org/sonar/rust/parser/expressions/ExpressionTest.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,28 @@
2626

2727
public class ExpressionTest {
2828

29+
@Test
30+
public void testExpressionExceptStruct() {
31+
assertThat(RustGrammar.create().build().rule(RustGrammar.EXPRESSION_EXCEPT_STRUCT))
32+
.matches("a")
33+
.matches("a || b")
34+
.matches("a() || b")
35+
.matches("a() || b()")
36+
.matches("a || b && c")
37+
.notMatches("my_struct{}")
38+
.notMatches("my_struct{}.field")
39+
.notMatches("a || not_struct {}")
40+
.matches("matches.value_of(\"log-level\").unwrap()")
41+
.matches("!a || b")
42+
.matches("a || !b")
43+
.matches("completions.is_empty() && !did_match")
44+
.notMatches("completions.is_empty() && !did_match { None } ")
45+
.matches("!c")
46+
.notMatches("!c { None }")
47+
.matches("continue 'outer")
2948

49+
;
50+
}
3051

3152
@Test
3253
public void testExpression() {
@@ -85,6 +106,13 @@ public void testExpression() {
85106
" Ok(())\n" +
86107
" }\n" +
87108
" .boxed_local()")
109+
.notMatches("is_ok {\n" +
110+
" // empty block" +
111+
" } ")
112+
.matches("..")
113+
.matches("break 42")
114+
.matches("break Ok(Poll::Pending)")
115+
88116
;
89117
}
90118
}

‎rust-frontend/src/test/java/org/sonar/rust/parser/expressions/ExpressionWithBlockTest.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ public void testExpressionWithBlock() {
3939
.matches("if run_coverage {\n" +
4040
" println!(\"Coverage is running\");" +
4141
" } ")
42+
.matches("async move {}")
43+
.notMatches("async move {}.f()")
4244
.matches("async move {\n" +
4345
" if check {\n" +
4446
" check_source_files(config, paths).await?;\n" +
@@ -47,6 +49,7 @@ public void testExpressionWithBlock() {
4749
" }\n" +
4850
" Ok(())\n" +
4951
" }")
52+
.notMatches("async {}.inc()")
5053
;
5154
}
5255
}

‎rust-frontend/src/test/java/org/sonar/rust/parser/expressions/ExpressionWithoutBlockTest.java‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,6 @@ public class ExpressionWithoutBlockTest {
2929
public void testExpressionWithoutBlock() {
3030
assertThat(RustGrammar.create().build().rule(RustGrammar.EXPRESSION_WITHOUT_BLOCK))
3131
.notMatches("== b")
32-
//TODO Litteral Expression
33-
//TODO Path expression
34-
//Operator expressdions
35-
//borrow expr
3632
.matches("&7")
3733
.matches("& Value")
3834
.matches("&mut array")
@@ -41,14 +37,14 @@ public void testExpressionWithoutBlock() {
4137
.matches("&&&& mut 10")
4238
.matches("&& && mut 10")
4339
.matches("& & & & mut 10")
44-
//deref
45-
.matches("*thing")
46-
//err propagation
47-
.matches("foo?")
40+
41+
.matches("*thing") //deref
42+
43+
.matches("foo?")//err propagation
4844
.matches("None?")
4945
.matches("Some(42)?")
50-
//negation expression
51-
.matches("!foo")
46+
47+
.matches("!foo")//negation expression
5248
.matches("-5")
5349
.matches("-bar")
5450
.notMatches("== b")
@@ -84,6 +80,7 @@ public void testExpressionWithoutBlock() {
8480
//TODO TupleExpression
8581
//TODO TupleIndexingExpression
8682
//TODO StructExpression
83+
.matches("mystruct{}")
8784
//TODO EnumerationVariantExpression
8885
// CallExpression
8986

@@ -94,13 +91,14 @@ public void testExpressionWithoutBlock() {
9491
//MethodCallExpression
9592
//
9693
//
97-
94+
.matches("async move {}.local()")
9895
.matches("\"123\".parse()")
9996
.matches("node_fetch::create_http_client(user_agent.clone(), my_data.clone()).unwrap()")
10097
//FieldExpression
10198
.matches("other.major")
10299
//TODO ClosureExpression
103-
//TODO ContinueExpression
100+
//ContinueExpression
101+
.matches("continue 'outer")
104102
//TODO BreakExpression
105103
//TODO RangeExpression
106104
//ReturnExpression
@@ -116,6 +114,14 @@ public void testExpressionWithoutBlock() {
116114
.matches("Vec::new")
117115
.matches("Identifier::Numeric")
118116
.matches("&[b' ', b' ', b' '][0..(4 - (len & 3)) & 3]")
117+
.matches("async move {}.await")
118+
.matches("async move {}.local()")
119+
.matches("async {}.inc()")
120+
121+
.matches("async move {}\n" +
122+
" .boxed()")
123+
124+
119125
;
120126
}
121127
}

‎rust-frontend/src/test/java/org/sonar/rust/parser/expressions/IfExpressionTest.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,20 @@ public void tesIfExpression() {
3939
.matches("if run_coverage {\n" +
4040
" println!(\"Coverage is running\");" +
4141
" } ")
42+
.matches("if is_ok {} ")
43+
.matches("if if_ok {} ")
44+
.matches("if match_ok {} ")
45+
.matches("if async_ok {} ")
46+
.matches("if is_red || is_black {let cpt = 1 ;} else {let cpt = 0 ;}")
47+
.matches("if is_red || is_black {}")
48+
.matches("if is_red || is_black {} else {let cpt = 0 ;}")
49+
.matches("if is_ok {\n" +
50+
" // empty block\n" +
51+
" } ")
52+
.matches("if is_ok {} else {let x = 42;}")
53+
.matches("if is_ok {\n" +
54+
" // empty block\n" +
55+
" } else {let x = 42;}")
4256
.matches("if bytes.len() < 3 * 4 {\n" +
4357
" println!(\"Too short\");" +
4458
" }")
@@ -68,6 +82,38 @@ public void tesIfExpression() {
6882
" }\n" +
6983
" }\n" +
7084
" }")
85+
.matches("if is_ok \n" +
86+
" {} else {\n" +
87+
" let msg = \"Module evaluation is still pending but there are no pending ops or dynamic imports. This situation is often caused by unresolved promise.\";\n" +
88+
" return Poll::Ready(Err(generic_error(msg)));\n" +
89+
" }")
90+
.matches("if is_ops || has_pending_dyn_imports || has_pending_dyn_module_evaluation\n" +
91+
" {} else {\n" +
92+
" let msg = \"Module evaluation is still pending but there are no pending ops or dynamic imports. This situation is often caused by unresolved promise.\";\n" +
93+
" return Poll::Ready(Err(generic_error(msg)));\n" +
94+
" }")
95+
.matches("if has_ops\n" +
96+
" || has_pending_dyn_imports\n" +
97+
" || has_pending_dyn_module_evaluation\n" +
98+
" {} else {\n" +
99+
" let msg = \"Module evaluation is still pending but there are no pending ops or dynamic imports. This situation is often caused by unresolved promise.\";\n" +
100+
" return Poll::Ready(Err(generic_error(msg)));\n" +
101+
" }")
102+
.matches("if has_pending_ops\n" +
103+
" || has_pending_dyn_imports\n" +
104+
" || has_pending_dyn_module_evaluation\n" +
105+
" {\n" +
106+
" // pass, will be polled again\n" +
107+
" } else {\n" +
108+
" let msg = \"Module evaluation is still pending but there are no pending ops or dynamic imports. This situation is often caused by unresolved promise.\";\n" +
109+
" return Poll::Ready(Err(generic_error(msg)));\n" +
110+
" }")
111+
.matches("if a && b { None }")
112+
.matches("if !c { None }")
113+
.matches("if a && !b { None }")
114+
.matches("if state.get_state() == MyState::KO {\n" +
115+
" continue 'outer;\n" +
116+
" }")
71117

72118

73119
;

0 commit comments

Comments
 (0)