Initial Soufflé codegen

langston-barrett · langston-barrett · commit 4454c41e8323 · 2022-10-26T18:48:03.000-04:00
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -11,6 +11,11 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
+    - name: Run Datalog generators
+      run: |
+        cargo run --bin treeedbgen-souffle-java -- --output treeedb-java.dl
+        cargo run --bin treeedbgen-souffle-souffle -- --output treeedb-souffle.dl
+
     - name: Deps
       run: |
         sudo apt-get install -y musl-tools
@@ -20,12 +25,11 @@ jobs:
       run: |
         make static
         cp target/x86_64-unknown-linux-musl/release/treeedb-java .
+        cp target/x86_64-unknown-linux-musl/release/treeedb-souffle .
 
     - uses: ncipollo/release-action@v1
       with:
-        artifacts: "treeedb-java"
+        artifacts: "treeedb-java,treeedb-java.dl,treeedb-souffle,treeedb-souffle.dl"
         artifactErrorsFailBuild: true
-        # TODO(lb):
-        # body: ""
         draft: true
         token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -2,5 +2,6 @@
 /target
 
 /*.csv
+/*.dl
 /*.java
 /*.tsv
diff --git a/Cargo.toml b/Cargo.toml
@@ -3,9 +3,11 @@
 members = [
     "treeedb",
     "treeedb-java",
+    "treeedb-souffle",
     "treeedbgen",
     "treeedbgen-souffle",
     "treeedbgen-souffle-java",
+    "treeedbgen-souffle-souffle",
 ]
 
 [profile.release]
diff --git a/README.md b/README.md
@@ -1,10 +1,47 @@
 # treeedb
 
-`treeedb` is a Rust library and collection of executables for generating
-relational views of [tree-sitter][tree-sitter] parse trees. These views may be
-used as inputs to a Datalog program or to construct a relational database.
+`treeedb` generates [Soufflé Datalog][souffle] types, relations, and facts that
+represent ASTs from a variety of programming languages. The parsers are based on
+[tree-sitter][tree-sitter] grammars.
+
+<!-- for f in **/Cargo.toml; do printf "- [\`%s\`](%s): %s\n" "$(dirname ${f})" "./$(dirname ${f})"  "$(grep descript "${f}" | grep -oP 'description = "\K[^"]+')"; done -->
+
+- [`treeedb`](./treeedb): Generate Datalog facts from tree-sitter parse trees
+- [`treeedbgen`](./treeedbgen): Parse node-types.json from a tree-sitter grammar
+- [`treeedbgen-souffle`](./treeedbgen-souffle): Generate Soufflé types and relations from tree-sitter grammars
+- [`treeedbgen-souffle-java`](./treeedbgen-souffle-java): Generate Soufflé types and relations from the Java tree-sitter grammar
+- [`treeedbgen-souffle-souffle`](./treeedbgen-souffle-souffle): Generate Soufflé types and relations from the Soufflé tree-sitter grammar
+- [`treeedb-java`](./treeedb-java): Generate Datalog facts from Java source code
+- [`treeedb-souffle`](./treeedb-souffle): Generate Datalog facts from Soufflé source code
+
+Don't see your favorite language? Adding a new one is *very* simple, see any of
+the existing languages. Or file an issue!
 
 The name is a portmanteau of "tree-sitter" with "EDB", where EDB stands for
 "extensional database" and refers to the set of facts in a Datalog program.
 
+## Usage
+
+### Example: Analyzing Java Code with Soufflé
+
+Navigate to the most recent release on the [releases page][releases]. Download
+the Java-related artifacts, namely the `treeedb-java` executable and
+`treeedb-java.dl`. Create a Java file:
+
+```java
+class HelloWorld {
+    public static void main(String[] args) {
+        int x = 2 + 2;
+    }
+}
+```
+
+Let's try to find constant-valued variables in this code!
+
+TODO(lb)!
+
+[cargo]: https://doc.rust-lang.org/cargo/
 [tree-sitter]: https://tree-sitter.github.io/tree-sitter/
+[releases]: https://github.com/langston-barrett/treeedb/releases
+[rustup]: https://rustup.rs/
+[souffle]: https://souffle-lang.github.io/index.html
diff --git a/treeedb-java/Cargo.toml b/treeedb-java/Cargo.toml
@@ -12,10 +12,8 @@ repository = "https://github.com/langston-barrett/treeedb"
 
 [dependencies]
 anyhow = "1.0"
-clap = { version = "4.0", features = ["derive"] }
-treeedb = { version = "0.1.0", path = "../treeedb" }
-tree-sitter = "0.20"
-tree-sitter-java = { version = "0.20", git = 'https://github.com/tree-sitter/tree-sitter-java', ref = "88339390aa70ca6784ecac82c34affa456dc6382" }
+treeedb = { version = "0.1.0", path = "../treeedb", features = ["cli"] }
+tree-sitter-java = { version = "0.20", git = 'https://github.com/tree-sitter/tree-sitter-java' }
 
 # TODO(lb): Enable when this hits stable
 # https://doc.rust-lang.org/cargo/reference/unstable.html#artifact-dependencies
diff --git a/treeedb-java/src/cli.rs b/treeedb-java/src/cli.rs
diff --git a/treeedb-java/src/main.rs b/treeedb-java/src/main.rs
@@ -1,29 +1,5 @@
-use std::fs;
-
-use anyhow::{Context, Result};
-use clap::Parser;
-
-mod cli;
-
-fn read_file(file: &str) -> Result<String> {
-    fs::read_to_string(file).with_context(|| format!("Failed to read file {}", file))
-}
-
-fn parse(language: tree_sitter::Language, code: &str) -> Result<tree_sitter::Tree> {
-    let mut parser = tree_sitter::Parser::new();
-    parser
-        .set_language(language)
-        .context("Failed to set tree-sitter parser language")?;
-    parser.parse(code, None).context("Failed to parse code")
-}
+use anyhow::Result;
 
 fn main() -> Result<()> {
-    let args = cli::Args::parse();
-    let language = tree_sitter_java::language();
-    let mut fc = treeedb::wide::WideCsvConsumer::new("node.csv".into(), "field.csv".into())?;
-    for path in args.java_files {
-        let tree = parse(language, &read_file(&path)?)?;
-        treeedb::facts(&mut fc, tree).unwrap();
-    }
-    Ok(())
+    treeedb::cli::main(tree_sitter_java::language())
 }
diff --git a/treeedb-souffle/Cargo.toml b/treeedb-souffle/Cargo.toml
@@ -0,0 +1,28 @@
+[package]
+name = "treeedb-souffle"
+version = "0.1.0"
+edition = "2021"
+description = "Generate Datalog facts from Soufflé source code"
+keywords = ["datalog", "parsing", "souffle"]
+authors = ["Langston Barrett <langston.barrett@gmail.com>"]
+license = "MIT"
+readme = "README.md"
+homepage = "https://github.com/langston-barrett/treeedb"
+repository = "https://github.com/langston-barrett/treeedb"
+
+[dependencies]
+anyhow = "1.0"
+treeedb = { version = "0.1.0", path = "../treeedb", features = ["cli"] }
+tree-sitter-souffle = { version = "0.4.0" }
+
+# TODO(lb): Enable when this hits stable
+# https://doc.rust-lang.org/cargo/reference/unstable.html#artifact-dependencies
+# [dev-dependencies]
+# cargo-deb = { version = "1", artifact = "bin" }
+
+# TODO(lb):
+# [package.metadata.deb]
+# assets = [
+#     ["../README.md", "usr/share/doc/treeedb-java/README.md", "644"],
+#     ["target/x86_64-unknown-linux-musl/release/treedb-java", "usr/bin/", "755"],
+# ]
diff --git a/treeedb-souffle/src/main.rs b/treeedb-souffle/src/main.rs
@@ -0,0 +1,5 @@
+use anyhow::Result;
+
+fn main() -> Result<()> {
+    treeedb::cli::main(tree_sitter_souffle::language())
+}
diff --git a/treeedb/Cargo.toml b/treeedb/Cargo.toml
@@ -11,5 +11,11 @@ homepage = "https://github.com/langston-barrett/treeedb"
 repository = "https://github.com/langston-barrett/treeedb"
 
 [dependencies]
+anyhow = { version = "1", optional = true }
+clap = { version = "4", features = ["derive"], optional = true }
 csv = "1"
-tree-sitter = "0.20"
+tree-sitter = "0.20"
+
+[features]
+default = []
+cli = ["dep:anyhow", "dep:clap"]
diff --git a/treeedb/src/cli.rs b/treeedb/src/cli.rs
@@ -0,0 +1,38 @@
+use std::fs;
+
+use anyhow::{Context, Result};
+use clap::Parser;
+
+/// Generate Datalog facts from source code
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+// TODO(lb): Option for output directory
+// TODO(lb): Option to raise an error on a parse failure
+pub struct Args {
+    /// Source code to consume; if empty, parse from stdin
+    #[arg(value_name = "SRC_FILE")]
+    pub source_files: Vec<String>,
+}
+
+fn read_file(file: &str) -> Result<String> {
+    fs::read_to_string(file).with_context(|| format!("Failed to read file {}", file))
+}
+
+fn parse(language: tree_sitter::Language, code: &str) -> Result<tree_sitter::Tree> {
+    let mut parser = tree_sitter::Parser::new();
+    parser
+        .set_language(language)
+        .context("Failed to set tree-sitter parser language")?;
+    parser.parse(code, None).context("Failed to parse code")
+}
+
+pub fn main(language: tree_sitter::Language) -> Result<()> {
+    let args = Args::parse();
+    let mut fc = super::wide::WideCsvConsumer::new("node.csv".into(), "field.csv".into())?;
+    // TODO(lb): read from stdin
+    for path in args.source_files {
+        let tree = parse(language, &read_file(&path)?)?;
+        super::facts(&mut fc, tree).unwrap();
+    }
+    Ok(())
+}
diff --git a/treeedb/src/lib.rs b/treeedb/src/lib.rs
@@ -5,6 +5,9 @@ pub mod count;
 pub mod narrow;
 pub mod wide;
 
+#[cfg(feature = "cli")]
+pub mod cli;
+
 pub fn facts<E>(fc: &mut impl consumer::FactConsumer<Err = E>, tree: Tree) -> Result<(), E> {
     let mut nodes = vec![tree.root_node()];
     let mut cursor = tree.walk();
diff --git a/treeedbgen-souffle-java/.gitignore b/treeedbgen-souffle-java/.gitignore
@@ -0,0 +1 @@
+*.csv
diff --git a/treeedbgen-souffle-java/Cargo.toml b/treeedbgen-souffle-java/Cargo.toml
@@ -12,10 +12,8 @@ repository = "https://github.com/langston-barrett/treeedb"
 
 [dependencies]
 anyhow = "1"
-clap = { version = "4", features = ["derive"] }
-treeedbgen-souffle = { version = "0.1.0", path = "../treeedbgen-souffle" }
-tree-sitter = "0.20"
-tree-sitter-java = { version = "0.20", git = 'https://github.com/tree-sitter/tree-sitter-java', ref = "88339390aa70ca6784ecac82c34affa456dc6382" }
+treeedbgen-souffle = { version = "0.1.0", path = "../treeedbgen-souffle", features = ["cli"] }
+tree-sitter-java = { version = "0.20", git = 'https://github.com/tree-sitter/tree-sitter-java' }
 
 # TODO(lb): Enable when this hits stable
 # https://doc.rust-lang.org/cargo/reference/unstable.html#artifact-dependencies
@@ -28,3 +26,8 @@ tree-sitter-java = { version = "0.20", git = 'https://github.com/tree-sitter/tre
 #     ["../README.md", "usr/share/doc/treeedb-java/README.md", "644"],
 #     ["target/x86_64-unknown-linux-musl/release/treedb-java", "usr/bin/", "755"],
 # ]
+
+[dev-dependencies]
+assert_cmd = "2.0"
+predicates = "2.1"
+tempfile = "3"
diff --git a/treeedbgen-souffle-java/src/cli.rs b/treeedbgen-souffle-java/src/cli.rs
diff --git a/treeedbgen-souffle-java/src/main.rs b/treeedbgen-souffle-java/src/main.rs
@@ -1,10 +1,5 @@
 use anyhow::Result;
-use clap::Parser;
-
-mod cli;
 
 fn main() -> Result<()> {
-    let _args = cli::Args::parse();
-    println!("{}", treeedbgen_souffle::gen(tree_sitter_java::NODE_TYPES)?);
-    Ok(())
+    treeedbgen_souffle::cli::main(tree_sitter_java::NODE_TYPES)
 }
diff --git a/treeedbgen-souffle-java/tests/java/HelloWorld.java b/treeedbgen-souffle-java/tests/java/HelloWorld.java
@@ -0,0 +1,5 @@
+class HelloWorld {
+    public static void main(String[] args) {
+        System.out.println("HelloWorld!");
+    }
+}
diff --git a/treeedbgen-souffle-java/tests/main.rs b/treeedbgen-souffle-java/tests/main.rs
@@ -0,0 +1,33 @@
+use std::process::Command;
+
+use assert_cmd::prelude::*;
+use predicates::prelude::*;
+use tempfile::NamedTempFile;
+
+#[test]
+fn gen() -> Result<(), Box<dyn std::error::Error>> {
+    let mut cmd = Command::cargo_bin("treeedbgen-souffle-java")?;
+    let tmp = NamedTempFile::new()?;
+    cmd.arg("-o").arg(tmp.path());
+    cmd.arg("--prefix=java").arg("--printsize");
+    cmd.assert()
+        .success()
+        .stdout(predicate::str::is_empty())
+        .stderr(predicate::str::is_empty());
+
+    let mut cmd = Command::cargo_bin("treeedb-java")?;
+    cmd.arg("tests/java/HelloWorld.java");
+    cmd.assert()
+        .success()
+        .stdout(predicate::str::is_empty())
+        .stderr(predicate::str::is_empty());
+
+    let mut souffle = Command::new("souffle");
+    souffle.arg(tmp.path());
+    souffle
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("java_node\t23"))
+        .stderr(predicate::str::is_empty());
+    Ok(())
+}
diff --git a/treeedbgen-souffle-souffle/.gitignore b/treeedbgen-souffle-souffle/.gitignore
@@ -0,0 +1 @@
+*.csv
diff --git a/treeedbgen-souffle-souffle/Cargo.toml b/treeedbgen-souffle-souffle/Cargo.toml
@@ -0,0 +1,33 @@
+[package]
+name = "treeedbgen-souffle-souffle"
+version = "0.1.0"
+edition = "2021"
+description = "Generate Soufflé types and relations from the Soufflé tree-sitter grammar"
+keywords = ["datalog", "java", "parsing", "souffle"]
+authors = ["Langston Barrett <langston.barrett@gmail.com>"]
+license = "MIT"
+readme = "README.md"
+homepage = "https://github.com/langston-barrett/treeedb"
+repository = "https://github.com/langston-barrett/treeedb"
+
+[dependencies]
+anyhow = "1"
+treeedbgen-souffle = { version = "0.1.0", path = "../treeedbgen-souffle", features = ["cli"] }
+tree-sitter-souffle = { version = "0.4.0" }
+
+# TODO(lb): Enable when this hits stable
+# https://doc.rust-lang.org/cargo/reference/unstable.html#artifact-dependencies
+# [dev-dependencies]
+# cargo-deb = { version = "1", artifact = "bin" }
+
+# TODO(lb):
+# [package.metadata.deb]
+# assets = [
+#     ["../README.md", "usr/share/doc/treeedb-java/README.md", "644"],
+#     ["target/x86_64-unknown-linux-musl/release/treedb-java", "usr/bin/", "755"],
+# ]
+
+[dev-dependencies]
+assert_cmd = "2.0"
+predicates = "2.1"
+tempfile = "3"
diff --git a/treeedbgen-souffle-souffle/src/main.rs b/treeedbgen-souffle-souffle/src/main.rs
@@ -0,0 +1,5 @@
+use anyhow::Result;
+
+fn main() -> Result<()> {
+    treeedbgen_souffle::cli::main(tree_sitter_souffle::NODE_TYPES)
+}
diff --git a/treeedbgen-souffle-souffle/tests/dl/hello-world.dl b/treeedbgen-souffle-souffle/tests/dl/hello-world.dl
@@ -0,0 +1,2 @@
+.decl hello_world(h: symbol)
+hello_world("Hello, World!").
diff --git a/treeedbgen-souffle-souffle/tests/main.rs b/treeedbgen-souffle-souffle/tests/main.rs
diff --git a/treeedbgen-souffle/Cargo.toml b/treeedbgen-souffle/Cargo.toml
diff --git a/treeedbgen-souffle/src/cli.rs b/treeedbgen-souffle/src/cli.rs
diff --git a/treeedbgen-souffle/src/gen.rs b/treeedbgen-souffle/src/gen.rs
diff --git a/treeedbgen-souffle/src/lib.rs b/treeedbgen-souffle/src/lib.rs
diff --git a/treeedbgen/Cargo.toml b/treeedbgen/Cargo.toml
diff --git a/treeedbgen/src/lib.rs b/treeedbgen/src/lib.rs

-Original file line number
+Diff line change
 /target
 /*.csv
 +/*.dl
 /*.java
 /*.tsv