Python bindings

Author: RunDevelopment

CMakeLists.txt

@@ -1,6 +1,8 @@
 cmake_minimum_required(VERSION 3.14)
 project(netspeak4)
 
+add_subdirectory(build/dependencies/pybind11-2.9.0)
+
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
@@ -14,9 +16,11 @@ set(CMAKE_CXX_FLAGS_MINSIZEREL     "-Os -DNDEBUG")
 set(CMAKE_CXX_FLAGS_RELEASE        "-O3 -DNDEBUG")
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g")
 
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 include_directories(
     "src"
+    "build/dependencies/pybind11-2.9.0/include"
     "/usr/include/antlr4-runtime"
 )
 
@@ -233,3 +237,49 @@ target_link_libraries(netspeak4 "${NETSPEAK_LINK_LIBS}")
 add_executable(netspeak4-test test/netspeak/runner.cpp)
 target_sources(netspeak4-test PRIVATE "${NETSPEAK_TEST_SOURCES}")
 target_link_libraries(netspeak4-test "${NETSPEAK_TEST_LINK_LIBS}")
+
+set(NETSPEAK_PY_LINK_LIBS
+dl
+z
+rt
+
+# boost
+boost_system
+boost_regex
+
+# antlr4
+antlr4-runtime
+)
+
+set(NETSPEAK_PY_SOURCES
+"src/py/QueryParser"
+
+"src/antlr4/parse"
+"src/antlr4/QueryErrorHandler"
+
+"src/antlr4/generated/QueryBaseListener"
+"src/antlr4/generated/QueryBaseVisitor"
+"src/antlr4/generated/QueryLexer"
+"src/antlr4/generated/QueryListener"
+"src/antlr4/generated/QueryParser"
+"src/antlr4/generated/QueryVisitor"
+
+"src/netspeak/Dictionaries"
+"src/netspeak/error"
+"src/netspeak/QueryNormalizer"
+
+"src/netspeak/model/LengthRange"
+"src/netspeak/model/NormQuery"
+"src/netspeak/model/SimpleQuery"
+"src/netspeak/model/Query"
+"src/netspeak/model/QuerySyntax"
+
+"src/netspeak/regex/DefaultRegexIndex"
+"src/netspeak/regex/parsers"
+"src/netspeak/regex/RegexIndex"
+"src/netspeak/regex/RegexQuery"
+)
+
+pybind11_add_module(netspeak4py src/PythonBindings.cpp "${NETSPEAK_PY_SOURCES}")
+set_property(TARGET netspeak4py PROPERTY POSITION_INDEPENDENT_CODE ON)
+target_link_libraries(netspeak4py PRIVATE pybind11::module "${NETSPEAK_PY_LINK_LIBS}")

README.md

@@ -321,6 +321,23 @@ Run the proxy:
 docker run -p 8080:8080 webis/grpcwebproxy:0.14.0 grpcwebproxy --allow_all_origins --backend_addr=host.docker.internal:9000 --backend_tls=false --run_tls_server=false
 ```
 
+
+## Python bindings
+
+This project also contains Python 3 bindings using pybind11.
+
+The Python bindings are a shared library (`.so`) and there are some limitations that come from this. The most notable limitation is that the bindings have runtime dependencies (the other Netspeak binaries all statically link their dependencies).
+
+To run the bindings, you need to have boost-regex and boost-system installed. Assuming that you copied `netspeak4py.cpython-38-x86_64-linux-gnu.so` from the builder Docker container, then you also need to copy `/usr/lib/libantlr4-runtime.so` and `/usr/lib/libantlr4-runtime.so.4.7.1`.
+
+To actually import the bindings in Python, make sure pass the directory `netspeak4py...so` is located via the [`PYTHONPATH`](https://docs.python.org/3/using/cmdline.html#envvar-PYTHONPATH) environment variable. If everything is setup correctly, Netspeak's Python bindings can be imported via:
+
+```py
+import netspeak4py
+```
+
+To see the API of the bindings, run `help(netspeak4py)`.
+
 ---
 
 ## Contributors

build/install-dependencies.sh

@@ -11,7 +11,7 @@ fi
 cd "$(dirname "$0")"
 
 apt-get update --allow-releaseinfo-change
-apt-get install make clang build-essential libboost-filesystem-dev libboost-system-dev libboost-test-dev libboost-program-options-dev libboost-stacktrace-dev libicu-dev libcmph-dev libaio-dev libboost-regex-dev libboost-date-time-dev -y
+apt-get install make clang build-essential libboost-filesystem-dev libboost-system-dev libboost-test-dev libboost-program-options-dev libboost-stacktrace-dev libicu-dev libcmph-dev libaio-dev libboost-regex-dev libboost-date-time-dev python3-dev -y
 
 
 bash ./env/install-antlr4.sh
@@ -22,6 +22,21 @@ mkdir -p ./dependencies
 chmod a+rw -R ./dependencies
 cd ./dependencies
 
+
+# Download pybind11
+if [ ! -d ./pybind11-2.9.0 ]; then
+    echo "Downloading pybind11"
+
+    apt-get install unzip wget -y
+    wget -O pybind.zip 'https://github.com/pybind/pybind11/archive/refs/tags/v2.9.0.zip'
+    unzip pybind.zip
+    rm pybind.zip
+    chmod a+rw ./**/*
+else
+    echo "pybind11 already present"
+fi
+
+
 if [[ "$1" != "ci" ]]; then
 
     # Download the JAR necessary to copmile .g4 files

build/run-tests.sh

@@ -30,3 +30,6 @@ fi
 
 echo "Running test..."
 ./build/debug/netspeak4-test --build_info=yes --detect_memory_leak=1 --log_level=test_suite
+
+echo "Running python tests..."
+PYTHONPATH="$(realpath ./build/debug):$PYTHONPATH" python3 ./test/py/test.py

src/PythonBindings.cpp

@@ -0,0 +1,92 @@
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
+#include <memory>
+
+#include "py/QueryParser.hpp"
+
+using namespace py;
+
+
+PYBIND11_MODULE(netspeak4py, m) {
+  m.doc() = "Netspeak4 Python bindings"; // optional module docstring
+
+  pybind11::class_<NormQueryUnit> normQueryUnit(m, "NormQueryUnit");
+  normQueryUnit
+      .def(pybind11::init<NormQueryUnit::Kind, std::string,
+                          NormQueryUnit::QueryUnitKind>(),
+           "Create a new norm query unit", pybind11::arg("kind"),
+           pybind11::arg("text"), pybind11::arg("sourceKind"))
+      .def("getKind", &NormQueryUnit::get_kind)
+      .def("getText", &NormQueryUnit::get_text)
+      .def("getSourceKind", &NormQueryUnit::get_source_kind)
+      .def("__repr__", [](const NormQueryUnit& u) {
+        const auto kind = u.get_kind();
+        const std::string& kind_str = kind == NormQueryUnit::Kind::WORD ? "NormWord"
+                               : kind == NormQueryUnit::Kind::QMARK
+                                   ? "NormQMark"
+                                   : "Invalid";
+
+        return "<netspeak4py.NormQueryUnit " + kind_str + " \"" +
+               (u.get_text()) + "\">";
+      });
+
+  pybind11::enum_<NormQueryUnit::Kind>(normQueryUnit, "Kind")
+      .value("NormWord", NormQueryUnit::Kind::WORD)
+      .value("NormQMark", NormQueryUnit::Kind::QMARK)
+      .export_values();
+
+  pybind11::enum_<NormQueryUnit::QueryUnitKind>(normQueryUnit, "QueryUnitKind")
+      .value("WORD", NormQueryUnit::QueryUnitKind::WORD)
+      .value("QMARK", NormQueryUnit::QueryUnitKind::QMARK)
+      .value("STAR", NormQueryUnit::QueryUnitKind::STAR)
+      .value("PLUS", NormQueryUnit::QueryUnitKind::PLUS)
+      .value("REGEX", NormQueryUnit::QueryUnitKind::REGEX)
+      .value("DICTSET", NormQueryUnit::QueryUnitKind::DICTSET)
+      .value("ORDERSET", NormQueryUnit::QueryUnitKind::ORDERSET)
+      .value("OPTIONSET", NormQueryUnit::QueryUnitKind::OPTIONSET)
+      .value("ALTERNATION", NormQueryUnit::QueryUnitKind::ALTERNATION)
+      .value("CONCAT", NormQueryUnit::QueryUnitKind::CONCAT)
+      .export_values();
+
+  pybind11::class_<NormQuery> normQuery(m, "NormQuery");
+  normQuery.def(pybind11::init<>())
+      .def("getUnits", &NormQuery::get_units)
+      .def("__repr__", [](NormQuery& q) {
+        std::string s = "<netspeak4py.NormQuery \"";
+        auto first = true;
+        for (const auto& u : q.get_units()) {
+          if (first) {
+            first = false;
+          } else {
+            s.push_back(' ');
+          }
+
+          s.append(u.get_text());
+        }
+
+        s.append("\">");
+        return s;
+      });
+  ;
+
+  pybind11::class_<QueryParserOptions> queryParserOptions(m,
+                                                          "QueryParserOptions");
+  queryParserOptions.def(pybind11::init<>())
+      .def_readwrite("maxNormQueries", &QueryParserOptions::max_norm_queries)
+      .def_readwrite("minLength", &QueryParserOptions::min_length)
+      .def_readwrite("maxLength", &QueryParserOptions::max_length)
+      .def_readwrite("maxRegexMatches", &QueryParserOptions::max_regex_matches)
+      .def_readwrite("maxRegexTimeMs", &QueryParserOptions::max_regex_time_ms);
+
+  pybind11::class_<QueryParser> queryParser(m, "QueryParser");
+  queryParser
+      .def(pybind11::init<const std::string&, const std::string&, bool>(),
+           "Creates a new query parser.", pybind11::arg("vocab") = "",
+           pybind11::arg("dictionaryCsv") = "",
+           pybind11::arg("lowerCase") = false)
+      .def("parse", &QueryParser::parse,
+           "Parses the given Netspeak 4 query and returns a list of norm "
+           "queries.",
+           pybind11::arg("query"), pybind11::arg("options"));
+}

src/netspeak/Dictionaries.cpp

@@ -15,11 +15,16 @@ const Dictionaries::Map Dictionaries::read_from_file(
     const boost::filesystem::path& csv) {
   boost::filesystem::ifstream ifs(csv);
   util::check(ifs.is_open(), error_message::cannot_open, csv);
+  return parse_csv(ifs);
+}
+
+const Dictionaries::Map Dictionaries::parse_csv(
+    std::basic_istream<char, std::char_traits<char>>& stream) {
   Map dict;
   std::string line;
   std::vector<std::string> tokens;
   const auto predicate = std::bind2nd(std::equal_to<char>(), '\t');
-  while (std::getline(ifs, line)) {
+  while (std::getline(stream, line)) {
     boost::split(tokens, line, predicate);
     if (tokens.size() < 2)
       continue;

src/netspeak/Dictionaries.hpp

@@ -22,6 +22,8 @@ class Dictionaries {
   typedef std::unordered_multimap<std::string, std::string> Map;
 
   static const Map read_from_file(const boost::filesystem::path& csv);
+  static const Map parse_csv(
+      std::basic_istream<char, std::char_traits<char>>& stream);
 };
 
 } // namespace netspeak

src/netspeak/model/QuerySyntax.hpp

@@ -4,8 +4,6 @@
 #include <ostream>
 #include <string>
 
-#include "netspeak/value/pair.hpp"
-
 namespace netspeak {
 namespace model {
 

src/py/QueryParser.cpp

@@ -0,0 +1,93 @@
+#include "py/QueryParser.hpp"
+
+#include <sstream>
+
+#include "antlr4/parse.hpp"
+
+#include "netspeak/Dictionaries.hpp"
+#include "netspeak/regex/DefaultRegexIndex.hpp"
+#include "netspeak/regex/RegexIndex.hpp"
+
+namespace py {
+
+NormQueryUnit::NormQueryUnit(NormQueryUnit::Kind kind, std::string text,
+                             QueryUnitKind source_kind)
+    : kind_(kind), text_(text), source_kind_(source_kind) {}
+
+NormQueryUnit::Kind NormQueryUnit::get_kind() const {
+  return kind_;
+}
+const std::string& NormQueryUnit::get_text() const {
+  return text_;
+}
+NormQueryUnit::QueryUnitKind NormQueryUnit::get_source_kind() const {
+  return source_kind_;
+}
+
+
+std::vector<NormQueryUnit>& NormQuery::get_units() {
+  return units_;
+}
+
+
+netspeak::QueryNormalizer::InitConfig get_init_config(
+    const std::string& vocab, const std::string& dictionary_csv,
+    bool lower_case) {
+  std::shared_ptr<netspeak::regex::RegexIndex> regex_index = nullptr;
+  if (!vocab.empty()) {
+    regex_index = std::make_shared<netspeak::regex::DefaultRegexIndex>(vocab);
+  }
+
+  std::shared_ptr<netspeak::Dictionaries::Map> dictionary = nullptr;
+  if (!dictionary_csv.empty()) {
+    std::istringstream iss(dictionary_csv);
+    dictionary = std::make_shared<netspeak::Dictionaries::Map>(
+        std::move(netspeak::Dictionaries::parse_csv(iss)));
+  }
+
+  return {
+    .regex_index = regex_index,
+    .dictionary = dictionary,
+    .lower_case = lower_case,
+  };
+}
+QueryParser::QueryParser(const std::string& vocab,
+                         const std::string& dictionary_csv, bool lower_case)
+    : normalizer(get_init_config(vocab, dictionary_csv, lower_case)) {}
+
+NormQuery convert_norm_query(netspeak::model::NormQuery input) {
+  NormQuery result;
+  for (const auto& unit : input.units()) {
+    NormQueryUnit::Kind kind = unit.tag();
+    std::string text = *unit.text();
+    NormQueryUnit::QueryUnitKind source_kind = unit.source().unit->tag();
+
+    result.get_units().push_back(NormQueryUnit(kind, text, source_kind));
+  }
+  return result;
+}
+std::vector<NormQuery> QueryParser::parse(const std::string& query,
+                                          QueryParserOptions options) {
+  const auto parsed_query = antlr4::parse_query(query);
+  std::vector<netspeak::model::NormQuery> norm_queries;
+  normalizer.normalize(parsed_query,
+                       {
+                           .max_norm_queries = options.max_norm_queries,
+                           .min_length = options.min_length,
+                           .max_length = options.max_length,
+                           .max_regex_matches = options.max_regex_matches,
+                           .max_regex_time = std::chrono::milliseconds(
+                               options.max_regex_time_ms),
+                       },
+                       norm_queries);
+
+  // convert norm query types
+  std::vector<NormQuery> result;
+  for (const auto& nq : norm_queries) {
+    result.push_back(convert_norm_query(nq));
+  }
+
+  return result;
+}
+
+} // namespace py