Merge pull request #120 from snipsco/task/portuguese-v0

ClemDoum · web-flow · commit 3f5a57aaf600 · 2019-02-28T17:53:38.000+01:00
Add bindings for Portuguese - PT and BR variants.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,11 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## [0.64.0] - 2019-02-28
+### Changed
+- Bumped `snips-nlu-ontology` to 0.64.4
+- Bumped `crf-suite` to 0.3.1 and cbindgen to
+
 ## [0.63.1] - 2019-02-11
 ### Fixed
 - Fix an issue regarding the way builtin entities were handled by the `CRFSlotFiller`
@@ -169,6 +174,7 @@ being statically hardcoded, reducing the binary size by 31Mb.
 - Improve support for japanese
 - Rename python package to `snips_nlu_rust`
 
+[0.64.0]: https://github.com/snipsco/snips-nlu-rs/compare/0.63.1...0.64.0
 [0.63.1]: https://github.com/snipsco/snips-nlu-rs/compare/0.63.0...0.63.1
 [0.63.0]: https://github.com/snipsco/snips-nlu-rs/compare/0.62.0...0.63.0
 [0.61.2]: https://github.com/snipsco/snips-nlu-rs/compare/0.61.1...0.61.2
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "snips-nlu-lib"
-version = "0.63.1"
+version = "0.64.0"
 authors = [
     "Adrien Ball <adrien.ball@snips.ai>",
     "Clement Doumouro <clement.doumouro@snips.ai>",
@@ -13,34 +13,33 @@ edition = "2018"
 
 [workspace]
 members = [
-    ".",
     "ffi",
     "platforms/python/ffi",
 ]
 
 [dependencies]
-crfsuite = { git = "https://github.com/snipsco/crfsuite-rs", rev = "91fda4b" }
-snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.63.1" }
-snips-nlu-parsers = { git = "https://github.com/snipsco/snips-nlu-parsers", tag = "0.1.2" }
-snips-nlu-utils = { git = "https://github.com/snipsco/snips-nlu-utils", tag = "0.7.1" }
+crfsuite = { git = "https://github.com/snipsco/crfsuite-rs", tag = "0.3.1" }
+snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.64.4" }
+snips-nlu-parsers = { git = "https://github.com/snipsco/snips-nlu-parsers", tag = "0.2.0" }
+snips-nlu-utils = { git = "https://github.com/snipsco/snips-nlu-utils", tag = "0.8.0" }
 failure = "0.1"
-base64 = "0.9"
-itertools = { version = "0.7", default-features = false }
+base64 = "0.10"
+itertools = { version = "0.8", default-features = false }
 log = "0.4"
 lru-cache = "0.1"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
 tempfile = "3"
-ndarray = "0.11"
+ndarray = "0.12"
 regex = "1.0"
 csv = "1.0"
-zip = { version = "0.3", features=["deflate"] }
+zip = { version = "0.5", default-features = false, features = ["deflate"] }
 
 [dev-dependencies]
 bencher = { git = "https://github.com/snipsco/bencher", rev = "63910ace" }
 clap = "2.32"
-dinghy-test = "0.3"
+dinghy-test = "0.4"
 maplit = "1.0"
 fs_extra = "1.1"
 
diff --git a/ffi/Cargo.toml b/ffi/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "snips-nlu-ffi"
-version = "0.63.1"
+version = "0.64.0"
 edition = "2018"
 authors = [
     "Adrien Ball <adrien.ball@snips.ai>",
@@ -12,7 +12,7 @@ authors = [
 [dependencies]
 ffi-utils = { git = "https://github.com/snipsco/snips-utils-rs", rev = "4292ad9" }
 snips-nlu-lib = { path = ".." }
-snips-nlu-ontology-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.63.1" }
+snips-nlu-ontology-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.64.4" }
 failure = "0.1"
 lazy_static = "1.0"
 libc = "0.2"
diff --git a/platforms/c/libsnips_nlu.h b/platforms/c/libsnips_nlu.h
@@ -5,7 +5,7 @@
 #include <stdlib.h>
 #include <stdbool.h>
 
-#define SNIPS_NLU_VERSION "0.63.1"
+#define SNIPS_NLU_VERSION "0.64.0"
 
 /*
  * Enum representing the grain of a resolved date related value
diff --git a/platforms/kotlin/build.gradle b/platforms/kotlin/build.gradle
@@ -11,7 +11,7 @@ buildscript {
 
 apply plugin: 'kotlin'
 
-version = "0.63.1"
+version = "0.64.0"
 group = "ai.snips"
 
 repositories {
@@ -32,7 +32,7 @@ configurations {
 dependencies {
     compile "org.jetbrains.kotlin:kotlin-stdlib-jre7:$kotlin_version"
     compile 'net.java.dev.jna:jna:4.5.0'
-    compile "ai.snips:snips-nlu-ontology:0.63.1"
+    compile "ai.snips:snips-nlu-ontology:0.64.4"
     testCompile 'junit:junit:4.12'
     testCompile 'com.google.truth:truth:0.36'
 }
diff --git a/platforms/python/ffi/Cargo.toml b/platforms/python/ffi/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "snips-nlu-python-ffi"
-version = "0.63.1"
+version = "0.64.0"
 authors = ["Adrien Ball <adrien.ball@snips.ai>"]
 edition = "2018"
 
@@ -11,4 +11,4 @@ crate-type = ["cdylib"]
 [dependencies]
 libc = "0.2"
 ffi-utils = { git = "https://github.com/snipsco/snips-utils-rs", rev = "4292ad9" }
-snips-nlu-ffi = { git = "https://github.com/snipsco/snips-nlu-rs", tag = "0.63.1" }
+snips-nlu-ffi = { git = "https://github.com/snipsco/snips-nlu-rs", tag = "0.64.0" }
diff --git a/platforms/python/snips_nlu_rust/__version__ b/platforms/python/snips_nlu_rust/__version__
@@ -1 +1 @@
-0.63.1
+0.64.0
diff --git a/platforms/swift/SnipsNlu/Dependencies/build.sh b/platforms/swift/SnipsNlu/Dependencies/build.sh
@@ -4,7 +4,7 @@
 
 set -e
 
-VERSION="0.63.1"
+VERSION="0.64.0"
 SYSTEM=$(echo $1 | tr '[:upper:]' '[:lower:]')
 LIBRARY_NAME=libsnips_nlu_ffi
 LIBRARY_NAME_A=${LIBRARY_NAME}.a
diff --git a/src/intent_classifier/featurizer.rs b/src/intent_classifier/featurizer.rs
@@ -188,7 +188,8 @@ impl TfidfVectorizer {
         let builtin_entities_features: Vec<String> = builtin_entities
             .iter()
             .map(|ent| get_builtin_entity_feature_name(ent.entity_kind, self.language))
-            .sorted();
+            .sorted()
+            .collect();
 
         // Extract custom entities on the normalized utterance
         let custom_entities = self
@@ -399,6 +400,7 @@ fn get_word_clusters(query_tokens: &[String], word_clusterer: Arc<WordClusterer>
         .into_iter()
         .filter_map(|ngram| word_clusterer.get_cluster(&ngram.0.to_lowercase()))
         .sorted()
+        .collect()
 }
 
 fn normalize_stem(tokens: &[String], opt_stemmer: Option<Arc<Stemmer>>) -> Vec<String> {
diff --git a/src/intent_classifier/log_reg_intent_classifier.rs b/src/intent_classifier/log_reg_intent_classifier.rs
@@ -75,7 +75,7 @@ impl IntentClassifier for LogRegIntentClassifier {
         Ok(if intents_results.is_empty() {
             IntentClassifierResult {
                 intent_name: None,
-                probability: 1.0,
+                confidence_score: 1.0,
             }
         } else {
             intents_results.into_iter().next().unwrap()
@@ -96,7 +96,7 @@ impl LogRegIntentClassifier {
         if self.intent_list.len() <= 1 {
             return Ok(vec![IntentClassifierResult {
                 intent_name: self.intent_list.first().cloned().unwrap_or(None),
-                probability: 1.0,
+                confidence_score: 1.0,
             }]);
         }
 
@@ -106,9 +106,10 @@ impl LogRegIntentClassifier {
                 .iter()
                 .map(|intent_name| IntentClassifierResult {
                     intent_name: intent_name.clone(),
-                    probability: if intent_name.is_none() { 1.0 } else { 0.0 },
+                    confidence_score: if intent_name.is_none() { 1.0 } else { 0.0 },
                 })
-                .sorted_by(|a, b| b.probability.partial_cmp(&a.probability).unwrap()));
+                .sorted_by(|a, b| b.confidence_score.partial_cmp(&a.confidence_score).unwrap())
+                .collect());
         }
 
         let opt_intents_set: Option<HashSet<&str>> =
@@ -120,15 +121,15 @@ impl LogRegIntentClassifier {
         let features = featurizer.transform(input)?;
         let filtered_out_indexes =
             get_filtered_out_intents_indexes(&self.intent_list, opt_intents_set.as_ref());
-        let probabilities = logreg.run(&features.view(), filtered_out_indexes)?;
+        let scores = logreg.run(&features.view(), filtered_out_indexes)?;
 
         Ok(self
             .intent_list
             .iter()
-            .zip(probabilities.into_iter())
-            .map(|(intent_name, probability)| IntentClassifierResult {
+            .zip(scores.into_iter())
+            .map(|(intent_name, score)| IntentClassifierResult {
                 intent_name: intent_name.clone(),
-                probability: *probability,
+                confidence_score: *score,
             })
             .filter(|res| {
                 if let Some(intent) = res.intent_name.as_ref() {
@@ -140,7 +141,8 @@ impl LogRegIntentClassifier {
                     true
                 }
             })
-            .sorted_by(|a, b| b.probability.partial_cmp(&a.probability).unwrap()))
+            .sorted_by(|a, b| b.confidence_score.partial_cmp(&a.confidence_score).unwrap())
+            .collect())
     }
 }
 
@@ -372,7 +374,7 @@ mod tests {
         let actual_result = classification_result.unwrap();
         let expected_result = IntentClassifierResult {
             intent_name: Some("MakeTea".to_string()),
-            probability: 0.9088109819597295,
+            confidence_score: 0.9088109819597295,
         };
 
         // Then
diff --git a/src/intent_parser/deterministic_intent_parser.rs b/src/intent_parser/deterministic_intent_parser.rs
@@ -95,7 +95,7 @@ impl IntentParser for DeterministicIntentParser {
             .unwrap_or_else(|| InternalParsingResult {
                 intent: IntentClassifierResult {
                     intent_name: None,
-                    probability: 1.0,
+                    confidence_score: 1.0,
                 },
                 slots: vec![],
             }))
@@ -116,15 +116,15 @@ impl IntentParser for DeterministicIntentParser {
             if !matched_intents.contains(intent) {
                 top_intents.push(IntentClassifierResult {
                     intent_name: Some(intent.to_string()),
-                    probability: 0.0,
+                    confidence_score: 0.0,
                 });
             }
         }
         // The None intent is not included in the regex patterns and thus it is
         // never matched by the deterministic parser
         top_intents.push(IntentClassifierResult {
             intent_name: None,
-            probability: 0.0,
+            confidence_score: 0.0,
         });
         Ok(top_intents)
     }
@@ -475,7 +475,7 @@ mod tests {
         // Then
         let expected_intent = IntentClassifierResult {
             intent_name: Some("dummy_intent_1".to_string()),
-            probability: 1.0,
+            confidence_score: 1.0,
         };
 
         assert_eq!(intent, expected_intent);
@@ -518,7 +518,7 @@ mod tests {
         // Then
         let expected_intent = IntentClassifierResult {
             intent_name: None,
-            probability: 1.0,
+            confidence_score: 1.0,
         };
 
         assert_eq!(intent, expected_intent);
@@ -558,13 +558,13 @@ mod tests {
         // Then
         let first_intent = IntentClassifierResult {
             intent_name: Some("dummy_intent_1".to_string()),
-            probability: 1.0,
+            confidence_score: 1.0,
         };
         assert_eq!(4, intents.len());
         assert_eq!(&first_intent, &intents[0]);
-        assert_eq!(0.0, intents[1].probability);
-        assert_eq!(0.0, intents[2].probability);
-        assert_eq!(0.0, intents[3].probability);
+        assert_eq!(0.0, intents[1].confidence_score);
+        assert_eq!(0.0, intents[2].confidence_score);
+        assert_eq!(0.0, intents[3].confidence_score);
     }
 
     #[test]
@@ -597,7 +597,7 @@ mod tests {
         // Then
         let expected_intent = IntentClassifierResult {
             intent_name: Some("dummy_intent_3".to_string()),
-            probability: 1.0,
+            confidence_score: 1.0,
         };
 
         assert_eq!(intent, expected_intent);
@@ -641,7 +641,7 @@ mod tests {
         // Then
         let expected_intent = IntentClassifierResult {
             intent_name: Some("dummy_intent_1".to_string()),
-            probability: 1.0,
+            confidence_score: 1.0,
         };
 
         assert_eq!(intent, expected_intent);
diff --git a/src/intent_parser/mod.rs b/src/intent_parser/mod.rs
@@ -28,7 +28,7 @@ pub fn internal_parsing_result(
     InternalParsingResult {
         intent: IntentClassifierResult {
             intent_name,
-            probability: intent_proba,
+            confidence_score: intent_proba,
         },
         slots,
     }
diff --git a/src/language.rs b/src/language.rs
@@ -15,6 +15,8 @@ impl FromLanguage for NluUtilsLanguage {
             Language::IT => NluUtilsLanguage::IT,
             Language::JA => NluUtilsLanguage::JA,
             Language::KO => NluUtilsLanguage::KO,
+            Language::PT_PT => NluUtilsLanguage::PT_PT,
+            Language::PT_BR => NluUtilsLanguage::PT_BR,
         }
     }
 }
diff --git a/src/nlu_engine.rs b/src/nlu_engine.rs
@@ -152,7 +152,7 @@ impl SnipsNluEngine {
         let intents_whitelist = intents_whitelist_owned
             .as_ref()
             .map(|whitelist| whitelist.as_ref());
-        let mut none_proba: f32 = 0.0;
+        let mut none_score: f32 = 0.0;
         for parser in &self.intent_parsers {
             let internal_parsing_result = parser.parse(input, intents_whitelist)?;
             if internal_parsing_result.intent.intent_name.is_some() {
@@ -166,17 +166,17 @@ impl SnipsNluEngine {
                     slots: resolved_slots,
                 });
             } else {
-                none_proba = internal_parsing_result.intent.probability;
+                none_score = internal_parsing_result.intent.confidence_score;
             }
         }
 
-        // If all parsers failed to extract an intent, we use the probability returned by the last
+        // If all parsers failed to extract an intent, we use the confidence score returned by the last
         // parser
         Ok(IntentParserResult {
             input: input.to_string(),
             intent: IntentClassifierResult {
                 intent_name: None,
-                probability: none_proba,
+                confidence_score: none_score,
             },
             slots: vec![],
         })
@@ -194,22 +194,23 @@ impl SnipsNluEngine {
                 continue;
             }
             for res in parser_results.into_iter() {
-                let existing_proba = results
+                let existing_score = results
                     .get(&res.intent_name)
-                    .map(|r| r.probability)
+                    .map(|r| r.confidence_score)
                     .unwrap_or(0.0);
-                if res.probability > existing_proba {
+                if res.confidence_score > existing_score {
                     results
                         .entry(res.intent_name.clone())
-                        .and_modify(|e| e.probability = res.probability)
+                        .and_modify(|e| e.confidence_score = res.confidence_score)
                         .or_insert_with(|| res);
                 }
             }
         }
         Ok(results
             .into_iter()
             .map(|(_, res)| res)
-            .sorted_by(|a, b| b.probability.partial_cmp(&a.probability).unwrap()))
+            .sorted_by(|a, b| b.confidence_score.partial_cmp(&a.confidence_score).unwrap())
+            .collect())
     }
 
     pub fn get_slots(&self, input: &str, intent: &str) -> Result<Vec<Slot>> {
diff --git a/src/slot_filler/feature_processor.rs b/src/slot_filler/feature_processor.rs
