Support RunAnalyzer() (#367)

Signed-off-by: yhmo <[email protected]>

Author: yhmo

DEVELOPMENT.md

@@ -157,6 +157,7 @@ Once the `make test` is done, you will see some executable examples under the pa
 - `./cmake_build/examples/sdk_nullalbe_field`: example to show the usage of nullable field.
 - `./cmake_build/examples/sdk_partition_key`: example to show the usage of partition key.
 - `./cmake_build/examples/sdk_rbac`: example to show the usage of RBAC.
+- `./cmake_build/examples/sdk_run_analyzer`: example to show the usage of run_analyzer().
 - `./cmake_build/examples/sdk_text_match`: example to show the usage of TEXT_MATCH.
 - `./cmake_build/examples/sdk_vector_binary`: example to show the usage of BinaryVector field.
 - `./cmake_build/examples/sdk_vector_fp16`: example to show the usage of Float16Vector/BFloat16Vector field.

examples/example.md

@@ -20,6 +20,7 @@ Once the `make test` is done, you will see some executable examples under the pa
 - `./cmake_build/examples/sdk_nullalbe_field`: example to show the usage of nullable field.
 - `./cmake_build/examples/sdk_partition_key`: example to show the usage of partition key.
 - `./cmake_build/examples/sdk_rbac`: example to show the usage of RBAC.
+- `./cmake_build/examples/sdk_run_analyzer`: example to show the usage of run_analyzer().
 - `./cmake_build/examples/sdk_text_match`: example to show the usage of TEXT_MATCH.
 - `./cmake_build/examples/sdk_vector_binary`: example to show the usage of BinaryVector field.
 - `./cmake_build/examples/sdk_vector_fp16`: example to show the usage of Float16Vector/BFloat16Vector field.

examples/src/run_analyzer.cpp

@@ -0,0 +1,93 @@
+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <iostream>
+#include <string>
+#include <thread>
+
+#include "ExampleUtils.h"
+#include "milvus/MilvusClient.h"
+
+namespace {
+void
+printAnalyzerResults(const milvus::AnalyzerResults& results) {
+    for (const auto& result : results.Results()) {
+        std::cout << "\t------------------------------" << std::endl;
+        for (const auto& token : result.Tokens()) {
+            std::cout << "\t{token: " << token.token_ << ", start: " << token.start_offset_
+                      << ", end: " << token.end_offset_ << ", position: " << token.position_
+                      << ", position_len: " << token.position_length_ << ", hash: " << token.hash_ << "}" << std::endl;
+        }
+        std::cout << "\t------------------------------" << std::endl;
+    }
+}
+
+void
+runAnalyzer(milvus::MilvusClientPtr& client, const nlohmann::json& analyzer_params) {
+    std::cout << "Run analyzer params: " << analyzer_params.dump() << std::endl;
+
+    const std::vector<std::string> text_content = {
+        "Milvus is an open-source vector database",
+        "AI applications help people better life",
+        "Will the electric car replace gas-powered car?",
+        "LangChain is a composable framework to build with LLMs. Milvus is integrated into LangChain.",
+        "RAG is the process of optimizing the output of a large language model",
+        "Newton is one of the greatest scientist of human history",
+        "Metric type L2 is Euclidean distance",
+        "Embeddings represent real-world objects, like words, images, or videos, in a form that computers can process.",
+        "The moon is 384,400 km distance away from earth",
+        "Milvus supports L2 distance and IP similarity for float vector.",
+    };
+
+    milvus::RunAnalyzerArguments args;
+    args.SetTexts(text_content);
+    args.SetAnalyzerParams(analyzer_params);
+    args.WithDetail(true);
+    args.WithHash(true);
+
+    milvus::AnalyzerResults results;
+    auto status = client->RunAnalyzer(args, results);
+    util::CheckStatus("run analyzer", status);
+    printAnalyzerResults(results);
+}
+
+}  // namespace
+
+int
+main(int argc, char* argv[]) {
+    printf("Example start...\n");
+
+    auto client = milvus::MilvusClient::Create();
+
+    milvus::ConnectParam connect_param{"localhost", 19530, "root", "Milvus"};
+    auto status = client->Connect(connect_param);
+    util::CheckStatus("connect milvus server", status);
+
+    nlohmann::json params_1 = {
+        {"tokenizer", "standard"},
+        {"filter", {{{"type", "stop"}, {"stop_words", {"of"}}}}},
+    };
+    runAnalyzer(client, params_1);
+
+    nlohmann::json params_2 = {
+        {"tokenizer", "standard"},
+        {"filter", {{{"type", "stop"}, {"stop_words", {"is", "of", "for"}}}}},
+    };
+    runAnalyzer(client, params_2);
+
+    client->Disconnect();
+    return 0;
+}

src/impl/MilvusClientImpl.cpp

@@ -1371,6 +1371,54 @@ MilvusClientImpl::QueryIterator(QueryIteratorArguments& arguments, QueryIterator
     return Status::OK();
 }
 
+Status
+MilvusClientImpl::RunAnalyzer(const RunAnalyzerArguments& arguments, AnalyzerResults& results) {
+    auto pre = [&arguments](proto::milvus::RunAnalyzerRequest& rpc_request) {
+        rpc_request.set_collection_name(arguments.CollectionName());
+        rpc_request.set_db_name(arguments.DatabaseName());
+        rpc_request.set_field_name(arguments.FieldName());
+        rpc_request.set_analyzer_params(arguments.AnalyzerParams().dump());
+        auto placeholder = rpc_request.mutable_placeholder();
+        for (std::string text : arguments.Texts()) {
+            placeholder->Add(std::move(text));
+        }
+        for (const auto& name : arguments.AnalyzerNames()) {
+            rpc_request.add_analyzer_names(name);
+        }
+        rpc_request.set_with_detail(arguments.IsWithDetail());
+        rpc_request.set_with_hash(arguments.IsWithHash());
+        return Status::OK();
+    };
+
+    auto post = [&results](const proto::milvus::RunAnalyzerResponse& response) {
+        std::vector<AnalyzerResult> results_list;
+        const auto& rpc_results = response.results();
+        for (auto i = 0; i < response.results_size(); i++) {
+            std::vector<AnalyzerToken> tokens;
+            const auto& rpc_tokens = rpc_results[i].tokens();
+            for (auto k = 0; k < rpc_results[i].tokens_size(); k++) {
+                const auto& rpc_token = rpc_tokens[k];
+                AnalyzerToken token;
+                token.token_ = rpc_token.token();
+                token.start_offset_ = rpc_token.start_offset();
+                token.end_offset_ = rpc_token.end_offset();
+                token.position_ = rpc_token.position();
+                token.position_length_ = rpc_token.position_length();
+                token.hash_ = rpc_token.hash();
+                tokens.emplace_back(std::move(token));
+            }
+
+            AnalyzerResult result{std::move(tokens)};
+            results_list.emplace_back(std::move(result));
+        }
+        results = AnalyzerResults(std::move(results_list));
+        return Status::OK();
+    };
+
+    return apiHandler<proto::milvus::RunAnalyzerRequest, proto::milvus::RunAnalyzerResponse>(
+        pre, &MilvusConnection::RunAnalyzer, post);
+}
+
 Status
 MilvusClientImpl::Flush(const std::vector<std::string>& collection_names, const ProgressMonitor& progress_monitor) {
     auto pre = [&collection_names](proto::milvus::FlushRequest& rpc_request) {

src/impl/MilvusClientImpl.h

@@ -233,6 +233,9 @@ class MilvusClientImpl : public MilvusClient {
     Status
     QueryIterator(QueryIteratorArguments& arguments, QueryIteratorPtr& iterator) final;
 
+    Status
+    RunAnalyzer(const RunAnalyzerArguments& arguments, AnalyzerResults& results) final;
+
     Status
     Flush(const std::vector<std::string>& collection_names, const ProgressMonitor& progress_monitor) final;
 

src/impl/MilvusConnection.cpp

@@ -447,6 +447,12 @@ MilvusConnection::Query(const proto::milvus::QueryRequest& request, proto::milvu
     return grpcCall("Query", &Stub::Query, request, response, options);
 }
 
+Status
+MilvusConnection::RunAnalyzer(const proto::milvus::RunAnalyzerRequest& request,
+                              proto::milvus::RunAnalyzerResponse& response, const GrpcContextOptions& options) {
+    return grpcCall("RunAnalyzer", &Stub::RunAnalyzer, request, response, options);
+}
+
 Status
 MilvusConnection::GetFlushState(const proto::milvus::GetFlushStateRequest& request,
                                 proto::milvus::GetFlushStateResponse& response, const GrpcContextOptions& options) {

src/impl/MilvusConnection.h

@@ -242,6 +242,10 @@ class MilvusConnection {
     Query(const proto::milvus::QueryRequest& request, proto::milvus::QueryResults& response,
           const GrpcContextOptions& options);
 
+    Status
+    RunAnalyzer(const proto::milvus::RunAnalyzerRequest& request, proto::milvus::RunAnalyzerResponse& response,
+                const GrpcContextOptions& options);
+
     Status
     GetFlushState(const proto::milvus::GetFlushStateRequest& request, proto::milvus::GetFlushStateResponse& response,
                   const GrpcContextOptions& options);

src/impl/types/AnalyzerResults.cpp

@@ -0,0 +1,51 @@
+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "milvus/types/AnalyzerResults.h"
+
+#include <memory>
+
+namespace milvus {
+
+AnalyzerResult::AnalyzerResult(std::vector<AnalyzerToken>&& tokens) : tokens_(std::move(tokens)) {
+}
+
+const std::vector<AnalyzerToken>&
+AnalyzerResult::Tokens() const {
+    return tokens_;
+}
+
+Status
+AnalyzerResult::AddToken(AnalyzerToken&& token) {
+    tokens_.emplace_back(token);
+    return Status::OK();
+}
+
+AnalyzerResults::AnalyzerResults(std::vector<AnalyzerResult>&& results) : results_(std::move(results)) {
+}
+
+const std::vector<AnalyzerResult>&
+AnalyzerResults::Results() const {
+    return results_;
+}
+
+Status
+AnalyzerResults::AddResult(AnalyzerResult&& result) {
+    results_.emplace_back(result);
+    return Status::OK();
+}
+
+}  // namespace milvus

src/impl/types/RunAnalyzerArguments.cpp

@@ -0,0 +1,144 @@
+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "milvus/types/RunAnalyzerArguments.h"
+
+namespace milvus {
+const std::string&
+RunAnalyzerArguments::DatabaseName() const {
+    return db_name_;
+}
+
+Status
+RunAnalyzerArguments::SetDatabaseName(const std::string& db_name) {
+    db_name_ = db_name;
+    return Status::OK();
+}
+
+RunAnalyzerArguments&
+RunAnalyzerArguments::WithDatabaseName(const std::string& db_name) {
+    db_name_ = db_name;
+    return *this;
+}
+
+const std::string&
+RunAnalyzerArguments::CollectionName() const {
+    return collection_name_;
+}
+
+Status
+RunAnalyzerArguments::SetCollectionName(std::string collection_name) {
+    collection_name_ = collection_name;
+    return Status::OK();
+}
+
+RunAnalyzerArguments&
+RunAnalyzerArguments::WithCollectionName(std::string collection_name) {
+    collection_name_ = collection_name;
+    return *this;
+}
+
+const std::string&
+RunAnalyzerArguments::FieldName() const {
+    return field_name_;
+}
+
+Status
+RunAnalyzerArguments::SetFieldName(std::string field_name) {
+    field_name_ = field_name;
+    return Status::OK();
+}
+
+RunAnalyzerArguments&
+RunAnalyzerArguments::WithFieldName(std::string field_name) {
+    field_name_ = field_name;
+    return *this;
+}
+
+const std::vector<std::string>&
+RunAnalyzerArguments::Texts() const {
+    return texts_;
+}
+
+Status
+RunAnalyzerArguments::SetTexts(const std::vector<std::string>& texts) {
+    texts_ = texts;
+    return Status::OK();
+}
+
+RunAnalyzerArguments&
+RunAnalyzerArguments::AddText(std::string text) {
+    texts_.emplace_back(text);
+    return *this;
+}
+
+const std::vector<std::string>&
+RunAnalyzerArguments::AnalyzerNames() const {
+    return analyzer_names_;
+}
+
+Status
+RunAnalyzerArguments::SetAnalyzerNames(const std::vector<std::string>& names) {
+    analyzer_names_ = names;
+    return Status::OK();
+}
+
+RunAnalyzerArguments&
+RunAnalyzerArguments::AddAnalyzerName(std::string name) {
+    analyzer_names_.emplace_back(name);
+    return *this;
+}
+
+const nlohmann::json&
+RunAnalyzerArguments::AnalyzerParams() const {
+    return analyzer_params_;
+}
+
+Status
+RunAnalyzerArguments::SetAnalyzerParams(const nlohmann::json& params) {
+    analyzer_params_ = params;
+    return Status::OK();
+}
+
+RunAnalyzerArguments&
+RunAnalyzerArguments::WithAnalyzerParams(const nlohmann::json& params) {
+    analyzer_params_ = params;
+    return *this;
+}
+
+bool
+RunAnalyzerArguments::IsWithDetail() const {
+    return with_detail_;
+}
+
+RunAnalyzerArguments&
+RunAnalyzerArguments::WithDetail(bool with_detail) {
+    with_detail_ = with_detail;
+    return *this;
+}
+
+RunAnalyzerArguments&
+RunAnalyzerArguments::WithHash(bool with_hash) {
+    with_hash_ = with_hash;
+    return *this;
+}
+
+bool
+RunAnalyzerArguments::IsWithHash() const {
+    return with_hash_;
+}
+
+}  // namespace milvus