fix build with newer cxx 17 (#336)

Co-authored-by: thucpham <[email protected]>

Author: minhthuc2502

CMakeLists.txt

@@ -10,7 +10,7 @@ project(OpenNMTTokenizer)
 option(BUILD_TESTS "Compile unit tests" OFF)
 option(BUILD_SHARED_LIBS "Build shared libraries" ON)
 
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 17)
 if(CMAKE_VERSION VERSION_LESS "3.7.0")
   set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 endif()

src/SentencePiece.cc

@@ -54,7 +54,12 @@ namespace onmt
     if (options && (options->joiner_annotate || options->spacer_new))
       throw std::invalid_argument("SentencePiece vocabulary restriction requires the tokenization "
                                   "to use \"spacer_annotate\" (same as spm_encode)");
-    auto status = _processor->SetVocabulary(vocabulary);
+    std::vector<std::string_view> vocabulary_views;
+    vocabulary_views.reserve(vocabulary.size());
+    for (const auto& s : vocabulary) {
+        vocabulary_views.emplace_back(s);
+    }
+    auto status = _processor->SetVocabulary(vocabulary_views);
     if (!status.ok())
       throw std::invalid_argument(status.ToString());
   }