is bug but faster

Author: RuABraun

libs/stringvector.cc

@@ -55,6 +55,13 @@ const std::string_view StringVector::next() {
     return (*this)[current_index_++];
 }
 
+std::string StringVector::Str() const {
+    std::string repr = "";
+    for (int i = 0; i < Size(); i++) {
+        repr += std::string{(*this)[i]} + " ";
+    }
+    return repr;
+}
 
 StringVector::~StringVector() {}
 
@@ -66,5 +73,6 @@ void init_stringvector(py::module &m) {
         .def("__len__", &StringVector::Size)
         .def("__getitem__", &StringVector::operator[])
         .def("__iter__", &StringVector::iter)
-        .def("__next__", &StringVector::next);
+        .def("__next__", &StringVector::next)
+        .def("__str__", &StringVector::Str);
 }

libs/stringvector.h

@@ -20,6 +20,7 @@ class StringVector {
     const std::string_view operator[](const int i) const;
     StringVector iter();
     const std::string_view next();
+    std::string Str() const;
 
     std::string data_;
     std::vector<int> wordend_index_;

libs/texterrors_align.cc

@@ -27,6 +27,56 @@ struct Pair {
     int16_t j;
 };
 
+
+int calc_edit_distance_fast(int32* cost_mat, const char* a, const char* b,
+                     const int32 M, const int32 N) {
+  int row_length = N+1;
+  // std::cout << "STARTING M="<< M<< " N="<<N<<std::endl;
+  for (int32 i = 0; i <= M; ++i) {
+    for (int32 j = 0; j <= N; ++j) {
+
+      if (i == 0 && j == 0) {
+        cost_mat[0] = 0;
+        continue;
+      }
+      if (i == 0) {
+        cost_mat[j] = cost_mat[j - 1] + 1;
+        continue;
+      }
+      if (j == 0) {
+        cost_mat[row_length] = cost_mat[0] + 1;
+        continue;
+      }
+      int32 transition_cost = a[i-1] == b[j-1] ? 0 : 1;
+
+      int32 upc = cost_mat[j] + 1;
+      int32 leftc = cost_mat[row_length + j - 1] + 1;
+      int32 diagc = cost_mat[j - 1] + transition_cost;
+      int32 cost = std::min(upc, std::min(leftc, diagc) );
+
+      cost_mat[row_length + j] = cost;
+      cost_mat[j - 1] = cost_mat[row_length + j - 1];  // copying result up after use
+    }
+    if (i > 0) {
+      cost_mat[N] = cost_mat[row_length + N];
+    }
+
+    // std::cout << "row "<<i;
+    // for (int32 j = 0; j <= N; ++j) {
+    //   std::cout << " "<<cost_mat[j];
+    // }
+    // std::cout << std::endl;
+  }
+  // std::cout << "last row";
+  // for (int32 j = 0; j <= N; ++j) {
+  //   std::cout <<" "<<cost_mat[row_length + j];
+  // }
+  // std::cout << std::endl;
+
+  return cost_mat[row_length - 1];
+}
+
+
 template <class T>
 void create_lev_cost_mat(int32* cost_mat, const T* a, const T* b,
                      const int32 M, const int32 N) {
@@ -109,6 +159,11 @@ int lev_distance_str(std::string a, std::string b) {
   return levdistance(a.data(), b.data(), a.size(), b.size());
 }
 
+int calc_edit_distance_fast_str(std::string a, std::string b) {
+  std::vector<int> buffer(a.size() + b.size() + 2);
+  return calc_edit_distance_fast(buffer.data(), a.data(), b.data(), a.size(), b.size());
+}
+
 enum direction{diag, move_left, up};
 
 std::vector<std::tuple<int32, int32> > get_best_path(py::array_t<double> array, 
@@ -117,6 +172,10 @@ std::vector<std::tuple<int32, int32> > get_best_path(py::array_t<double> array,
   auto buf = array.request();
   double* cost_mat = (double*) buf.ptr;
   int32_t numr = array.shape()[0], numc = array.shape()[1];
+  std::vector<int32> char_dist_buffer;
+  if (use_chardiff) {
+    char_dist_buffer.resize(100);
+  }
 
   std::vector<std::tuple<int, int> > bestpath;
   int i = numr - 1, j = numc - 1;
@@ -138,8 +197,13 @@ std::vector<std::tuple<int32, int32> > get_best_path(py::array_t<double> array,
       double left_trans_cost = 1.0;
       double diag_trans_cost;
       if (use_chardiff) {
+        int alen = a.size();
+        int blen = b.size();
+        if (alen >= 50 || blen >= 50) {
+          throw std::runtime_error("Word is too long! Increase buffer");
+        }
         diag_trans_cost =
-          levdistance(a.data(), b.data(), a.size(), b.size()) / static_cast<double>(std::max(a.size(), b.size())) * 1.5;
+          calc_edit_distance_fast(char_dist_buffer.data(), a.data(), b.data(), a.size(), b.size()) / static_cast<double>(std::max(a.size(), b.size())) * 1.5;
       } else {
         diag_trans_cost = a == b ? 0. : 1.;
       }
@@ -266,6 +330,12 @@ int calc_sum_cost(py::array_t<double> array, const StringVector& words_a,
   if (M1 - 1 != words_a.Size() || N1 - 1 != words_b.Size()) throw std::runtime_error("Sizes do not match!");
   auto buf = array.request();
   double* ptr = (double*) buf.ptr;
+
+  std::vector<int32> char_dist_buffer;
+  if (use_chardist) {
+    char_dist_buffer.resize(100);
+  }
+
   ptr[0] = 0;
   for (int32 i = 1; i < M1; i++) ptr[i*N1] = ptr[(i-1)*N1] + 1;
   for (int32 j = 1; j < N1; j++) ptr[j] = ptr[j-1] + 1;
@@ -275,8 +345,13 @@ int calc_sum_cost(py::array_t<double> array, const StringVector& words_a,
       if (use_chardist) {
         const std::string_view a = words_a[i-1];
         const std::string_view b = words_b[j-1];
-        transition_cost = levdistance(a.data(), b.data(), 
-          a.size(), b.size()) / static_cast<double>(std::max(a.size(), b.size())) * 1.5;
+        int alen = a.size();
+        int blen = b.size();
+        if (alen >= 50 || blen >= 50) {
+          throw std::runtime_error("Word is too long! Increase buffer");
+        }
+        transition_cost = calc_edit_distance_fast(char_dist_buffer.data(), a.data(), b.data(), a.size(), b.size()) 
+          / static_cast<double>(std::max(a.size(), b.size())) * 1.5;
       } else {
         transition_cost = words_a[i-1] == words_b[j-1] ? 0. : 1.;
       }
@@ -369,5 +444,6 @@ PYBIND11_MODULE(texterrors_align,m) {
   m.def("lev_distance", lev_distance<int>);
   m.def("lev_distance", lev_distance<char>);
   m.def("lev_distance_str", &lev_distance_str);
+  m.def("calc_edit_distance_fast_str", &calc_edit_distance_fast_str);
   init_stringvector(m);
 }

tests/test_functions.py

@@ -3,16 +3,17 @@
 import os
 import io
 import time
-import logging
+import sys
+from loguru import logger
 
 import Levenshtein as levd
 from texterrors import texterrors
 from texterrors.texterrors import StringVector
 from dataclasses import dataclass
 import difflib
 
-
-logger = logging.getLogger(__name__)
+logger.remove()
+logger.add(sys.stderr, level="INFO")
 
 def show_diff(text1, text2):
     # Split the strings into lines to compare them line by line
@@ -38,6 +39,14 @@ def test_levd():
         assert d1 == d2, f'{a} {b} {d1} {d2}'
 
 
+# def test_calc_edit_distance_fast():
+#     pairs = ['a', '', '', 'a', 'MOZILLA', 'MUSIAL', 'ARE', 'MOZILLA', 'TURNIPS', 'TENTH', 'POSTERS', 'POSTURE']
+#     for a, b in zip(pairs[:-1:2], pairs[1::2]):
+#         d1 = texterrors.calc_edit_distance_fast(a, b)
+#         d2 = levd.distance(a, b)
+#         assert d1 == d2, f'{a} {b} fasteditdist={d1} ref={d2}'
+
+
 def calc_wer(ref, b):
     cnt = 0
     err = 0
@@ -266,7 +275,7 @@ def test_process_output_colored():
     hyps = create_inp(hyplines)
 
     buffer = io.StringIO()
-    texterrors.process_output(refs, hyps, buffer, ref_file='A', hyp_file='B', nocolor=False)
+    texterrors.process_output(refs, hyps, buffer, ref_file='A', hyp_file='B', nocolor=False, terminal_width=80)
     output = buffer.getvalue()
     ref = """\"A\" is treated as reference (white and green), \"B\" as hypothesis (white and red).
 Per utt details:
@@ -297,8 +306,8 @@ def test_process_output_colored():
 es>sie\t1\t2
 ja>auch\t1\t1
 """
-    #print(ref, file=open('ref', 'w'))
-    #print(output, file=open('output', 'w'))
+    print(ref, file=open('ref', 'w'))
+    print(output, file=open('output', 'w'))
     assert ref == output
 
 
@@ -340,16 +349,17 @@ def test_cli_basic():
 def test_speed():
     ref = create_inp(open('tests/reftext').read().splitlines())
     hyp = create_inp(open('tests/hyptext').read().splitlines())
-    import cProfile
-    pr = cProfile.Profile()
+    # import cProfile
+    # pr = cProfile.Profile()
 
-    pr.enable()
+    # pr.enable()
     buffer = io.StringIO()
     start_time = time.perf_counter()
-    texterrors.process_output(ref, hyp, fh=buffer, ref_file='ref', hyp_file='hyp', skip_detailed=True)
+    texterrors.process_output(ref, hyp, fh=buffer, ref_file='ref', hyp_file='hyp', 
+                              skip_detailed=True, use_chardiff=True, debug=False)
     process_time = time.perf_counter() - start_time
-    pr.disable()
-    pr.dump_stats('speed.prof')
+    # pr.disable()
+    # pr.dump_stats('speed.prof')
 
     logger.info(f'Processing time for speed test is {process_time}')
     assert process_time < 2.

texterrors/__init__.py

@@ -1,2 +1,2 @@
 from .texterrors import align_texts, process_lines, lev_distance, get_oov_cer, align_texts_ctm, seq_distance, \
-    process_output, process_multiple_outputs
+    process_output, process_multiple_outputs, calc_edit_distance_fast

texterrors/texterrors.py

@@ -40,6 +40,10 @@ def lev_distance(a, b):
         return texterrors_align.lev_distance(a, b)
 
 
+def calc_edit_distance_fast(a, b):
+    return texterrors_align.calc_edit_distance_fast_str(a, b)
+
+
 def seq_distance(a, b):
     """ This function is for when a and b have strings as elements (variable length). """
     assert isinstance(a, StringVector) and isinstance(b, StringVector), 'Input types should be of type StringVector!'
@@ -53,6 +57,10 @@ def seq_distance(a, b):
 def _align_texts(words_a, words_b, use_chardiff, debug, insert_tok): 
     summed_cost = np.zeros((len(words_a) + 1, len(words_b) + 1), dtype=np.float64, 
         order="C")
+    
+    if debug:
+        print(words_a)
+        print(words_b)
     cost = texterrors_align.calc_sum_cost(summed_cost, words_a, words_b, use_chardiff)
 
     if debug:
@@ -656,10 +664,11 @@ def process_multiple_outputs(ref_utts, hypa_utts, hypb_utts, fh, num_top_errors,
 def process_output(ref_utts, hyp_utts, fh, ref_file, hyp_file, cer=False, num_top_errors=10, oov_set=None, debug=False,
                   use_chardiff=True, isctm=False, skip_detailed=False,
                   keywords=None, utt_group_map=None, oracle_wer=False,
-                  freq_sort=False, nocolor=False, insert_tok='<eps>'):
+                  freq_sort=False, nocolor=False, insert_tok='<eps>', terminal_width=None):
 
-    terminal_width, _ = shutil.get_terminal_size()
-    terminal_width = 120 if terminal_width >= 120 else terminal_width
+    if terminal_width is None:
+        terminal_width, _ = shutil.get_terminal_size()
+        terminal_width = 120 if terminal_width >= 120 else terminal_width
 
     if oov_set is None:
         oov_set = set()