diff --git a/python/_kenlm.pxd b/python/_kenlm.pxd index 57b056faf..8f861a16b 100644 --- a/python/_kenlm.pxd +++ b/python/_kenlm.pxd @@ -1,3 +1,8 @@ +from libcpp.string cimport string +from libcpp.vector cimport vector +from libc.stdint cimport uint64_t +from libcpp cimport bool + cdef extern from "lm/word_index.hh" namespace "lm": ctypedef unsigned WordIndex @@ -39,7 +44,7 @@ cdef extern from "util/mmap.hh" namespace "util": cdef extern from "lm/config.hh" namespace "lm::ngram": cdef cppclass Config: - Config() + Config() except + float probing_multiplier LoadMethod load_method @@ -48,3 +53,83 @@ cdef extern from "lm/model.hh" namespace "lm::ngram": #default constructor cdef Model *LoadVirtual(char *) except + +cdef extern from "util/file.hh" namespace "util": + cdef cppclass scoped_fd: + scoped_fd() except + + scoped_fd(int) except + + void reset(int) except + + int release() except + + + int OpenReadOrThrow(char*) except + + int CreateOrThrow(char*) except + + string DefaultTempDirectory() except + + void NormalizeTempPrefix(string) except + + +cdef extern from "util/usage.hh" namespace "util": + uint64_t GuessPhysicalMemory() except + + uint64_t ParseSize(string) except + + +cdef extern from "util/stream/config.hh" namespace "util::stream": + cdef struct SortConfig: + string temp_prefix + size_t buffer_size + size_t total_memory + + cdef struct ChainConfig: + size_t entry_size + size_t block_count + size_t total_memory + +cdef extern from "lm/builder/initial_probabilities.hh" namespace "lm::builder": + cdef struct InitialProbabilitiesConfig: + ChainConfig adder_in + ChainConfig adder_out + bool interpolate_unigrams + +cdef extern from "lm/lm_exception.hh" namespace "lm": + ctypedef enum WarningAction: + THROW_UP + COMPLAIN + SILENT + +cdef extern from "lm/builder/discount.hh" namespace "lm::builder": + cdef struct Discount: + float amount[4] + +cdef extern from "lm/builder/adjust_counts.hh" namespace "lm::builder": + cdef struct DiscountConfig: + vector[Discount] overwrite + Discount fallback + WarningAction bad_action + +cdef extern from "lm/builder/output.hh" namespace "lm::builder": + + cdef cppclass OutputHook: + pass + + cdef cppclass Output: + Output(char*, bool, bool) except + + void Add(OutputHook*) except + + + cdef cppclass PrintHook(OutputHook): + PrintHook(int, bool) except + + +cdef extern from "lm/builder/pipeline.hh" namespace "lm::builder": + struct PipelineConfig: + size_t order + SortConfig sort + InitialProbabilitiesConfig initial_probs + ChainConfig read_backoffs + WordIndex vocab_estimate + size_t minimum_block + size_t block_count + vector[uint64_t] prune_thresholds + bool prune_vocab + string prune_vocab_file + bool renumber_vocabulary + DiscountConfig discount + bool output_q + uint64_t vocab_size_for_unk + WarningAction disallowed_symbol_action + + void Pipeline(PipelineConfig, int, Output) except + diff --git a/python/kenlm.cpp b/python/kenlm.cpp index 0b4467d8d..3ec07e897 100644 --- a/python/kenlm.cpp +++ b/python/kenlm.cpp @@ -1,13 +1,14 @@ -/* Generated by Cython 0.25.2 */ +/* Generated by Cython 0.27.3 */ #define PY_SSIZE_T_CLEAN #include "Python.h" #ifndef Py_PYTHON_H #error Python headers needed to compile C extensions, please install development version of Python. -#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000) - #error Cython requires Python 2.6+ or Python 3.2+. +#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03030000) + #error Cython requires Python 2.6+ or Python 3.3+. #else -#define CYTHON_ABI "0_25_2" +#define CYTHON_ABI "0_27_3" +#define CYTHON_FUTURE_DIVISION 0 #include #ifndef offsetof #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) @@ -29,8 +30,9 @@ #ifndef DL_EXPORT #define DL_EXPORT(t) t #endif +#define __PYX_COMMA , #ifndef HAVE_LONG_LONG - #if PY_VERSION_HEX >= 0x03030000 || (PY_MAJOR_VERSION == 2 && PY_VERSION_HEX >= 0x02070000) + #if PY_VERSION_HEX >= 0x02070000 #define HAVE_LONG_LONG #endif #endif @@ -46,8 +48,14 @@ #define CYTHON_COMPILING_IN_CPYTHON 0 #undef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 0 - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #if PY_VERSION_HEX < 0x03050000 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #elif !defined(CYTHON_USE_ASYNC_SLOTS) + #define CYTHON_USE_ASYNC_SLOTS 1 + #endif #undef CYTHON_USE_PYLIST_INTERNALS #define CYTHON_USE_PYLIST_INTERNALS 0 #undef CYTHON_USE_UNICODE_INTERNALS @@ -66,6 +74,10 @@ #define CYTHON_FAST_THREAD_STATE 0 #undef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 0 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 #elif defined(PYSTON_VERSION) #define CYTHON_COMPILING_IN_PYPY 0 #define CYTHON_COMPILING_IN_PYSTON 1 @@ -73,6 +85,8 @@ #ifndef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 1 #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 #undef CYTHON_USE_ASYNC_SLOTS #define CYTHON_USE_ASYNC_SLOTS 0 #undef CYTHON_USE_PYLIST_INTERNALS @@ -97,6 +111,10 @@ #define CYTHON_FAST_THREAD_STATE 0 #undef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 0 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 #else #define CYTHON_COMPILING_IN_PYPY 0 #define CYTHON_COMPILING_IN_PYSTON 0 @@ -104,6 +122,12 @@ #ifndef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 1 #endif + #if PY_VERSION_HEX < 0x02070000 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #elif !defined(CYTHON_USE_PYTYPE_LOOKUP) + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif #if PY_MAJOR_VERSION < 3 #undef CYTHON_USE_ASYNC_SLOTS #define CYTHON_USE_ASYNC_SLOTS 0 @@ -143,6 +167,12 @@ #ifndef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 1 #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT (0 && PY_VERSION_HEX >= 0x03050000) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PY_VERSION_HEX >= 0x030400a1) + #endif #endif #if !defined(CYTHON_FAST_PYCCALL) #define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) @@ -181,19 +211,44 @@ #ifndef Py_TPFLAGS_HAVE_FINALIZE #define Py_TPFLAGS_HAVE_FINALIZE 0 #endif -#ifndef METH_FASTCALL - #define METH_FASTCALL 0x80 - typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject **args, - Py_ssize_t nargs, PyObject *kwnames); +#if PY_VERSION_HEX < 0x030700A0 || !defined(METH_FASTCALL) + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject **args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames); #else #define __Pyx_PyCFunctionFast _PyCFunctionFast + #define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords #endif #if CYTHON_FAST_PYCCALL #define __Pyx_PyFastCFunction_Check(func)\ - ((PyCFunction_Check(func) && (METH_FASTCALL == (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))))) + ((PyCFunction_Check(func) && (METH_FASTCALL == (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS))))) #else #define __Pyx_PyFastCFunction_Check(func) 0 #endif +#if !CYTHON_FAST_THREAD_STATE || PY_VERSION_HEX < 0x02070000 + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x03060000 + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#elif PY_VERSION_HEX >= 0x03000000 + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_Current +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#if PY_MAJOR_VERSION >= 3 || CYTHON_FUTURE_DIVISION + #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#else + #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) +#endif #if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) #define CYTHON_PEP393_ENABLED 1 #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ @@ -277,7 +332,6 @@ #ifndef PySet_CheckExact #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) #endif -#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) #define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) #if PY_MAJOR_VERSION >= 3 #define PyIntObject PyLongObject @@ -317,20 +371,28 @@ #else #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) #endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif #if CYTHON_USE_ASYNC_SLOTS #if PY_VERSION_HEX >= 0x030500B1 #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) #else + #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) + #endif +#else + #define __Pyx_PyType_AsAsync(obj) NULL +#endif +#ifndef __Pyx_PyAsyncMethodsStruct typedef struct { unaryfunc am_await; unaryfunc am_aiter; unaryfunc am_anext; } __Pyx_PyAsyncMethodsStruct; - #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) - #endif -#else - #define __Pyx_PyType_AsAsync(obj) NULL #endif #ifndef CYTHON_RESTRICT #if defined(__GNUC__) @@ -371,6 +433,43 @@ # endif #endif #define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#ifdef _MSC_VER + #ifndef _MSC_STDINT_H_ + #if _MSC_VER < 1300 + typedef unsigned char uint8_t; + typedef unsigned int uint32_t; + #else + typedef unsigned __int8 uint8_t; + typedef unsigned __int32 uint32_t; + #endif + #endif +#else + #include +#endif +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) && __cplusplus >= 201103L + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #elif __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__ ) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif #ifndef __cplusplus #error "Cython files generated with the C++ option must be compiled with a C++ compiler." @@ -425,14 +524,6 @@ static CYTHON_INLINE float __PYX_NAN() { __pyx_filename = __pyx_f[f_index]; __pyx_lineno = lineno; __pyx_clineno = __LINE__; goto Ln_error; \ } -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) - #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) -#else - #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) - #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) -#endif - #ifndef __PYX_EXTERN_C #ifdef __cplusplus #define __PYX_EXTERN_C extern "C" @@ -443,6 +534,14 @@ static CYTHON_INLINE float __PYX_NAN() { #define __PYX_HAVE__kenlm #define __PYX_HAVE_API__kenlm +#include +#include +#include "ios" +#include "new" +#include "stdexcept" +#include "typeinfo" +#include +#include #include "lm/word_index.hh" #include "lm/return.hh" #include "lm/state.hh" @@ -450,15 +549,20 @@ static CYTHON_INLINE float __PYX_NAN() { #include "util/mmap.hh" #include "lm/config.hh" #include "lm/model.hh" -#include "ios" -#include "new" -#include "stdexcept" -#include "typeinfo" +#include "util/file.hh" +#include "util/usage.hh" +#include "util/stream/config.hh" +#include "lm/builder/initial_probabilities.hh" +#include "lm/lm_exception.hh" +#include "lm/builder/discount.hh" +#include "lm/builder/adjust_counts.hh" +#include "lm/builder/output.hh" +#include "lm/builder/pipeline.hh" #ifdef _OPENMP #include #endif /* _OPENMP */ -#ifdef PYREX_WITHOUT_ASSERTIONS +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) #define CYTHON_WITHOUT_ASSERTIONS #endif @@ -489,8 +593,8 @@ typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* enc #define __Pyx_sst_abs(value) abs(value) #elif SIZEOF_LONG >= SIZEOF_SIZE_T #define __Pyx_sst_abs(value) labs(value) -#elif defined (_MSC_VER) && defined (_M_X64) - #define __Pyx_sst_abs(value) _abs64(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L #define __Pyx_sst_abs(value) llabs(value) #elif defined (__GNUC__) @@ -498,8 +602,8 @@ typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* enc #else #define __Pyx_sst_abs(value) ((value<0) ? -value : value) #endif -static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); -static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); #define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) #define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) #define __Pyx_PyBytes_FromString PyBytes_FromString @@ -512,23 +616,27 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize #endif -#define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyObject_AsWritableString(s) ((char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) #define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) #define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) #define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) #define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) #define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) -#if PY_MAJOR_VERSION < 3 -static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) -{ +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) { const Py_UNICODE *u_end = u; while (*u_end++) ; return (size_t)(u_end - u - 1); } -#else -#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen -#endif #define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) #define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode #define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode @@ -537,6 +645,8 @@ static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) #define __Pyx_PyBool_FromLong(b) ((b) ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False)) static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); #if CYTHON_ASSUME_SAFE_MACROS @@ -635,10 +745,12 @@ static int __Pyx_init_sys_getdefaultencoding_params(void) { #define likely(x) (x) #define unlikely(x) (x) #endif /* __GNUC__ */ +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } -static PyObject *__pyx_m; +static PyObject *__pyx_m = NULL; static PyObject *__pyx_d; static PyObject *__pyx_b; +static PyObject *__pyx_cython_runtime; static PyObject *__pyx_empty_tuple; static PyObject *__pyx_empty_bytes; static PyObject *__pyx_empty_unicode; @@ -649,10 +761,13 @@ static const char *__pyx_filename; static const char *__pyx_f[] = { - "kenlm.pyx", + "python/kenlm.pyx", + "stringsource", }; /*--- Type declarations ---*/ +struct __pyx_obj_5kenlm_Output; +struct __pyx_obj_5kenlm_PrintHook; struct __pyx_obj_5kenlm_FullScoreReturn; struct __pyx_obj_5kenlm_State; struct __pyx_obj_5kenlm_Config; @@ -662,6 +777,32 @@ struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores; /* "kenlm.pyx":11 * raise TypeError('Cannot convert %s to string' % type(data)) * + * cdef class Output: # <<<<<<<<<<<<<< + * """ + * Wrapper around lm::builder::Output. + */ +struct __pyx_obj_5kenlm_Output { + PyObject_HEAD + lm::builder::Output *_c_output; +}; + + +/* "kenlm.pyx":26 + * del self._c_output + * + * cdef class PrintHook: # <<<<<<<<<<<<<< + * """ + * Wrapper around lm::builder::Printhook + */ +struct __pyx_obj_5kenlm_PrintHook { + PyObject_HEAD + lm::builder::PrintHook *_c_printhook; +}; + + +/* "kenlm.pyx":170 + * + * * cdef class FullScoreReturn: # <<<<<<<<<<<<<< * """ * Wrapper around FullScoreReturn. @@ -674,7 +815,7 @@ struct __pyx_obj_5kenlm_FullScoreReturn { }; -/* "kenlm.pyx":44 +/* "kenlm.pyx":203 * return self.oov * * cdef class State: # <<<<<<<<<<<<<< @@ -687,7 +828,7 @@ struct __pyx_obj_5kenlm_State { }; -/* "kenlm.pyx":80 +/* "kenlm.pyx":239 * PARALLEL_READ = _kenlm.PARALLEL_READ * * cdef class Config: # <<<<<<<<<<<<<< @@ -700,7 +841,7 @@ struct __pyx_obj_5kenlm_Config { }; -/* "kenlm.pyx":96 +/* "kenlm.pyx":255 * self._c_config.load_method = to * * cdef class Model: # <<<<<<<<<<<<<< @@ -715,7 +856,7 @@ struct __pyx_obj_5kenlm_Model { }; -/* "kenlm.pyx":190 +/* "kenlm.pyx":349 * return 10.0**(-self.score(sentence) / words) * * def full_scores(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<< @@ -833,23 +974,35 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg /* PyThreadStateGet.proto */ #if CYTHON_FAST_THREAD_STATE #define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; -#define __Pyx_PyThreadState_assign __pyx_tstate = PyThreadState_GET(); +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#define __Pyx_PyErr_Occurred() __pyx_tstate->curexc_type #else #define __Pyx_PyThreadState_declare #define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() PyErr_Occurred() #endif /* PyErrFetchRestore.proto */ #if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) #define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) #define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) #define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) #define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif #else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) #define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) #define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) #define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) #define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) #endif @@ -869,6 +1022,36 @@ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ const char* function_name); +/* GetItemInt.proto */ +#define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Fast(o, (Py_ssize_t)i, is_list, wraparound, boundscheck) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL) :\ + __Pyx_GetItemInt_Generic(o, to_py_func(i)))) +#define __Pyx_GetItemInt_List(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_List_Fast(o, (Py_ssize_t)i, wraparound, boundscheck) :\ + (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck); +#define __Pyx_GetItemInt_Tuple(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Tuple_Fast(o, (Py_ssize_t)i, wraparound, boundscheck) :\ + (PyErr_SetString(PyExc_IndexError, "tuple index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck); +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j); +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, + int is_list, int wraparound, int boundscheck); + +/* PyIntBinop.proto */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject* __Pyx_PyInt_AddObjC(PyObject *op1, PyObject *op2, long intval, int inplace); +#else +#define __Pyx_PyInt_AddObjC(op1, op2, intval, inplace)\ + (inplace ? PyNumber_InPlaceAdd(op1, op2) : PyNumber_Add(op1, op2)) +#endif + /* PyFunctionFastCall.proto */ #if CYTHON_FAST_PYCALL #define __Pyx_PyFunction_FastCall(func, args, nargs)\ @@ -888,11 +1071,13 @@ static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObje #endif /* ArgTypeTest.proto */ -static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, - const char *name, int exact); +#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ + ((likely((Py_TYPE(obj) == type) | (none_allowed && (obj == Py_None)))) ? 1 :\ + __Pyx__ArgTypeTest(obj, type, name, exact)) +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); /* KeywordStringCheck.proto */ -static CYTHON_INLINE int __Pyx_CheckKeywordStrings(PyObject *kwdict, const char* function_name, int kw_allowed); +static int __Pyx_CheckKeywordStrings(PyObject *kwdict, const char* function_name, int kw_allowed); /* GetModuleGlobalName.proto */ static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); @@ -942,6 +1127,9 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); /* IncludeStringH.proto */ #include +/* SetupReduce.proto */ +static int __Pyx_setup_reduce(PyObject* type_obj); + /* Import.proto */ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); @@ -954,6 +1142,13 @@ static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, PyObject *dict, PyObject *mkw, int calculate_metaclass, int allow_py2_metaclass); +/* CLineInTraceback.proto */ +#ifdef CYTHON_CLINE_IN_TRACEBACK +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#else +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#endif + /* CodeObjectCache.proto */ typedef struct { PyCodeObject* code_object; @@ -979,9 +1174,6 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, /* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_enum__util_3a__3a_LoadMethod(enum util::LoadMethod value); -/* CIntToPy.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); - /* CppExceptionConversion.proto */ #ifndef __Pyx_CppExn2PyErr #include @@ -1024,6 +1216,12 @@ static void __Pyx_CppExn2PyErr() { } #endif +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint64_t(uint64_t value); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); + /* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value); @@ -1033,6 +1231,15 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_char(unsigned char valu /* CIntFromPy.proto */ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); +/* CIntFromPy.proto */ +static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE lm::WordIndex __Pyx_PyInt_As_lm_3a__3a_WordIndex(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE uint64_t __Pyx_PyInt_As_uint64_t(PyObject *); + /* CIntFromPy.proto */ static CYTHON_INLINE enum util::LoadMethod __Pyx_PyInt_As_enum__util_3a__3a_LoadMethod(PyObject *); @@ -1042,6 +1249,18 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); /* CIntFromPy.proto */ static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +#define __Pyx_PyErr_GivenExceptionMatches2(err, type1, type2) (PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2)) +#endif + /* FetchCommonType.proto */ static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); @@ -1055,9 +1274,10 @@ static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, /* PyObjectCallMethod1.proto */ static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); +static PyObject* __Pyx__PyObject_CallMethod1(PyObject* method, PyObject* arg); /* CoroutineBase.proto */ -typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyObject *); +typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyThreadState *, PyObject *); typedef struct { PyObject_HEAD __pyx_coroutine_body_t body; @@ -1077,12 +1297,30 @@ typedef struct { static __pyx_CoroutineObject *__Pyx__Coroutine_New( PyTypeObject *type, __pyx_coroutine_body_t body, PyObject *closure, PyObject *name, PyObject *qualname, PyObject *module_name); +static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit( + __pyx_CoroutineObject *gen, __pyx_coroutine_body_t body, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name); static int __Pyx_Coroutine_clear(PyObject *self); -#if 1 || PY_VERSION_HEX < 0x030300B0 -static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue); +static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value); +static PyObject *__Pyx_Coroutine_Close(PyObject *self); +static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args); +#define __Pyx_Coroutine_SwapException(self) {\ + __Pyx_ExceptionSwap(&(self)->exc_type, &(self)->exc_value, &(self)->exc_traceback);\ + __Pyx_Coroutine_ResetFrameBackpointer(self);\ + } +#define __Pyx_Coroutine_ResetAndClearException(self) {\ + __Pyx_ExceptionReset((self)->exc_type, (self)->exc_value, (self)->exc_traceback);\ + (self)->exc_type = (self)->exc_value = (self)->exc_traceback = NULL;\ + } +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyGen_FetchStopIterationValue(pvalue)\ + __Pyx_PyGen__FetchStopIterationValue(__pyx_tstate, pvalue) #else -#define __Pyx_PyGen_FetchStopIterationValue(pvalue) PyGen_FetchStopIterationValue(pvalue) +#define __Pyx_PyGen_FetchStopIterationValue(pvalue)\ + __Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, pvalue) #endif +static int __Pyx_PyGen__FetchStopIterationValue(PyThreadState *tstate, PyObject **pvalue); +static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__pyx_CoroutineObject *self); /* PatchModuleWithCoroutine.proto */ static PyObject* __Pyx_Coroutine_patch_module(PyObject* module, const char* py_code); @@ -1106,29 +1344,60 @@ static int __Pyx_check_binary_version(void); static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); +/* Module declarations from 'libc.string' */ + +/* Module declarations from 'libcpp.string' */ + +/* Module declarations from 'libcpp.vector' */ + +/* Module declarations from 'libc.stdint' */ + +/* Module declarations from 'libcpp' */ + /* Module declarations from '_kenlm' */ /* Module declarations from 'kenlm' */ +static PyTypeObject *__pyx_ptype_5kenlm_Output = 0; +static PyTypeObject *__pyx_ptype_5kenlm_PrintHook = 0; static PyTypeObject *__pyx_ptype_5kenlm_FullScoreReturn = 0; static PyTypeObject *__pyx_ptype_5kenlm_State = 0; static PyTypeObject *__pyx_ptype_5kenlm_Config = 0; static PyTypeObject *__pyx_ptype_5kenlm_Model = 0; static PyTypeObject *__pyx_ptype_5kenlm___pyx_scope_struct__full_scores = 0; static PyObject *__pyx_f_5kenlm_as_str(PyObject *); /*proto*/ +static PyObject *__pyx_f_5kenlm_Pipeline(struct lm::builder::PipelineConfig, PyObject *, struct __pyx_obj_5kenlm_Output *); /*proto*/ +static std::string __pyx_convert_string_from_py_std__in_string(PyObject *); /*proto*/ +static CYTHON_INLINE PyObject *__pyx_convert_PyObject_string_to_py_std__in_string(std::string const &); /*proto*/ +static CYTHON_INLINE PyObject *__pyx_convert_PyUnicode_string_to_py_std__in_string(std::string const &); /*proto*/ +static CYTHON_INLINE PyObject *__pyx_convert_PyStr_string_to_py_std__in_string(std::string const &); /*proto*/ +static CYTHON_INLINE PyObject *__pyx_convert_PyBytes_string_to_py_std__in_string(std::string const &); /*proto*/ +static CYTHON_INLINE PyObject *__pyx_convert_PyByteArray_string_to_py_std__in_string(std::string const &); /*proto*/ #define __Pyx_MODULE_NAME "kenlm" +extern int __pyx_module_is_main_kenlm; int __pyx_module_is_main_kenlm = 0; /* Implementation of 'kenlm' */ static PyObject *__pyx_builtin_TypeError; static PyObject *__pyx_builtin_RuntimeError; +static PyObject *__pyx_builtin_range; static PyObject *__pyx_builtin_IOError; -static const char __pyx_k__3[] = "\n"; -static const char __pyx_k__4[] = " "; +static const char __pyx_k_e[] = "e"; +static const char __pyx_k_i[] = "i"; +static const char __pyx_k_1G[] = "1G"; +static const char __pyx_k_8K[] = "8K"; +static const char __pyx_k__7[] = ""; +static const char __pyx_k_in[] = "_in"; static const char __pyx_k_os[] = "os"; +static const char __pyx_k_64M[] = "64M"; +static const char __pyx_k_Add[] = "Add"; +static const char __pyx_k__14[] = "]."; +static const char __pyx_k__22[] = "\n"; +static const char __pyx_k__23[] = " "; static const char __pyx_k_bos[] = "bos"; static const char __pyx_k_doc[] = "__doc__"; static const char __pyx_k_eos[] = "eos"; static const char __pyx_k_oov[] = "oov"; +static const char __pyx_k_out[] = "_out"; static const char __pyx_k_LAZY[] = "LAZY"; static const char __pyx_k_READ[] = "READ"; static const char __pyx_k_args[] = "args"; @@ -1142,6 +1411,9 @@ static const char __pyx_k_word[] = "word"; static const char __pyx_k_class[] = "__class__"; static const char __pyx_k_close[] = "close"; static const char __pyx_k_kenlm[] = "kenlm"; +static const char __pyx_k_lmplz[] = "lmplz"; +static const char __pyx_k_order[] = "order"; +static const char __pyx_k_range[] = "range"; static const char __pyx_k_score[] = "score"; static const char __pyx_k_split[] = "split"; static const char __pyx_k_throw[] = "throw"; @@ -1149,37 +1421,86 @@ static const char __pyx_k_config[] = "config"; static const char __pyx_k_encode[] = "encode"; static const char __pyx_k_format[] = "format"; static const char __pyx_k_import[] = "__import__"; +static const char __pyx_k_memory[] = "memory"; static const char __pyx_k_module[] = "__module__"; +static const char __pyx_k_output[] = "output"; +static const char __pyx_k_reduce[] = "__reduce__"; static const char __pyx_k_0_1_2_3[] = "{0}({1}, {2}, {3})"; static const char __pyx_k_IOError[] = "IOError"; static const char __pyx_k_abspath[] = "abspath"; static const char __pyx_k_prepare[] = "__prepare__"; +static const char __pyx_k_pruning[] = "pruning"; static const char __pyx_k_replace[] = "replace"; static const char __pyx_k_basename[] = "basename"; +static const char __pyx_k_discount[] = "discount"; +static const char __pyx_k_getstate[] = "__getstate__"; static const char __pyx_k_in_state[] = "in_state"; static const char __pyx_k_log_prob[] = "log_prob"; +static const char __pyx_k_output_q[] = "output_q"; +static const char __pyx_k_pipeline[] = "pipeline"; static const char __pyx_k_qualname[] = "__qualname__"; +static const char __pyx_k_renumber[] = "renumber"; static const char __pyx_k_sentence[] = "sentence"; +static const char __pyx_k_setstate[] = "__setstate__"; +static const char __pyx_k_write_fd[] = "write_fd"; static const char __pyx_k_TypeError[] = "TypeError"; +static const char __pyx_k_file_base[] = "file_base"; static const char __pyx_k_metaclass[] = "__metaclass__"; static const char __pyx_k_out_state[] = "out_state"; +static const char __pyx_k_reduce_ex[] = "__reduce_ex__"; +static const char __pyx_k_vocab_pad[] = "vocab_pad"; static const char __pyx_k_LoadMethod[] = "LoadMethod"; +static const char __pyx_k_sort_block[] = "sort_block"; +static const char __pyx_k_block_count[] = "block_count"; static const char __pyx_k_full_scores[] = "full_scores"; +static const char __pyx_k_keep_buffer[] = "keep_buffer"; +static const char __pyx_k_temp_prefix[] = "temp_prefix"; static const char __pyx_k_Model_from_0[] = ""; static const char __pyx_k_RuntimeError[] = "RuntimeError"; +static const char __pyx_k_intermediate[] = "intermediate"; static const char __pyx_k_ngram_length[] = "ngram_length"; +static const char __pyx_k_skip_symbols[] = "skip_symbols"; static const char __pyx_k_LanguageModel[] = "LanguageModel"; static const char __pyx_k_PARALLEL_READ[] = "PARALLEL_READ"; +static const char __pyx_k_minimum_block[] = "minimum_block"; +static const char __pyx_k_reduce_cython[] = "__reduce_cython__"; +static const char __pyx_k_was_parsed_as[] = " was parsed as "; +static const char __pyx_k_path_arpa_file[] = "path_arpa_file"; +static const char __pyx_k_path_text_file[] = "path_text_file"; +static const char __pyx_k_verbose_header[] = "verbose_header"; +static const char __pyx_k_vocab_estimate[] = "vocab_estimate"; +static const char __pyx_k_collapse_values[] = "collapse_values"; +static const char __pyx_k_setstate_cython[] = "__setstate_cython__"; static const char __pyx_k_POPULATE_OR_LAZY[] = "POPULATE_OR_LAZY"; static const char __pyx_k_POPULATE_OR_READ[] = "POPULATE_OR_READ"; +static const char __pyx_k_limit_vocab_file[] = "limit_vocab_file"; +static const char __pyx_k_python_kenlm_pyx[] = "python/kenlm.pyx"; static const char __pyx_k_Cannot_read_model[] = "Cannot read model '{}' ({})"; static const char __pyx_k_Model_full_scores[] = "Model.full_scores"; +static const char __pyx_k_discount_fallback[] = "discount_fallback"; +static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; +static const char __pyx_k_interpolate_unigrams[] = "interpolate_unigrams"; +static const char __pyx_k_The_discount_for_count[] = "The discount for count "; static const char __pyx_k_Cannot_convert_s_to_string[] = "Cannot convert %s to string"; +static const char __pyx_k_which_is_not_in_the_range_0[] = " which is not in the range [0, "; +static const char __pyx_k_but_the_model_only_has_order[] = " but the model only has order "; +static const char __pyx_k_vocab_pad_requires_interpolate[] = "--vocab_pad requires --interpolate_unigrams be on"; static const char __pyx_k_Backwards_compatability_stub_Use[] = "Backwards compatability stub. Use Model."; +static const char __pyx_k_Fallback_discounting_enabled_but[] = "Fallback discounting enabled, but no discount specified"; +static const char __pyx_k_Specify_at_most_three_fallback_d[] = "Specify at most three fallback discounts: 1, 2, and 3+"; +static const char __pyx_k_You_specified_pruning_thresholds[] = "You specified pruning thresholds for orders 1 through "; +static const char __pyx_k_no_default___reduce___due_to_non[] = "no default __reduce__ due to non-trivial __cinit__"; +static const char __pyx_k_self__c_config_cannot_be_convert[] = "self._c_config cannot be converted to a Python object for pickling"; +static const char __pyx_k_self__c_state_cannot_be_converte[] = "self._c_state cannot be converted to a Python object for pickling"; static PyObject *__pyx_kp_s_0_1_2_3; +static PyObject *__pyx_kp_s_1G; +static PyObject *__pyx_kp_s_64M; +static PyObject *__pyx_kp_s_8K; +static PyObject *__pyx_n_s_Add; static PyObject *__pyx_kp_s_Backwards_compatability_stub_Use; static PyObject *__pyx_kp_s_Cannot_convert_s_to_string; static PyObject *__pyx_kp_s_Cannot_read_model; +static PyObject *__pyx_kp_s_Fallback_discounting_enabled_but; static PyObject *__pyx_n_s_IOError; static PyObject *__pyx_n_s_LAZY; static PyObject *__pyx_n_s_LanguageModel; @@ -1191,55 +1512,123 @@ static PyObject *__pyx_n_s_POPULATE_OR_LAZY; static PyObject *__pyx_n_s_POPULATE_OR_READ; static PyObject *__pyx_n_s_READ; static PyObject *__pyx_n_s_RuntimeError; +static PyObject *__pyx_kp_s_Specify_at_most_three_fallback_d; +static PyObject *__pyx_kp_s_The_discount_for_count; static PyObject *__pyx_n_s_TypeError; -static PyObject *__pyx_kp_s__3; -static PyObject *__pyx_kp_s__4; +static PyObject *__pyx_kp_s_You_specified_pruning_thresholds; +static PyObject *__pyx_kp_s__14; +static PyObject *__pyx_kp_s__22; +static PyObject *__pyx_kp_s__23; +static PyObject *__pyx_kp_s__7; static PyObject *__pyx_n_s_abspath; static PyObject *__pyx_n_s_args; static PyObject *__pyx_n_s_basename; +static PyObject *__pyx_n_s_block_count; static PyObject *__pyx_n_s_bos; +static PyObject *__pyx_kp_s_but_the_model_only_has_order; static PyObject *__pyx_n_s_class; +static PyObject *__pyx_n_s_cline_in_traceback; static PyObject *__pyx_n_s_close; +static PyObject *__pyx_n_s_collapse_values; static PyObject *__pyx_n_s_config; +static PyObject *__pyx_n_s_discount; +static PyObject *__pyx_n_s_discount_fallback; static PyObject *__pyx_n_s_doc; +static PyObject *__pyx_n_s_e; static PyObject *__pyx_n_s_encode; static PyObject *__pyx_n_s_eos; +static PyObject *__pyx_n_s_file_base; static PyObject *__pyx_n_s_format; static PyObject *__pyx_n_s_full_scores; +static PyObject *__pyx_n_s_getstate; +static PyObject *__pyx_n_s_i; static PyObject *__pyx_n_s_import; +static PyObject *__pyx_n_s_in; static PyObject *__pyx_n_s_in_state; +static PyObject *__pyx_n_s_intermediate; +static PyObject *__pyx_n_s_interpolate_unigrams; +static PyObject *__pyx_n_s_keep_buffer; static PyObject *__pyx_n_s_kenlm; +static PyObject *__pyx_n_s_limit_vocab_file; +static PyObject *__pyx_n_s_lmplz; static PyObject *__pyx_n_s_log_prob; static PyObject *__pyx_n_s_main; +static PyObject *__pyx_n_s_memory; static PyObject *__pyx_n_s_metaclass; +static PyObject *__pyx_n_s_minimum_block; static PyObject *__pyx_n_s_module; static PyObject *__pyx_n_s_name; static PyObject *__pyx_n_s_ngram_length; +static PyObject *__pyx_kp_s_no_default___reduce___due_to_non; static PyObject *__pyx_n_s_oov; +static PyObject *__pyx_n_s_order; static PyObject *__pyx_n_s_os; +static PyObject *__pyx_n_s_out; static PyObject *__pyx_n_s_out_state; +static PyObject *__pyx_n_s_output; +static PyObject *__pyx_n_s_output_q; static PyObject *__pyx_n_s_path; +static PyObject *__pyx_n_s_path_arpa_file; +static PyObject *__pyx_n_s_path_text_file; +static PyObject *__pyx_n_s_pipeline; static PyObject *__pyx_n_s_prepare; +static PyObject *__pyx_n_s_pruning; +static PyObject *__pyx_kp_s_python_kenlm_pyx; static PyObject *__pyx_n_s_qualname; +static PyObject *__pyx_n_s_range; +static PyObject *__pyx_n_s_reduce; +static PyObject *__pyx_n_s_reduce_cython; +static PyObject *__pyx_n_s_reduce_ex; +static PyObject *__pyx_n_s_renumber; static PyObject *__pyx_n_s_replace; static PyObject *__pyx_n_s_score; +static PyObject *__pyx_kp_s_self__c_config_cannot_be_convert; +static PyObject *__pyx_kp_s_self__c_state_cannot_be_converte; static PyObject *__pyx_n_s_send; static PyObject *__pyx_n_s_sentence; +static PyObject *__pyx_n_s_setstate; +static PyObject *__pyx_n_s_setstate_cython; +static PyObject *__pyx_n_s_skip_symbols; +static PyObject *__pyx_n_s_sort_block; static PyObject *__pyx_n_s_split; +static PyObject *__pyx_n_s_temp_prefix; static PyObject *__pyx_n_s_test; static PyObject *__pyx_n_s_throw; static PyObject *__pyx_n_s_utf8; +static PyObject *__pyx_n_s_verbose_header; +static PyObject *__pyx_n_s_vocab_estimate; +static PyObject *__pyx_n_s_vocab_pad; +static PyObject *__pyx_kp_s_vocab_pad_requires_interpolate; +static PyObject *__pyx_kp_s_was_parsed_as; +static PyObject *__pyx_kp_s_which_is_not_in_the_range_0; static PyObject *__pyx_n_s_word; +static PyObject *__pyx_n_s_write_fd; +static int __pyx_pf_5kenlm_6Output___cinit__(struct __pyx_obj_5kenlm_Output *__pyx_v_self, PyObject *__pyx_v_file_base, PyObject *__pyx_v_keep_buffer, PyObject *__pyx_v_output_q); /* proto */ +static PyObject *__pyx_pf_5kenlm_6Output_2Add(struct __pyx_obj_5kenlm_Output *__pyx_v_self, PyObject *__pyx_v_write_fd, PyObject *__pyx_v_verbose_header); /* proto */ +static void __pyx_pf_5kenlm_6Output_4__dealloc__(struct __pyx_obj_5kenlm_Output *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5kenlm_6Output_6__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_Output *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5kenlm_6Output_8__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_Output *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static int __pyx_pf_5kenlm_9PrintHook___cinit__(struct __pyx_obj_5kenlm_PrintHook *__pyx_v_self, PyObject *__pyx_v_write_fd, PyObject *__pyx_v_verbose_header); /* proto */ +static void __pyx_pf_5kenlm_9PrintHook_2__dealloc__(struct __pyx_obj_5kenlm_PrintHook *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5kenlm_9PrintHook_4__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_PrintHook *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5kenlm_9PrintHook_6__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_PrintHook *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static PyObject *__pyx_pf_5kenlm_lmplz(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_path_text_file, PyObject *__pyx_v_path_arpa_file, PyObject *__pyx_v_order, PyObject *__pyx_v_interpolate_unigrams, PyObject *__pyx_v_skip_symbols, PyObject *__pyx_v_temp_prefix, PyObject *__pyx_v_memory, PyObject *__pyx_v_minimum_block, PyObject *__pyx_v_sort_block, PyObject *__pyx_v_block_count, PyObject *__pyx_v_vocab_estimate, PyObject *__pyx_v_vocab_pad, PyObject *__pyx_v_verbose_header, PyObject *__pyx_v_intermediate, PyObject *__pyx_v_renumber, PyObject *__pyx_v_collapse_values, PyObject *__pyx_v_pruning, PyObject *__pyx_v_limit_vocab_file, PyObject *__pyx_v_discount_fallback); /* proto */ static int __pyx_pf_5kenlm_15FullScoreReturn___cinit__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self, PyObject *__pyx_v_log_prob, PyObject *__pyx_v_ngram_length, PyObject *__pyx_v_oov); /* proto */ static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_2__repr__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_8log_prob___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_12ngram_length___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_3oov___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_4__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_6__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ static PyObject *__pyx_pf_5kenlm_5State___richcmp__(struct __pyx_obj_5kenlm_State *__pyx_v_qa, struct __pyx_obj_5kenlm_State *__pyx_v_qb, int __pyx_v_op); /* proto */ static Py_hash_t __pyx_pf_5kenlm_5State_2__hash__(struct __pyx_obj_5kenlm_State *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5kenlm_5State_4__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_State *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5kenlm_5State_6__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_State *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ static int __pyx_pf_5kenlm_6Config___init__(struct __pyx_obj_5kenlm_Config *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5kenlm_6Config_11load_method___get__(struct __pyx_obj_5kenlm_Config *__pyx_v_self); /* proto */ static int __pyx_pf_5kenlm_6Config_11load_method_2__set__(struct __pyx_obj_5kenlm_Config *__pyx_v_self, PyObject *__pyx_v_to); /* proto */ +static PyObject *__pyx_pf_5kenlm_6Config_2__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_Config *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5kenlm_6Config_4__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_Config *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ static int __pyx_pf_5kenlm_5Model___init__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_path, struct __pyx_obj_5kenlm_Config *__pyx_v_config); /* proto */ static void __pyx_pf_5kenlm_5Model_2__dealloc__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5kenlm_5Model_5order___get__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */ @@ -1256,15 +1645,45 @@ static PyObject *__pyx_pf_5kenlm_5Model_23__reduce__(struct __pyx_obj_5kenlm_Mod static PyObject *__pyx_pf_5kenlm_5Model_4path___get__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */ static int __pyx_pf_5kenlm_5Model_4path_2__set__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_value); /* proto */ static int __pyx_pf_5kenlm_5Model_4path_4__del__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */ +static PyObject *__pyx_tp_new_5kenlm_Output(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5kenlm_PrintHook(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5kenlm_FullScoreReturn(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5kenlm_State(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5kenlm_Config(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5kenlm_Model(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5kenlm___pyx_scope_struct__full_scores(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_float_0_0; +static PyObject *__pyx_float_0_5; +static PyObject *__pyx_float_1_5; static PyObject *__pyx_float_10_0; -static struct __pyx_obj_5kenlm_Config *__pyx_k__2; +static PyObject *__pyx_int_0; +static PyObject *__pyx_int_1; +static PyObject *__pyx_int_2; +static PyObject *__pyx_int_3; +static PyObject *__pyx_int_4; +static PyObject *__pyx_int_1000000; +static PyObject *__pyx_k__6; +static PyObject *__pyx_k__8; +static struct __pyx_obj_5kenlm_Config *__pyx_k__21; static PyObject *__pyx_tuple_; +static PyObject *__pyx_tuple__2; +static PyObject *__pyx_tuple__3; +static PyObject *__pyx_tuple__4; static PyObject *__pyx_tuple__5; +static PyObject *__pyx_tuple__9; +static PyObject *__pyx_tuple__10; +static PyObject *__pyx_tuple__11; +static PyObject *__pyx_tuple__12; +static PyObject *__pyx_tuple__13; +static PyObject *__pyx_tuple__15; +static PyObject *__pyx_tuple__16; +static PyObject *__pyx_tuple__17; +static PyObject *__pyx_tuple__18; +static PyObject *__pyx_tuple__19; +static PyObject *__pyx_tuple__20; +static PyObject *__pyx_tuple__24; +static PyObject *__pyx_tuple__25; +static PyObject *__pyx_codeobj__26; /* "kenlm.pyx":4 * cimport _kenlm @@ -1359,7 +1778,7 @@ static PyObject *__pyx_f_5kenlm_as_str(PyObject *__pyx_v_data) { * return data.encode('utf8') * raise TypeError('Cannot convert %s to string' % type(data)) # <<<<<<<<<<<<<< * - * cdef class FullScoreReturn: + * cdef class Output: */ __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_Cannot_convert_s_to_string, ((PyObject *)Py_TYPE(__pyx_v_data))); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 9, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); @@ -1395,54 +1814,59 @@ static PyObject *__pyx_f_5kenlm_as_str(PyObject *__pyx_v_data) { return __pyx_r; } -/* "kenlm.pyx":24 - * cdef bint oov +/* "kenlm.pyx":17 + * cdef _kenlm.Output* _c_output + * + * def __cinit__(self, file_base, keep_buffer, output_q): # <<<<<<<<<<<<<< + * self._c_output = new _kenlm.Output(file_base, keep_buffer, output_q) * - * def __cinit__(self, log_prob, ngram_length, oov): # <<<<<<<<<<<<<< - * self.log_prob = log_prob - * self.ngram_length = ngram_length */ /* Python wrapper */ -static int __pyx_pw_5kenlm_15FullScoreReturn_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pw_5kenlm_15FullScoreReturn_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_log_prob = 0; - PyObject *__pyx_v_ngram_length = 0; - PyObject *__pyx_v_oov = 0; +static int __pyx_pw_5kenlm_6Output_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5kenlm_6Output_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_file_base = 0; + PyObject *__pyx_v_keep_buffer = 0; + PyObject *__pyx_v_output_q = 0; int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_log_prob,&__pyx_n_s_ngram_length,&__pyx_n_s_oov,0}; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_file_base,&__pyx_n_s_keep_buffer,&__pyx_n_s_output_q,0}; PyObject* values[3] = {0,0,0}; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_log_prob)) != 0)) kw_args--; + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_file_base)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_ngram_length)) != 0)) kw_args--; + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_keep_buffer)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, 1); __PYX_ERR(0, 24, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, 1); __PYX_ERR(0, 17, __pyx_L3_error) } + CYTHON_FALLTHROUGH; case 2: - if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_oov)) != 0)) kw_args--; + if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_output_q)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, 2); __PYX_ERR(0, 24, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, 2); __PYX_ERR(0, 17, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(0, 24, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(0, 17, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { goto __pyx_L5_argtuple_error; @@ -1451,229 +1875,183 @@ static int __pyx_pw_5kenlm_15FullScoreReturn_1__cinit__(PyObject *__pyx_v_self, values[1] = PyTuple_GET_ITEM(__pyx_args, 1); values[2] = PyTuple_GET_ITEM(__pyx_args, 2); } - __pyx_v_log_prob = values[0]; - __pyx_v_ngram_length = values[1]; - __pyx_v_oov = values[2]; + __pyx_v_file_base = values[0]; + __pyx_v_keep_buffer = values[1]; + __pyx_v_output_q = values[2]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 24, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 17, __pyx_L3_error) __pyx_L3_error:; - __Pyx_AddTraceback("kenlm.FullScoreReturn.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("kenlm.Output.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return -1; __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn___cinit__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self), __pyx_v_log_prob, __pyx_v_ngram_length, __pyx_v_oov); + __pyx_r = __pyx_pf_5kenlm_6Output___cinit__(((struct __pyx_obj_5kenlm_Output *)__pyx_v_self), __pyx_v_file_base, __pyx_v_keep_buffer, __pyx_v_output_q); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static int __pyx_pf_5kenlm_15FullScoreReturn___cinit__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self, PyObject *__pyx_v_log_prob, PyObject *__pyx_v_ngram_length, PyObject *__pyx_v_oov) { +static int __pyx_pf_5kenlm_6Output___cinit__(struct __pyx_obj_5kenlm_Output *__pyx_v_self, PyObject *__pyx_v_file_base, PyObject *__pyx_v_keep_buffer, PyObject *__pyx_v_output_q) { int __pyx_r; __Pyx_RefNannyDeclarations - float __pyx_t_1; - int __pyx_t_2; - int __pyx_t_3; + char *__pyx_t_1; + bool __pyx_t_2; + bool __pyx_t_3; + lm::builder::Output *__pyx_t_4; __Pyx_RefNannySetupContext("__cinit__", 0); - /* "kenlm.pyx":25 + /* "kenlm.pyx":18 * - * def __cinit__(self, log_prob, ngram_length, oov): - * self.log_prob = log_prob # <<<<<<<<<<<<<< - * self.ngram_length = ngram_length - * self.oov = oov - */ - __pyx_t_1 = __pyx_PyFloat_AsFloat(__pyx_v_log_prob); if (unlikely((__pyx_t_1 == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 25, __pyx_L1_error) - __pyx_v_self->log_prob = __pyx_t_1; - - /* "kenlm.pyx":26 - * def __cinit__(self, log_prob, ngram_length, oov): - * self.log_prob = log_prob - * self.ngram_length = ngram_length # <<<<<<<<<<<<<< - * self.oov = oov + * def __cinit__(self, file_base, keep_buffer, output_q): + * self._c_output = new _kenlm.Output(file_base, keep_buffer, output_q) # <<<<<<<<<<<<<< * + * def Add(self, write_fd, verbose_header): */ - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_ngram_length); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 26, __pyx_L1_error) - __pyx_v_self->ngram_length = __pyx_t_2; + __pyx_t_1 = __Pyx_PyObject_AsWritableString(__pyx_v_file_base); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) __PYX_ERR(0, 18, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_keep_buffer); if (unlikely((__pyx_t_2 == ((bool)-1)) && PyErr_Occurred())) __PYX_ERR(0, 18, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v_output_q); if (unlikely((__pyx_t_3 == ((bool)-1)) && PyErr_Occurred())) __PYX_ERR(0, 18, __pyx_L1_error) + try { + __pyx_t_4 = new lm::builder::Output(__pyx_t_1, __pyx_t_2, __pyx_t_3); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 18, __pyx_L1_error) + } + __pyx_v_self->_c_output = __pyx_t_4; - /* "kenlm.pyx":27 - * self.log_prob = log_prob - * self.ngram_length = ngram_length - * self.oov = oov # <<<<<<<<<<<<<< + /* "kenlm.pyx":17 + * cdef _kenlm.Output* _c_output * - * def __repr__(self): - */ - __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v_oov); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 27, __pyx_L1_error) - __pyx_v_self->oov = __pyx_t_3; - - /* "kenlm.pyx":24 - * cdef bint oov + * def __cinit__(self, file_base, keep_buffer, output_q): # <<<<<<<<<<<<<< + * self._c_output = new _kenlm.Output(file_base, keep_buffer, output_q) * - * def __cinit__(self, log_prob, ngram_length, oov): # <<<<<<<<<<<<<< - * self.log_prob = log_prob - * self.ngram_length = ngram_length */ /* function exit code */ __pyx_r = 0; goto __pyx_L0; __pyx_L1_error:; - __Pyx_AddTraceback("kenlm.FullScoreReturn.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("kenlm.Output.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = -1; __pyx_L0:; __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "kenlm.pyx":29 - * self.oov = oov +/* "kenlm.pyx":20 + * self._c_output = new _kenlm.Output(file_base, keep_buffer, output_q) * - * def __repr__(self): # <<<<<<<<<<<<<< - * return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov)) + * def Add(self, write_fd, verbose_header): # <<<<<<<<<<<<<< + * self._c_output.Add(new _kenlm.PrintHook(write_fd, verbose_header)) * */ /* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3__repr__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3__repr__(PyObject *__pyx_v_self) { +static PyObject *__pyx_pw_5kenlm_6Output_3Add(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_pw_5kenlm_6Output_3Add(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_write_fd = 0; + PyObject *__pyx_v_verbose_header = 0; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_2__repr__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self)); + __Pyx_RefNannySetupContext("Add (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_write_fd,&__pyx_n_s_verbose_header,0}; + PyObject* values[2] = {0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_write_fd)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_verbose_header)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("Add", 1, 2, 2, 1); __PYX_ERR(0, 20, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "Add") < 0)) __PYX_ERR(0, 20, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + } + __pyx_v_write_fd = values[0]; + __pyx_v_verbose_header = values[1]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("Add", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 20, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("kenlm.Output.Add", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5kenlm_6Output_2Add(((struct __pyx_obj_5kenlm_Output *)__pyx_v_self), __pyx_v_write_fd, __pyx_v_verbose_header); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_2__repr__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) { +static PyObject *__pyx_pf_5kenlm_6Output_2Add(struct __pyx_obj_5kenlm_Output *__pyx_v_self, PyObject *__pyx_v_write_fd, PyObject *__pyx_v_verbose_header) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - int __pyx_t_8; - PyObject *__pyx_t_9 = NULL; - __Pyx_RefNannySetupContext("__repr__", 0); + int __pyx_t_1; + bool __pyx_t_2; + lm::builder::PrintHook *__pyx_t_3; + __Pyx_RefNannySetupContext("Add", 0); - /* "kenlm.pyx":30 + /* "kenlm.pyx":21 * - * def __repr__(self): - * return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov)) # <<<<<<<<<<<<<< + * def Add(self, write_fd, verbose_header): + * self._c_output.Add(new _kenlm.PrintHook(write_fd, verbose_header)) # <<<<<<<<<<<<<< * - * property log_prob: + * def __dealloc__(self): */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_kp_s_0_1_2_3, __pyx_n_s_format); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 30, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_class); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 30, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_name); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 30, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyFloat_FromDouble(__pyx_v_self->log_prob); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 30, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_5 = PyObject_Repr(__pyx_t_3); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 30, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_self->ngram_length); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 30, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_6 = PyObject_Repr(__pyx_t_3); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 30, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = __Pyx_PyBool_FromLong(__pyx_v_self->oov); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 30, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_7 = PyObject_Repr(__pyx_t_3); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 30, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = NULL; - __pyx_t_8 = 0; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_2))) { - __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); - if (likely(__pyx_t_3)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); - __Pyx_INCREF(__pyx_t_3); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_2, function); - __pyx_t_8 = 1; - } + __pyx_t_1 = __Pyx_PyInt_As_int(__pyx_v_write_fd); if (unlikely((__pyx_t_1 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 21, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_verbose_header); if (unlikely((__pyx_t_2 == ((bool)-1)) && PyErr_Occurred())) __PYX_ERR(0, 21, __pyx_L1_error) + try { + __pyx_t_3 = new lm::builder::PrintHook(__pyx_t_1, __pyx_t_2); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 21, __pyx_L1_error) } - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[5] = {__pyx_t_3, __pyx_t_4, __pyx_t_5, __pyx_t_6, __pyx_t_7}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_8, 4+__pyx_t_8); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 30, __pyx_L1_error) - __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[5] = {__pyx_t_3, __pyx_t_4, __pyx_t_5, __pyx_t_6, __pyx_t_7}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_8, 4+__pyx_t_8); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 30, __pyx_L1_error) - __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - } else - #endif - { - __pyx_t_9 = PyTuple_New(4+__pyx_t_8); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 30, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_9); - if (__pyx_t_3) { - __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_3); __pyx_t_3 = NULL; - } - __Pyx_GIVEREF(__pyx_t_4); - PyTuple_SET_ITEM(__pyx_t_9, 0+__pyx_t_8, __pyx_t_4); - __Pyx_GIVEREF(__pyx_t_5); - PyTuple_SET_ITEM(__pyx_t_9, 1+__pyx_t_8, __pyx_t_5); - __Pyx_GIVEREF(__pyx_t_6); - PyTuple_SET_ITEM(__pyx_t_9, 2+__pyx_t_8, __pyx_t_6); - __Pyx_GIVEREF(__pyx_t_7); - PyTuple_SET_ITEM(__pyx_t_9, 3+__pyx_t_8, __pyx_t_7); - __pyx_t_4 = 0; - __pyx_t_5 = 0; - __pyx_t_6 = 0; - __pyx_t_7 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_9, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 30, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + try { + __pyx_v_self->_c_output->Add(__pyx_t_3); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 21, __pyx_L1_error) } - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - /* "kenlm.pyx":29 - * self.oov = oov + /* "kenlm.pyx":20 + * self._c_output = new _kenlm.Output(file_base, keep_buffer, output_q) * - * def __repr__(self): # <<<<<<<<<<<<<< - * return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov)) + * def Add(self, write_fd, verbose_header): # <<<<<<<<<<<<<< + * self._c_output.Add(new _kenlm.PrintHook(write_fd, verbose_header)) * */ /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_XDECREF(__pyx_t_5); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_XDECREF(__pyx_t_9); - __Pyx_AddTraceback("kenlm.FullScoreReturn.__repr__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("kenlm.Output.Add", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); @@ -1681,2342 +2059,2743 @@ static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_2__repr__(struct __pyx_obj_5k return __pyx_r; } -/* "kenlm.pyx":33 +/* "kenlm.pyx":23 + * self._c_output.Add(new _kenlm.PrintHook(write_fd, verbose_header)) * - * property log_prob: - * def __get__(self): # <<<<<<<<<<<<<< - * return self.log_prob + * def __dealloc__(self): # <<<<<<<<<<<<<< + * del self._c_output * */ /* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_8log_prob_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_8log_prob_1__get__(PyObject *__pyx_v_self) { +static void __pyx_pw_5kenlm_6Output_5__dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_pw_5kenlm_6Output_5__dealloc__(PyObject *__pyx_v_self) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_pf_5kenlm_6Output_4__dealloc__(((struct __pyx_obj_5kenlm_Output *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +static void __pyx_pf_5kenlm_6Output_4__dealloc__(struct __pyx_obj_5kenlm_Output *__pyx_v_self) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__", 0); + + /* "kenlm.pyx":24 + * + * def __dealloc__(self): + * del self._c_output # <<<<<<<<<<<<<< + * + * cdef class PrintHook: + */ + delete __pyx_v_self->_c_output; + + /* "kenlm.pyx":23 + * self._c_output.Add(new _kenlm.PrintHook(write_fd, verbose_header)) + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * del self._c_output + * + */ + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_6Output_7__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5kenlm_6Output_7__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_8log_prob___get__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self)); + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_6Output_6__reduce_cython__(((struct __pyx_obj_5kenlm_Output *)__pyx_v_self)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_8log_prob___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) { +static PyObject *__pyx_pf_5kenlm_6Output_6__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_Output *__pyx_v_self) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_RefNannySetupContext("__reduce_cython__", 0); - /* "kenlm.pyx":34 - * property log_prob: - * def __get__(self): - * return self.log_prob # <<<<<<<<<<<<<< - * - * property ngram_length: + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->log_prob); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 34, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 2, __pyx_L1_error) - /* "kenlm.pyx":33 - * - * property log_prob: - * def __get__(self): # <<<<<<<<<<<<<< - * return self.log_prob - * + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.FullScoreReturn.log_prob.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("kenlm.Output.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; - __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "kenlm.pyx":37 - * - * property ngram_length: - * def __get__(self): # <<<<<<<<<<<<<< - * return self.ngram_length - * +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") */ /* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_12ngram_length_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_12ngram_length_1__get__(PyObject *__pyx_v_self) { +static PyObject *__pyx_pw_5kenlm_6Output_9__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state); /*proto*/ +static PyObject *__pyx_pw_5kenlm_6Output_9__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_12ngram_length___get__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self)); + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_6Output_8__setstate_cython__(((struct __pyx_obj_5kenlm_Output *)__pyx_v_self), ((PyObject *)__pyx_v___pyx_state)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_12ngram_length___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) { +static PyObject *__pyx_pf_5kenlm_6Output_8__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_Output *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_RefNannySetupContext("__setstate_cython__", 0); - /* "kenlm.pyx":38 - * property ngram_length: - * def __get__(self): - * return self.ngram_length # <<<<<<<<<<<<<< - * - * property oov: + /* "(tree fragment)":4 + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_self->ngram_length); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 38, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 4, __pyx_L1_error) - /* "kenlm.pyx":37 - * - * property ngram_length: - * def __get__(self): # <<<<<<<<<<<<<< - * return self.ngram_length - * + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.FullScoreReturn.ngram_length.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("kenlm.Output.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; - __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "kenlm.pyx":41 +/* "kenlm.pyx":32 + * cdef _kenlm.PrintHook* _c_printhook * - * property oov: - * def __get__(self): # <<<<<<<<<<<<<< - * return self.oov + * def __cinit__(self, write_fd, verbose_header): # <<<<<<<<<<<<<< + * self._c_printhook = new _kenlm.PrintHook(write_fd, verbose_header) * */ /* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3oov_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3oov_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; +static int __pyx_pw_5kenlm_9PrintHook_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5kenlm_9PrintHook_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_write_fd = 0; + PyObject *__pyx_v_verbose_header = 0; + int __pyx_r; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_3oov___get__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self)); + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_write_fd,&__pyx_n_s_verbose_header,0}; + PyObject* values[2] = {0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_write_fd)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_verbose_header)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 2, 2, 1); __PYX_ERR(0, 32, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(0, 32, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + } + __pyx_v_write_fd = values[0]; + __pyx_v_verbose_header = values[1]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 32, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("kenlm.PrintHook.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5kenlm_9PrintHook___cinit__(((struct __pyx_obj_5kenlm_PrintHook *)__pyx_v_self), __pyx_v_write_fd, __pyx_v_verbose_header); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_3oov___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) { - PyObject *__pyx_r = NULL; +static int __pyx_pf_5kenlm_9PrintHook___cinit__(struct __pyx_obj_5kenlm_PrintHook *__pyx_v_self, PyObject *__pyx_v_write_fd, PyObject *__pyx_v_verbose_header) { + int __pyx_r; __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); + int __pyx_t_1; + bool __pyx_t_2; + lm::builder::PrintHook *__pyx_t_3; + __Pyx_RefNannySetupContext("__cinit__", 0); - /* "kenlm.pyx":42 - * property oov: - * def __get__(self): - * return self.oov # <<<<<<<<<<<<<< + /* "kenlm.pyx":33 * - * cdef class State: + * def __cinit__(self, write_fd, verbose_header): + * self._c_printhook = new _kenlm.PrintHook(write_fd, verbose_header) # <<<<<<<<<<<<<< + * + * def __dealloc__(self): */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->oov); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 42, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; + __pyx_t_1 = __Pyx_PyInt_As_int(__pyx_v_write_fd); if (unlikely((__pyx_t_1 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 33, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_verbose_header); if (unlikely((__pyx_t_2 == ((bool)-1)) && PyErr_Occurred())) __PYX_ERR(0, 33, __pyx_L1_error) + try { + __pyx_t_3 = new lm::builder::PrintHook(__pyx_t_1, __pyx_t_2); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 33, __pyx_L1_error) + } + __pyx_v_self->_c_printhook = __pyx_t_3; - /* "kenlm.pyx":41 + /* "kenlm.pyx":32 + * cdef _kenlm.PrintHook* _c_printhook * - * property oov: - * def __get__(self): # <<<<<<<<<<<<<< - * return self.oov + * def __cinit__(self, write_fd, verbose_header): # <<<<<<<<<<<<<< + * self._c_printhook = new _kenlm.PrintHook(write_fd, verbose_header) * */ /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.FullScoreReturn.oov.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; + __Pyx_AddTraceback("kenlm.PrintHook.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "kenlm.pyx":55 - * cdef _kenlm.State _c_state +/* "kenlm.pyx":35 + * self._c_printhook = new _kenlm.PrintHook(write_fd, verbose_header) + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * del self._c_printhook * - * def __richcmp__(State qa, State qb, int op): # <<<<<<<<<<<<<< - * r = qa._c_state.Compare(qb._c_state) - * if op == 0: # < */ /* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5State_1__richcmp__(PyObject *__pyx_v_qa, PyObject *__pyx_v_qb, int __pyx_v_op); /*proto*/ -static PyObject *__pyx_pw_5kenlm_5State_1__richcmp__(PyObject *__pyx_v_qa, PyObject *__pyx_v_qb, int __pyx_v_op) { - PyObject *__pyx_r = 0; +static void __pyx_pw_5kenlm_9PrintHook_3__dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_pw_5kenlm_9PrintHook_3__dealloc__(PyObject *__pyx_v_self) { __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__richcmp__ (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_qa), __pyx_ptype_5kenlm_State, 1, "qa", 0))) __PYX_ERR(0, 55, __pyx_L1_error) - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_qb), __pyx_ptype_5kenlm_State, 1, "qb", 0))) __PYX_ERR(0, 55, __pyx_L1_error) - __pyx_r = __pyx_pf_5kenlm_5State___richcmp__(((struct __pyx_obj_5kenlm_State *)__pyx_v_qa), ((struct __pyx_obj_5kenlm_State *)__pyx_v_qb), ((int)__pyx_v_op)); + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_pf_5kenlm_9PrintHook_2__dealloc__(((struct __pyx_obj_5kenlm_PrintHook *)__pyx_v_self)); /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = NULL; - __pyx_L0:; __Pyx_RefNannyFinishContext(); - return __pyx_r; } -static PyObject *__pyx_pf_5kenlm_5State___richcmp__(struct __pyx_obj_5kenlm_State *__pyx_v_qa, struct __pyx_obj_5kenlm_State *__pyx_v_qb, int __pyx_v_op) { - int __pyx_v_r; - PyObject *__pyx_r = NULL; +static void __pyx_pf_5kenlm_9PrintHook_2__dealloc__(struct __pyx_obj_5kenlm_PrintHook *__pyx_v_self) { __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__richcmp__", 0); + __Pyx_RefNannySetupContext("__dealloc__", 0); - /* "kenlm.pyx":56 + /* "kenlm.pyx":36 * - * def __richcmp__(State qa, State qb, int op): - * r = qa._c_state.Compare(qb._c_state) # <<<<<<<<<<<<<< - * if op == 0: # < - * return r < 0 + * def __dealloc__(self): + * del self._c_printhook # <<<<<<<<<<<<<< + * + * cdef Pipeline(_kenlm.PipelineConfig pipeline, __in, Output output): */ - __pyx_v_r = __pyx_v_qa->_c_state.Compare(__pyx_v_qb->_c_state); + delete __pyx_v_self->_c_printhook; - /* "kenlm.pyx":57 - * def __richcmp__(State qa, State qb, int op): - * r = qa._c_state.Compare(qb._c_state) - * if op == 0: # < # <<<<<<<<<<<<<< - * return r < 0 - * elif op == 1: # <= + /* "kenlm.pyx":35 + * self._c_printhook = new _kenlm.PrintHook(write_fd, verbose_header) + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * del self._c_printhook + * */ - switch (__pyx_v_op) { - case 0: - - /* "kenlm.pyx":58 - * r = qa._c_state.Compare(qb._c_state) - * if op == 0: # < - * return r < 0 # <<<<<<<<<<<<<< - * elif op == 1: # <= - * return r <= 0 - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r < 0)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 58, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":57 - * def __richcmp__(State qa, State qb, int op): - * r = qa._c_state.Compare(qb._c_state) - * if op == 0: # < # <<<<<<<<<<<<<< - * return r < 0 - * elif op == 1: # <= - */ - break; - - /* "kenlm.pyx":59 - * if op == 0: # < - * return r < 0 - * elif op == 1: # <= # <<<<<<<<<<<<<< - * return r <= 0 - * elif op == 2: # == - */ - case 1: - - /* "kenlm.pyx":60 - * return r < 0 - * elif op == 1: # <= - * return r <= 0 # <<<<<<<<<<<<<< - * elif op == 2: # == - * return r == 0 - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r <= 0)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 60, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":59 - * if op == 0: # < - * return r < 0 - * elif op == 1: # <= # <<<<<<<<<<<<<< - * return r <= 0 - * elif op == 2: # == - */ - break; - - /* "kenlm.pyx":61 - * elif op == 1: # <= - * return r <= 0 - * elif op == 2: # == # <<<<<<<<<<<<<< - * return r == 0 - * elif op == 3: # != - */ - case 2: - - /* "kenlm.pyx":62 - * return r <= 0 - * elif op == 2: # == - * return r == 0 # <<<<<<<<<<<<<< - * elif op == 3: # != - * return r != 0 - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r == 0)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 62, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":61 - * elif op == 1: # <= - * return r <= 0 - * elif op == 2: # == # <<<<<<<<<<<<<< - * return r == 0 - * elif op == 3: # != - */ - break; - - /* "kenlm.pyx":63 - * elif op == 2: # == - * return r == 0 - * elif op == 3: # != # <<<<<<<<<<<<<< - * return r != 0 - * elif op == 4: # > - */ - case 3: - /* "kenlm.pyx":64 - * return r == 0 - * elif op == 3: # != - * return r != 0 # <<<<<<<<<<<<<< - * elif op == 4: # > - * return r > 0 - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r != 0)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 64, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} - /* "kenlm.pyx":63 - * elif op == 2: # == - * return r == 0 - * elif op == 3: # != # <<<<<<<<<<<<<< - * return r != 0 - * elif op == 4: # > +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): */ - break; - /* "kenlm.pyx":65 - * elif op == 3: # != - * return r != 0 - * elif op == 4: # > # <<<<<<<<<<<<<< - * return r > 0 - * else: # >= - */ - case 4: +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_9PrintHook_5__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5kenlm_9PrintHook_5__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_9PrintHook_4__reduce_cython__(((struct __pyx_obj_5kenlm_PrintHook *)__pyx_v_self)); - /* "kenlm.pyx":66 - * return r != 0 - * elif op == 4: # > - * return r > 0 # <<<<<<<<<<<<<< - * else: # >= - * return r >= 0 - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r > 0)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 66, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} - /* "kenlm.pyx":65 - * elif op == 3: # != - * return r != 0 - * elif op == 4: # > # <<<<<<<<<<<<<< - * return r > 0 - * else: # >= - */ - break; - default: +static PyObject *__pyx_pf_5kenlm_9PrintHook_4__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_PrintHook *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); - /* "kenlm.pyx":68 - * return r > 0 - * else: # >= - * return r >= 0 # <<<<<<<<<<<<<< - * - * def __hash__(self): + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r >= 0)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 68, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - break; - } + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 2, __pyx_L1_error) - /* "kenlm.pyx":55 - * cdef _kenlm.State _c_state - * - * def __richcmp__(State qa, State qb, int op): # <<<<<<<<<<<<<< - * r = qa._c_state.Compare(qb._c_state) - * if op == 0: # < + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.State.__richcmp__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("kenlm.PrintHook.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; - __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "kenlm.pyx":70 - * return r >= 0 - * - * def __hash__(self): # <<<<<<<<<<<<<< - * return _kenlm.hash_value(self._c_state) - * +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") */ /* Python wrapper */ -static Py_hash_t __pyx_pw_5kenlm_5State_3__hash__(PyObject *__pyx_v_self); /*proto*/ -static Py_hash_t __pyx_pw_5kenlm_5State_3__hash__(PyObject *__pyx_v_self) { - Py_hash_t __pyx_r; +static PyObject *__pyx_pw_5kenlm_9PrintHook_7__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state); /*proto*/ +static PyObject *__pyx_pw_5kenlm_9PrintHook_7__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__hash__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5State_2__hash__(((struct __pyx_obj_5kenlm_State *)__pyx_v_self)); + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_9PrintHook_6__setstate_cython__(((struct __pyx_obj_5kenlm_PrintHook *)__pyx_v_self), ((PyObject *)__pyx_v___pyx_state)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static Py_hash_t __pyx_pf_5kenlm_5State_2__hash__(struct __pyx_obj_5kenlm_State *__pyx_v_self) { - Py_hash_t __pyx_r; +static PyObject *__pyx_pf_5kenlm_9PrintHook_6__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_PrintHook *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__hash__", 0); + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); - /* "kenlm.pyx":71 - * - * def __hash__(self): - * return _kenlm.hash_value(self._c_state) # <<<<<<<<<<<<<< - * - * class LoadMethod: + /* "(tree fragment)":4 + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< */ - __pyx_r = lm::ngram::hash_value(__pyx_v_self->_c_state); - goto __pyx_L0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 4, __pyx_L1_error) - /* "kenlm.pyx":70 - * return r >= 0 - * - * def __hash__(self): # <<<<<<<<<<<<<< - * return _kenlm.hash_value(self._c_state) - * + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") */ /* function exit code */ - __pyx_L0:; - if (unlikely(__pyx_r == -1) && !PyErr_Occurred()) __pyx_r = -2; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("kenlm.PrintHook.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "kenlm.pyx":87 - * cdef _kenlm.Config _c_config +/* "kenlm.pyx":38 + * del self._c_printhook * - * def __init__(self): # <<<<<<<<<<<<<< - * self._c_config = _kenlm.Config() + * cdef Pipeline(_kenlm.PipelineConfig pipeline, __in, Output output): # <<<<<<<<<<<<<< + * _kenlm.Pipeline(pipeline, __in, output._c_output[0]) * */ -/* Python wrapper */ -static int __pyx_pw_5kenlm_6Config_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pw_5kenlm_6Config_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); - if (unlikely(PyTuple_GET_SIZE(__pyx_args) > 0)) { - __Pyx_RaiseArgtupleInvalid("__init__", 1, 0, 0, PyTuple_GET_SIZE(__pyx_args)); return -1;} - if (unlikely(__pyx_kwds) && unlikely(PyDict_Size(__pyx_kwds) > 0) && unlikely(!__Pyx_CheckKeywordStrings(__pyx_kwds, "__init__", 0))) return -1; - __pyx_r = __pyx_pf_5kenlm_6Config___init__(((struct __pyx_obj_5kenlm_Config *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_5kenlm_6Config___init__(struct __pyx_obj_5kenlm_Config *__pyx_v_self) { - int __pyx_r; +static PyObject *__pyx_f_5kenlm_Pipeline(struct lm::builder::PipelineConfig __pyx_v_pipeline, PyObject *__pyx_v___in, struct __pyx_obj_5kenlm_Output *__pyx_v_output) { + PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__init__", 0); + int __pyx_t_1; + __Pyx_RefNannySetupContext("Pipeline", 0); - /* "kenlm.pyx":88 + /* "kenlm.pyx":39 + * + * cdef Pipeline(_kenlm.PipelineConfig pipeline, __in, Output output): + * _kenlm.Pipeline(pipeline, __in, output._c_output[0]) # <<<<<<<<<<<<<< * - * def __init__(self): - * self._c_config = _kenlm.Config() # <<<<<<<<<<<<<< * - * property load_method: */ - __pyx_v_self->_c_config = lm::ngram::Config(); + __pyx_t_1 = __Pyx_PyInt_As_int(__pyx_v___in); if (unlikely((__pyx_t_1 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 39, __pyx_L1_error) + try { + lm::builder::Pipeline(__pyx_v_pipeline, __pyx_t_1, (__pyx_v_output->_c_output[0])); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 39, __pyx_L1_error) + } - /* "kenlm.pyx":87 - * cdef _kenlm.Config _c_config + /* "kenlm.pyx":38 + * del self._c_printhook * - * def __init__(self): # <<<<<<<<<<<<<< - * self._c_config = _kenlm.Config() + * cdef Pipeline(_kenlm.PipelineConfig pipeline, __in, Output output): # <<<<<<<<<<<<<< + * _kenlm.Pipeline(pipeline, __in, output._c_output[0]) * */ /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("kenlm.Pipeline", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "kenlm.pyx":91 +/* "kenlm.pyx":42 * - * property load_method: - * def __get__(self): # <<<<<<<<<<<<<< - * return self._c_config.load_method - * def __set__(self, to): + * + * def lmplz( # <<<<<<<<<<<<<< + * path_text_file, path_arpa_file, + * order=3, */ /* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_6Config_11load_method_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5kenlm_6Config_11load_method_1__get__(PyObject *__pyx_v_self) { +static PyObject *__pyx_pw_5kenlm_1lmplz(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_mdef_5kenlm_1lmplz = {"lmplz", (PyCFunction)__pyx_pw_5kenlm_1lmplz, METH_VARARGS|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_5kenlm_1lmplz(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_path_text_file = 0; + PyObject *__pyx_v_path_arpa_file = 0; + PyObject *__pyx_v_order = 0; + PyObject *__pyx_v_interpolate_unigrams = 0; + PyObject *__pyx_v_skip_symbols = 0; + PyObject *__pyx_v_temp_prefix = 0; + PyObject *__pyx_v_memory = 0; + PyObject *__pyx_v_minimum_block = 0; + PyObject *__pyx_v_sort_block = 0; + PyObject *__pyx_v_block_count = 0; + PyObject *__pyx_v_vocab_estimate = 0; + PyObject *__pyx_v_vocab_pad = 0; + PyObject *__pyx_v_verbose_header = 0; + PyObject *__pyx_v_intermediate = 0; + PyObject *__pyx_v_renumber = 0; + PyObject *__pyx_v_collapse_values = 0; + PyObject *__pyx_v_pruning = 0; + PyObject *__pyx_v_limit_vocab_file = 0; + PyObject *__pyx_v_discount_fallback = 0; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_6Config_11load_method___get__(((struct __pyx_obj_5kenlm_Config *)__pyx_v_self)); + __Pyx_RefNannySetupContext("lmplz (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_path_text_file,&__pyx_n_s_path_arpa_file,&__pyx_n_s_order,&__pyx_n_s_interpolate_unigrams,&__pyx_n_s_skip_symbols,&__pyx_n_s_temp_prefix,&__pyx_n_s_memory,&__pyx_n_s_minimum_block,&__pyx_n_s_sort_block,&__pyx_n_s_block_count,&__pyx_n_s_vocab_estimate,&__pyx_n_s_vocab_pad,&__pyx_n_s_verbose_header,&__pyx_n_s_intermediate,&__pyx_n_s_renumber,&__pyx_n_s_collapse_values,&__pyx_n_s_pruning,&__pyx_n_s_limit_vocab_file,&__pyx_n_s_discount_fallback,0}; + PyObject* values[19] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + values[2] = ((PyObject *)__pyx_int_3); + + /* "kenlm.pyx":45 + * path_text_file, path_arpa_file, + * order=3, + * interpolate_unigrams=True, # <<<<<<<<<<<<<< + * skip_symbols=False, + * temp_prefix=None, + */ + values[3] = ((PyObject *)Py_True); + + /* "kenlm.pyx":46 + * order=3, + * interpolate_unigrams=True, + * skip_symbols=False, # <<<<<<<<<<<<<< + * temp_prefix=None, + * memory="1G", + */ + values[4] = ((PyObject *)Py_False); + + /* "kenlm.pyx":47 + * interpolate_unigrams=True, + * skip_symbols=False, + * temp_prefix=None, # <<<<<<<<<<<<<< + * memory="1G", + * minimum_block="8K", + */ + values[5] = ((PyObject *)Py_None); + values[6] = ((PyObject *)__pyx_kp_s_1G); + values[7] = ((PyObject *)__pyx_kp_s_8K); + values[8] = ((PyObject *)__pyx_kp_s_64M); + values[9] = ((PyObject *)__pyx_int_2); + values[10] = ((PyObject *)__pyx_int_1000000); + values[11] = ((PyObject *)__pyx_int_0); + + /* "kenlm.pyx":54 + * vocab_estimate=1000000, + * vocab_pad=0, + * verbose_header=False, # <<<<<<<<<<<<<< + * intermediate=None, + * renumber=False, + */ + values[12] = ((PyObject *)Py_False); + + /* "kenlm.pyx":55 + * vocab_pad=0, + * verbose_header=False, + * intermediate=None, # <<<<<<<<<<<<<< + * renumber=False, + * collapse_values=False, + */ + values[13] = ((PyObject *)Py_None); + + /* "kenlm.pyx":56 + * verbose_header=False, + * intermediate=None, + * renumber=False, # <<<<<<<<<<<<<< + * collapse_values=False, + * pruning=[], + */ + values[14] = ((PyObject *)Py_False); - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_6Config_11load_method___get__(struct __pyx_obj_5kenlm_Config *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "kenlm.pyx":92 - * property load_method: - * def __get__(self): - * return self._c_config.load_method # <<<<<<<<<<<<<< - * def __set__(self, to): - * self._c_config.load_method = to - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_enum__util_3a__3a_LoadMethod(__pyx_v_self->_c_config.load_method); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 92, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":91 - * - * property load_method: - * def __get__(self): # <<<<<<<<<<<<<< - * return self._c_config.load_method - * def __set__(self, to): - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.Config.load_method.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":93 - * def __get__(self): - * return self._c_config.load_method - * def __set__(self, to): # <<<<<<<<<<<<<< - * self._c_config.load_method = to - * - */ - -/* Python wrapper */ -static int __pyx_pw_5kenlm_6Config_11load_method_3__set__(PyObject *__pyx_v_self, PyObject *__pyx_v_to); /*proto*/ -static int __pyx_pw_5kenlm_6Config_11load_method_3__set__(PyObject *__pyx_v_self, PyObject *__pyx_v_to) { - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__set__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_6Config_11load_method_2__set__(((struct __pyx_obj_5kenlm_Config *)__pyx_v_self), ((PyObject *)__pyx_v_to)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_5kenlm_6Config_11load_method_2__set__(struct __pyx_obj_5kenlm_Config *__pyx_v_self, PyObject *__pyx_v_to) { - int __pyx_r; - __Pyx_RefNannyDeclarations - enum util::LoadMethod __pyx_t_1; - __Pyx_RefNannySetupContext("__set__", 0); - - /* "kenlm.pyx":94 - * return self._c_config.load_method - * def __set__(self, to): - * self._c_config.load_method = to # <<<<<<<<<<<<<< - * - * cdef class Model: - */ - __pyx_t_1 = ((enum util::LoadMethod)__Pyx_PyInt_As_enum__util_3a__3a_LoadMethod(__pyx_v_to)); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 94, __pyx_L1_error) - __pyx_v_self->_c_config.load_method = __pyx_t_1; - - /* "kenlm.pyx":93 - * def __get__(self): - * return self._c_config.load_method - * def __set__(self, to): # <<<<<<<<<<<<<< - * self._c_config.load_method = to - * - */ - - /* function exit code */ - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_AddTraceback("kenlm.Config.load_method.__set__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":105 - * cdef _kenlm.const_Vocabulary* vocab - * - * def __init__(self, path, Config config = Config()): # <<<<<<<<<<<<<< - * """ - * Load the language model. - */ - -/* Python wrapper */ -static int __pyx_pw_5kenlm_5Model_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_5kenlm_5Model___init__[] = "\n Load the language model.\n\n :param path: path to an arpa file or a kenlm binary file.\n :param config: configuration options (see lm/config.hh for documentation)\n "; -#if CYTHON_COMPILING_IN_CPYTHON -struct wrapperbase __pyx_wrapperbase_5kenlm_5Model___init__; -#endif -static int __pyx_pw_5kenlm_5Model_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_path = 0; - struct __pyx_obj_5kenlm_Config *__pyx_v_config = 0; - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_path,&__pyx_n_s_config,0}; - PyObject* values[2] = {0,0}; - values[1] = (PyObject *)__pyx_k__2; + /* "kenlm.pyx":57 + * intermediate=None, + * renumber=False, + * collapse_values=False, # <<<<<<<<<<<<<< + * pruning=[], + * limit_vocab_file='', + */ + values[15] = ((PyObject *)Py_False); + values[16] = __pyx_k__6; + values[17] = ((PyObject *)__pyx_kp_s__7); + values[18] = __pyx_k__8; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { + case 19: values[18] = PyTuple_GET_ITEM(__pyx_args, 18); + CYTHON_FALLTHROUGH; + case 18: values[17] = PyTuple_GET_ITEM(__pyx_args, 17); + CYTHON_FALLTHROUGH; + case 17: values[16] = PyTuple_GET_ITEM(__pyx_args, 16); + CYTHON_FALLTHROUGH; + case 16: values[15] = PyTuple_GET_ITEM(__pyx_args, 15); + CYTHON_FALLTHROUGH; + case 15: values[14] = PyTuple_GET_ITEM(__pyx_args, 14); + CYTHON_FALLTHROUGH; + case 14: values[13] = PyTuple_GET_ITEM(__pyx_args, 13); + CYTHON_FALLTHROUGH; + case 13: values[12] = PyTuple_GET_ITEM(__pyx_args, 12); + CYTHON_FALLTHROUGH; + case 12: values[11] = PyTuple_GET_ITEM(__pyx_args, 11); + CYTHON_FALLTHROUGH; + case 11: values[10] = PyTuple_GET_ITEM(__pyx_args, 10); + CYTHON_FALLTHROUGH; + case 10: values[9] = PyTuple_GET_ITEM(__pyx_args, 9); + CYTHON_FALLTHROUGH; + case 9: values[8] = PyTuple_GET_ITEM(__pyx_args, 8); + CYTHON_FALLTHROUGH; + case 8: values[7] = PyTuple_GET_ITEM(__pyx_args, 7); + CYTHON_FALLTHROUGH; + case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); + CYTHON_FALLTHROUGH; + case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); + CYTHON_FALLTHROUGH; + case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); + CYTHON_FALLTHROUGH; + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + CYTHON_FALLTHROUGH; + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_path)) != 0)) kw_args--; + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_path_text_file)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_path_arpa_file)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("lmplz", 0, 2, 19, 1); __PYX_ERR(0, 42, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 2: if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_config); - if (value) { values[1] = value; kw_args--; } + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_order); + if (value) { values[2] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 3: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_interpolate_unigrams); + if (value) { values[3] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 4: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_skip_symbols); + if (value) { values[4] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 5: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_temp_prefix); + if (value) { values[5] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 6: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_memory); + if (value) { values[6] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 7: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_minimum_block); + if (value) { values[7] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 8: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_sort_block); + if (value) { values[8] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 9: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_block_count); + if (value) { values[9] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 10: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_vocab_estimate); + if (value) { values[10] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 11: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_vocab_pad); + if (value) { values[11] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 12: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_verbose_header); + if (value) { values[12] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 13: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_intermediate); + if (value) { values[13] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 14: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_renumber); + if (value) { values[14] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 15: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_collapse_values); + if (value) { values[15] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 16: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_pruning); + if (value) { values[16] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 17: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_limit_vocab_file); + if (value) { values[17] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 18: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_discount_fallback); + if (value) { values[18] = value; kw_args--; } } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(0, 105, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "lmplz") < 0)) __PYX_ERR(0, 42, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { + case 19: values[18] = PyTuple_GET_ITEM(__pyx_args, 18); + CYTHON_FALLTHROUGH; + case 18: values[17] = PyTuple_GET_ITEM(__pyx_args, 17); + CYTHON_FALLTHROUGH; + case 17: values[16] = PyTuple_GET_ITEM(__pyx_args, 16); + CYTHON_FALLTHROUGH; + case 16: values[15] = PyTuple_GET_ITEM(__pyx_args, 15); + CYTHON_FALLTHROUGH; + case 15: values[14] = PyTuple_GET_ITEM(__pyx_args, 14); + CYTHON_FALLTHROUGH; + case 14: values[13] = PyTuple_GET_ITEM(__pyx_args, 13); + CYTHON_FALLTHROUGH; + case 13: values[12] = PyTuple_GET_ITEM(__pyx_args, 12); + CYTHON_FALLTHROUGH; + case 12: values[11] = PyTuple_GET_ITEM(__pyx_args, 11); + CYTHON_FALLTHROUGH; + case 11: values[10] = PyTuple_GET_ITEM(__pyx_args, 10); + CYTHON_FALLTHROUGH; + case 10: values[9] = PyTuple_GET_ITEM(__pyx_args, 9); + CYTHON_FALLTHROUGH; + case 9: values[8] = PyTuple_GET_ITEM(__pyx_args, 8); + CYTHON_FALLTHROUGH; + case 8: values[7] = PyTuple_GET_ITEM(__pyx_args, 7); + CYTHON_FALLTHROUGH; + case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); + CYTHON_FALLTHROUGH; + case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); + CYTHON_FALLTHROUGH; + case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); + CYTHON_FALLTHROUGH; + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + CYTHON_FALLTHROUGH; + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); break; default: goto __pyx_L5_argtuple_error; } } - __pyx_v_path = values[0]; - __pyx_v_config = ((struct __pyx_obj_5kenlm_Config *)values[1]); + __pyx_v_path_text_file = values[0]; + __pyx_v_path_arpa_file = values[1]; + __pyx_v_order = values[2]; + __pyx_v_interpolate_unigrams = values[3]; + __pyx_v_skip_symbols = values[4]; + __pyx_v_temp_prefix = values[5]; + __pyx_v_memory = values[6]; + __pyx_v_minimum_block = values[7]; + __pyx_v_sort_block = values[8]; + __pyx_v_block_count = values[9]; + __pyx_v_vocab_estimate = values[10]; + __pyx_v_vocab_pad = values[11]; + __pyx_v_verbose_header = values[12]; + __pyx_v_intermediate = values[13]; + __pyx_v_renumber = values[14]; + __pyx_v_collapse_values = values[15]; + __pyx_v_pruning = values[16]; + __pyx_v_limit_vocab_file = values[17]; + __pyx_v_discount_fallback = values[18]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 105, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("lmplz", 0, 2, 19, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 42, __pyx_L3_error) __pyx_L3_error:; - __Pyx_AddTraceback("kenlm.Model.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("kenlm.lmplz", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); - return -1; + return NULL; __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_config), __pyx_ptype_5kenlm_Config, 1, "config", 0))) __PYX_ERR(0, 105, __pyx_L1_error) - __pyx_r = __pyx_pf_5kenlm_5Model___init__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), __pyx_v_path, __pyx_v_config); + __pyx_r = __pyx_pf_5kenlm_lmplz(__pyx_self, __pyx_v_path_text_file, __pyx_v_path_arpa_file, __pyx_v_order, __pyx_v_interpolate_unigrams, __pyx_v_skip_symbols, __pyx_v_temp_prefix, __pyx_v_memory, __pyx_v_minimum_block, __pyx_v_sort_block, __pyx_v_block_count, __pyx_v_vocab_estimate, __pyx_v_vocab_pad, __pyx_v_verbose_header, __pyx_v_intermediate, __pyx_v_renumber, __pyx_v_collapse_values, __pyx_v_pruning, __pyx_v_limit_vocab_file, __pyx_v_discount_fallback); + + /* "kenlm.pyx":42 + * + * + * def lmplz( # <<<<<<<<<<<<<< + * path_text_file, path_arpa_file, + * order=3, + */ /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = -1; - __pyx_L0:; __Pyx_RefNannyFinishContext(); return __pyx_r; } -static int __pyx_pf_5kenlm_5Model___init__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_path, struct __pyx_obj_5kenlm_Config *__pyx_v_config) { - PyObject *__pyx_v_exception = NULL; - PyObject *__pyx_v_exception_message = NULL; - int __pyx_r; +static PyObject *__pyx_pf_5kenlm_lmplz(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_path_text_file, PyObject *__pyx_v_path_arpa_file, PyObject *__pyx_v_order, PyObject *__pyx_v_interpolate_unigrams, PyObject *__pyx_v_skip_symbols, PyObject *__pyx_v_temp_prefix, PyObject *__pyx_v_memory, PyObject *__pyx_v_minimum_block, PyObject *__pyx_v_sort_block, PyObject *__pyx_v_block_count, PyObject *__pyx_v_vocab_estimate, PyObject *__pyx_v_vocab_pad, PyObject *__pyx_v_verbose_header, PyObject *__pyx_v_intermediate, PyObject *__pyx_v_renumber, PyObject *__pyx_v_collapse_values, PyObject *__pyx_v_pruning, PyObject *__pyx_v_limit_vocab_file, PyObject *__pyx_v_discount_fallback) { + struct lm::builder::PipelineConfig __pyx_v_pipeline; + PyObject *__pyx_v_i = NULL; + PyObject *__pyx_v_discount = NULL; + PyObject *__pyx_v_e = NULL; + util::scoped_fd __pyx_v__in; + util::scoped_fd __pyx_v__out; + struct __pyx_obj_5kenlm_Output *__pyx_v_output = NULL; + PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; + size_t __pyx_t_1; + bool __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + std::string __pyx_t_5; + uint64_t __pyx_t_6; + lm::WordIndex __pyx_t_7; PyObject *__pyx_t_8 = NULL; - char *__pyx_t_9; - lm::base::Model *__pyx_t_10; - int __pyx_t_11; - PyObject *__pyx_t_12 = NULL; - PyObject *__pyx_t_13 = NULL; - __Pyx_RefNannySetupContext("__init__", 0); + PyObject *__pyx_t_9 = NULL; + Py_ssize_t __pyx_t_10; + PyObject *(*__pyx_t_11)(PyObject *); + Py_ssize_t __pyx_t_12; + Py_ssize_t __pyx_t_13; + PyObject *__pyx_t_14 = NULL; + PyObject *__pyx_t_15 = NULL; + float __pyx_t_16; + std::vector ::size_type __pyx_t_17; + char *__pyx_t_18; + int __pyx_t_19; + PyObject *__pyx_t_20 = NULL; + __Pyx_RefNannySetupContext("lmplz", 0); - /* "kenlm.pyx":112 - * :param config: configuration options (see lm/config.hh for documentation) - * """ - * self.path = os.path.abspath(as_str(path)) # <<<<<<<<<<<<<< - * try: - * self.model = _kenlm.LoadVirtual(self.path, config._c_config) + /* "kenlm.pyx":63 + * + * cdef _kenlm.PipelineConfig pipeline + * pipeline.order = order # <<<<<<<<<<<<<< + * pipeline.initial_probs.interpolate_unigrams = interpolate_unigrams + * */ - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_os); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 112, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_path); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 112, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_abspath); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 112, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = __pyx_f_5kenlm_as_str(__pyx_v_path); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 112, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = NULL; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_2))) { - __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_2); - if (likely(__pyx_t_4)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); - __Pyx_INCREF(__pyx_t_4); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_2, function); - } - } - if (!__pyx_t_4) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 112, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_GOTREF(__pyx_t_1); - } else { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_t_3}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 112, __pyx_L1_error) - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_t_3}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 112, __pyx_L1_error) - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - } else - #endif - { - __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 112, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4); __pyx_t_4 = NULL; - __Pyx_GIVEREF(__pyx_t_3); - PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_t_3); - __pyx_t_3 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 112, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - } - } - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 112, __pyx_L1_error) - __Pyx_GIVEREF(__pyx_t_1); - __Pyx_GOTREF(__pyx_v_self->path); - __Pyx_DECREF(__pyx_v_self->path); - __pyx_v_self->path = ((PyObject*)__pyx_t_1); - __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyInt_As_size_t(__pyx_v_order); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 63, __pyx_L1_error) + __pyx_v_pipeline.order = __pyx_t_1; - /* "kenlm.pyx":113 - * """ - * self.path = os.path.abspath(as_str(path)) - * try: # <<<<<<<<<<<<<< - * self.model = _kenlm.LoadVirtual(self.path, config._c_config) - * except RuntimeError as exception: + /* "kenlm.pyx":64 + * cdef _kenlm.PipelineConfig pipeline + * pipeline.order = order + * pipeline.initial_probs.interpolate_unigrams = interpolate_unigrams # <<<<<<<<<<<<<< + * + * if temp_prefix is None: */ - { - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign - __Pyx_ExceptionSave(&__pyx_t_6, &__pyx_t_7, &__pyx_t_8); - __Pyx_XGOTREF(__pyx_t_6); - __Pyx_XGOTREF(__pyx_t_7); - __Pyx_XGOTREF(__pyx_t_8); - /*try:*/ { + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_interpolate_unigrams); if (unlikely((__pyx_t_2 == ((bool)-1)) && PyErr_Occurred())) __PYX_ERR(0, 64, __pyx_L1_error) + __pyx_v_pipeline.initial_probs.interpolate_unigrams = __pyx_t_2; - /* "kenlm.pyx":114 - * self.path = os.path.abspath(as_str(path)) - * try: - * self.model = _kenlm.LoadVirtual(self.path, config._c_config) # <<<<<<<<<<<<<< - * except RuntimeError as exception: - * exception_message = str(exception).replace('\n', ' ') + /* "kenlm.pyx":66 + * pipeline.initial_probs.interpolate_unigrams = interpolate_unigrams + * + * if temp_prefix is None: # <<<<<<<<<<<<<< + * pipeline.sort.temp_prefix = _kenlm.DefaultTempDirectory() + * else: */ - __pyx_t_9 = __Pyx_PyObject_AsString(__pyx_v_self->path); if (unlikely((!__pyx_t_9) && PyErr_Occurred())) __PYX_ERR(0, 114, __pyx_L3_error) - try { - __pyx_t_10 = lm::ngram::LoadVirtual(__pyx_t_9, __pyx_v_config->_c_config); - } catch(...) { - __Pyx_CppExn2PyErr(); - __PYX_ERR(0, 114, __pyx_L3_error) - } - __pyx_v_self->model = __pyx_t_10; + __pyx_t_3 = (__pyx_v_temp_prefix == Py_None); + __pyx_t_4 = (__pyx_t_3 != 0); + if (__pyx_t_4) { - /* "kenlm.pyx":113 - * """ - * self.path = os.path.abspath(as_str(path)) - * try: # <<<<<<<<<<<<<< - * self.model = _kenlm.LoadVirtual(self.path, config._c_config) - * except RuntimeError as exception: + /* "kenlm.pyx":67 + * + * if temp_prefix is None: + * pipeline.sort.temp_prefix = _kenlm.DefaultTempDirectory() # <<<<<<<<<<<<<< + * else: + * pipeline.sort.temp_prefix = temp_prefix */ + try { + __pyx_t_5 = util::DefaultTempDirectory(); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 67, __pyx_L1_error) } - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; - goto __pyx_L10_try_end; - __pyx_L3_error:; - __Pyx_PyThreadState_assign - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_pipeline.sort.temp_prefix = __pyx_t_5; - /* "kenlm.pyx":115 - * try: - * self.model = _kenlm.LoadVirtual(self.path, config._c_config) - * except RuntimeError as exception: # <<<<<<<<<<<<<< - * exception_message = str(exception).replace('\n', ' ') - * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ + /* "kenlm.pyx":66 + * pipeline.initial_probs.interpolate_unigrams = interpolate_unigrams + * + * if temp_prefix is None: # <<<<<<<<<<<<<< + * pipeline.sort.temp_prefix = _kenlm.DefaultTempDirectory() + * else: */ - __pyx_t_11 = __Pyx_PyErr_ExceptionMatches(__pyx_builtin_RuntimeError); - if (__pyx_t_11) { - __Pyx_AddTraceback("kenlm.Model.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_1, &__pyx_t_2, &__pyx_t_5) < 0) __PYX_ERR(0, 115, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_GOTREF(__pyx_t_2); - __Pyx_GOTREF(__pyx_t_5); - __Pyx_INCREF(__pyx_t_2); - __pyx_v_exception = __pyx_t_2; + goto __pyx_L3; + } - /* "kenlm.pyx":116 - * self.model = _kenlm.LoadVirtual(self.path, config._c_config) - * except RuntimeError as exception: - * exception_message = str(exception).replace('\n', ' ') # <<<<<<<<<<<<<< - * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ - * from exception + /* "kenlm.pyx":69 + * pipeline.sort.temp_prefix = _kenlm.DefaultTempDirectory() + * else: + * pipeline.sort.temp_prefix = temp_prefix # <<<<<<<<<<<<<< + * + * if memory is None: */ - __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 116, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_INCREF(__pyx_v_exception); - __Pyx_GIVEREF(__pyx_v_exception); - PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_exception); - __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)(&PyString_Type)), __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 116, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_replace); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 116, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 116, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_v_exception_message = __pyx_t_4; - __pyx_t_4 = 0; + /*else*/ { + __pyx_t_5 = __pyx_convert_string_from_py_std__in_string(__pyx_v_temp_prefix); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 69, __pyx_L1_error) + __pyx_v_pipeline.sort.temp_prefix = __pyx_t_5; + } + __pyx_L3:; - /* "kenlm.pyx":117 - * except RuntimeError as exception: - * exception_message = str(exception).replace('\n', ' ') - * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ # <<<<<<<<<<<<<< - * from exception - * self.vocab = &self.model.BaseVocabulary() + /* "kenlm.pyx":71 + * pipeline.sort.temp_prefix = temp_prefix + * + * if memory is None: # <<<<<<<<<<<<<< + * pipeline.sort.total_memory = _kenlm.GuessPhysicalMemory() + * else: */ - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_kp_s_Cannot_read_model, __pyx_n_s_format); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 117, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_12 = NULL; - __pyx_t_11 = 0; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { - __pyx_t_12 = PyMethod_GET_SELF(__pyx_t_3); - if (likely(__pyx_t_12)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); - __Pyx_INCREF(__pyx_t_12); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_3, function); - __pyx_t_11 = 1; - } - } - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_3)) { - PyObject *__pyx_temp[3] = {__pyx_t_12, __pyx_v_path, __pyx_v_exception_message}; - __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_11, 2+__pyx_t_11); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 117, __pyx_L5_except_error) - __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0; - __Pyx_GOTREF(__pyx_t_4); - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_3)) { - PyObject *__pyx_temp[3] = {__pyx_t_12, __pyx_v_path, __pyx_v_exception_message}; - __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_11, 2+__pyx_t_11); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 117, __pyx_L5_except_error) - __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0; - __Pyx_GOTREF(__pyx_t_4); - } else - #endif - { - __pyx_t_13 = PyTuple_New(2+__pyx_t_11); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 117, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_13); - if (__pyx_t_12) { - __Pyx_GIVEREF(__pyx_t_12); PyTuple_SET_ITEM(__pyx_t_13, 0, __pyx_t_12); __pyx_t_12 = NULL; - } - __Pyx_INCREF(__pyx_v_path); - __Pyx_GIVEREF(__pyx_v_path); - PyTuple_SET_ITEM(__pyx_t_13, 0+__pyx_t_11, __pyx_v_path); - __Pyx_INCREF(__pyx_v_exception_message); - __Pyx_GIVEREF(__pyx_v_exception_message); - PyTuple_SET_ITEM(__pyx_t_13, 1+__pyx_t_11, __pyx_v_exception_message); - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_13, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 117, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; - } - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 117, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_GIVEREF(__pyx_t_4); - PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4); - __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_IOError, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 117, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_4 = (__pyx_v_memory == Py_None); + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { - /* "kenlm.pyx":118 - * exception_message = str(exception).replace('\n', ' ') - * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ - * from exception # <<<<<<<<<<<<<< - * self.vocab = &self.model.BaseVocabulary() + /* "kenlm.pyx":72 * + * if memory is None: + * pipeline.sort.total_memory = _kenlm.GuessPhysicalMemory() # <<<<<<<<<<<<<< + * else: + * pipeline.sort.total_memory = _kenlm.ParseSize(memory) */ - __Pyx_Raise(__pyx_t_4, 0, 0, __pyx_v_exception); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(0, 117, __pyx_L5_except_error) + try { + __pyx_t_6 = util::GuessPhysicalMemory(); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 72, __pyx_L1_error) } - goto __pyx_L5_except_error; - __pyx_L5_except_error:; + __pyx_v_pipeline.sort.total_memory = __pyx_t_6; - /* "kenlm.pyx":113 - * """ - * self.path = os.path.abspath(as_str(path)) - * try: # <<<<<<<<<<<<<< - * self.model = _kenlm.LoadVirtual(self.path, config._c_config) - * except RuntimeError as exception: + /* "kenlm.pyx":71 + * pipeline.sort.temp_prefix = temp_prefix + * + * if memory is None: # <<<<<<<<<<<<<< + * pipeline.sort.total_memory = _kenlm.GuessPhysicalMemory() + * else: */ - __Pyx_PyThreadState_assign - __Pyx_XGIVEREF(__pyx_t_6); - __Pyx_XGIVEREF(__pyx_t_7); - __Pyx_XGIVEREF(__pyx_t_8); - __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); - goto __pyx_L1_error; - __pyx_L10_try_end:; + goto __pyx_L4; } - /* "kenlm.pyx":119 - * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ - * from exception - * self.vocab = &self.model.BaseVocabulary() # <<<<<<<<<<<<<< + /* "kenlm.pyx":74 + * pipeline.sort.total_memory = _kenlm.GuessPhysicalMemory() + * else: + * pipeline.sort.total_memory = _kenlm.ParseSize(memory) # <<<<<<<<<<<<<< * - * def __dealloc__(self): + * pipeline.minimum_block = _kenlm.ParseSize(minimum_block) */ - __pyx_v_self->vocab = (&__pyx_v_self->model->BaseVocabulary()); + /*else*/ { + __pyx_t_5 = __pyx_convert_string_from_py_std__in_string(__pyx_v_memory); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 74, __pyx_L1_error) + try { + __pyx_t_6 = util::ParseSize(__pyx_t_5); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 74, __pyx_L1_error) + } + __pyx_v_pipeline.sort.total_memory = __pyx_t_6; + } + __pyx_L4:; - /* "kenlm.pyx":105 - * cdef _kenlm.const_Vocabulary* vocab + /* "kenlm.pyx":76 + * pipeline.sort.total_memory = _kenlm.ParseSize(memory) * - * def __init__(self, path, Config config = Config()): # <<<<<<<<<<<<<< - * """ - * Load the language model. + * pipeline.minimum_block = _kenlm.ParseSize(minimum_block) # <<<<<<<<<<<<<< + * pipeline.sort.buffer_size = _kenlm.ParseSize(sort_block) + * pipeline.block_count = block_count */ + __pyx_t_5 = __pyx_convert_string_from_py_std__in_string(__pyx_v_minimum_block); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 76, __pyx_L1_error) + try { + __pyx_t_6 = util::ParseSize(__pyx_t_5); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 76, __pyx_L1_error) + } + __pyx_v_pipeline.minimum_block = __pyx_t_6; - /* function exit code */ - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_XDECREF(__pyx_t_5); - __Pyx_XDECREF(__pyx_t_12); - __Pyx_XDECREF(__pyx_t_13); - __Pyx_AddTraceback("kenlm.Model.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_exception); - __Pyx_XDECREF(__pyx_v_exception_message); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":121 - * self.vocab = &self.model.BaseVocabulary() - * - * def __dealloc__(self): # <<<<<<<<<<<<<< - * del self.model + /* "kenlm.pyx":77 * + * pipeline.minimum_block = _kenlm.ParseSize(minimum_block) + * pipeline.sort.buffer_size = _kenlm.ParseSize(sort_block) # <<<<<<<<<<<<<< + * pipeline.block_count = block_count + * pipeline.vocab_estimate = vocab_estimate */ + __pyx_t_5 = __pyx_convert_string_from_py_std__in_string(__pyx_v_sort_block); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 77, __pyx_L1_error) + try { + __pyx_t_6 = util::ParseSize(__pyx_t_5); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 77, __pyx_L1_error) + } + __pyx_v_pipeline.sort.buffer_size = __pyx_t_6; -/* Python wrapper */ -static void __pyx_pw_5kenlm_5Model_3__dealloc__(PyObject *__pyx_v_self); /*proto*/ -static void __pyx_pw_5kenlm_5Model_3__dealloc__(PyObject *__pyx_v_self) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); - __pyx_pf_5kenlm_5Model_2__dealloc__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); + /* "kenlm.pyx":78 + * pipeline.minimum_block = _kenlm.ParseSize(minimum_block) + * pipeline.sort.buffer_size = _kenlm.ParseSize(sort_block) + * pipeline.block_count = block_count # <<<<<<<<<<<<<< + * pipeline.vocab_estimate = vocab_estimate + * pipeline.vocab_size_for_unk = vocab_pad + */ + __pyx_t_1 = __Pyx_PyInt_As_size_t(__pyx_v_block_count); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 78, __pyx_L1_error) + __pyx_v_pipeline.block_count = __pyx_t_1; - /* function exit code */ - __Pyx_RefNannyFinishContext(); -} + /* "kenlm.pyx":79 + * pipeline.sort.buffer_size = _kenlm.ParseSize(sort_block) + * pipeline.block_count = block_count + * pipeline.vocab_estimate = vocab_estimate # <<<<<<<<<<<<<< + * pipeline.vocab_size_for_unk = vocab_pad + * pipeline.renumber_vocabulary = renumber + */ + __pyx_t_7 = __Pyx_PyInt_As_lm_3a__3a_WordIndex(__pyx_v_vocab_estimate); if (unlikely((__pyx_t_7 == ((lm::WordIndex)-1)) && PyErr_Occurred())) __PYX_ERR(0, 79, __pyx_L1_error) + __pyx_v_pipeline.vocab_estimate = __pyx_t_7; -static void __pyx_pf_5kenlm_5Model_2__dealloc__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__dealloc__", 0); + /* "kenlm.pyx":80 + * pipeline.block_count = block_count + * pipeline.vocab_estimate = vocab_estimate + * pipeline.vocab_size_for_unk = vocab_pad # <<<<<<<<<<<<<< + * pipeline.renumber_vocabulary = renumber + * pipeline.output_q = collapse_values + */ + __pyx_t_6 = __Pyx_PyInt_As_uint64_t(__pyx_v_vocab_pad); if (unlikely((__pyx_t_6 == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 80, __pyx_L1_error) + __pyx_v_pipeline.vocab_size_for_unk = __pyx_t_6; - /* "kenlm.pyx":122 + /* "kenlm.pyx":81 + * pipeline.vocab_estimate = vocab_estimate + * pipeline.vocab_size_for_unk = vocab_pad + * pipeline.renumber_vocabulary = renumber # <<<<<<<<<<<<<< + * pipeline.output_q = collapse_values * - * def __dealloc__(self): - * del self.model # <<<<<<<<<<<<<< + */ + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_renumber); if (unlikely((__pyx_t_2 == ((bool)-1)) && PyErr_Occurred())) __PYX_ERR(0, 81, __pyx_L1_error) + __pyx_v_pipeline.renumber_vocabulary = __pyx_t_2; + + /* "kenlm.pyx":82 + * pipeline.vocab_size_for_unk = vocab_pad + * pipeline.renumber_vocabulary = renumber + * pipeline.output_q = collapse_values # <<<<<<<<<<<<<< * - * property order: + * if pipeline.vocab_size_for_unk and not pipeline.initial_probs.interpolate_unigrams: */ - delete __pyx_v_self->model; + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_collapse_values); if (unlikely((__pyx_t_2 == ((bool)-1)) && PyErr_Occurred())) __PYX_ERR(0, 82, __pyx_L1_error) + __pyx_v_pipeline.output_q = __pyx_t_2; - /* "kenlm.pyx":121 - * self.vocab = &self.model.BaseVocabulary() + /* "kenlm.pyx":84 + * pipeline.output_q = collapse_values * - * def __dealloc__(self): # <<<<<<<<<<<<<< - * del self.model + * if pipeline.vocab_size_for_unk and not pipeline.initial_probs.interpolate_unigrams: # <<<<<<<<<<<<<< + * raise RuntimeError('--vocab_pad requires --interpolate_unigrams be on') * */ + __pyx_t_4 = (__pyx_v_pipeline.vocab_size_for_unk != 0); + if (__pyx_t_4) { + } else { + __pyx_t_3 = __pyx_t_4; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_4 = ((!(__pyx_v_pipeline.initial_probs.interpolate_unigrams != 0)) != 0); + __pyx_t_3 = __pyx_t_4; + __pyx_L6_bool_binop_done:; + if (__pyx_t_3) { - /* function exit code */ - __Pyx_RefNannyFinishContext(); -} + /* "kenlm.pyx":85 + * + * if pipeline.vocab_size_for_unk and not pipeline.initial_probs.interpolate_unigrams: + * raise RuntimeError('--vocab_pad requires --interpolate_unigrams be on') # <<<<<<<<<<<<<< + * + * if skip_symbols: + */ + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_builtin_RuntimeError, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 85, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_Raise(__pyx_t_8, 0, 0, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __PYX_ERR(0, 85, __pyx_L1_error) -/* "kenlm.pyx":125 + /* "kenlm.pyx":84 + * pipeline.output_q = collapse_values * - * property order: - * def __get__(self): # <<<<<<<<<<<<<< - * return self.model.Order() + * if pipeline.vocab_size_for_unk and not pipeline.initial_probs.interpolate_unigrams: # <<<<<<<<<<<<<< + * raise RuntimeError('--vocab_pad requires --interpolate_unigrams be on') * */ + } -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_5order_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5kenlm_5Model_5order_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5Model_5order___get__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); + /* "kenlm.pyx":87 + * raise RuntimeError('--vocab_pad requires --interpolate_unigrams be on') + * + * if skip_symbols: # <<<<<<<<<<<<<< + * pipeline.disallowed_symbol_action = _kenlm.COMPLAIN + * else: + */ + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v_skip_symbols); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 87, __pyx_L1_error) + if (__pyx_t_3) { - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} + /* "kenlm.pyx":88 + * + * if skip_symbols: + * pipeline.disallowed_symbol_action = _kenlm.COMPLAIN # <<<<<<<<<<<<<< + * else: + * pipeline.disallowed_symbol_action = _kenlm.THROW_UP + */ + __pyx_v_pipeline.disallowed_symbol_action = lm::COMPLAIN; -static PyObject *__pyx_pf_5kenlm_5Model_5order___get__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "kenlm.pyx":126 - * property order: - * def __get__(self): - * return self.model.Order() # <<<<<<<<<<<<<< + /* "kenlm.pyx":87 + * raise RuntimeError('--vocab_pad requires --interpolate_unigrams be on') * - * def score(self, sentence, bos = True, eos = True): + * if skip_symbols: # <<<<<<<<<<<<<< + * pipeline.disallowed_symbol_action = _kenlm.COMPLAIN + * else: */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_unsigned_int(__pyx_v_self->model->Order()); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 126, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; + goto __pyx_L8; + } - /* "kenlm.pyx":125 - * - * property order: - * def __get__(self): # <<<<<<<<<<<<<< - * return self.model.Order() + /* "kenlm.pyx":90 + * pipeline.disallowed_symbol_action = _kenlm.COMPLAIN + * else: + * pipeline.disallowed_symbol_action = _kenlm.THROW_UP # <<<<<<<<<<<<<< * + * for i in range(4): */ + /*else*/ { + __pyx_v_pipeline.disallowed_symbol_action = lm::THROW_UP; + } + __pyx_L8:; - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.Model.order.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":128 - * return self.model.Order() + /* "kenlm.pyx":92 + * pipeline.disallowed_symbol_action = _kenlm.THROW_UP + * + * for i in range(4): # <<<<<<<<<<<<<< + * pipeline.discount.fallback.amount[i] = 0.0 * - * def score(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<< - * """ - * Return the log10 probability of a string. By default, the string is */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_5score(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_5kenlm_5Model_4score[] = "\n Return the log10 probability of a string. By default, the string is\n treated as a sentence. \n return log10 p(sentence | )\n\n If you do not want to condition on the beginning of sentence, pass\n bos = False\n Never include as part of the string. That would be predicting the\n beginning of sentence. Language models are only supposed to condition\n on it as context.\n\n Similarly, the end of sentence token can be omitted with\n eos = False\n Since language models explicitly predict , it can be part of the\n string.\n\n Examples:\n\n #Good: returns log10 p(this is a sentence . | )\n model.score(\"this is a sentence .\")\n #Good: same as the above but more explicit\n model.score(\"this is a sentence .\", bos = True, eos = True)\n\n #Bad: never include \n model.score(\" this is a sentence\")\n #Bad: never include , even if bos = False.\n model.score(\" this is a sentence\", bos = False)\n\n #Good: returns log10 p(a fragment)\n model.score(\"a fragment\", bos = False, eos = False)\n\n #Good: returns log10 p(a fragment )\n model.score(\"a fragment\", bos = False, eos = True)\n\n #Ok, but bad practice: returns log10 p(a fragment )\n #Unlike , the end of sentence token can appear explicitly.\n model.score(\"a fragment \", bos = False, eos = False)\n "; -static PyObject *__pyx_pw_5kenlm_5Model_5score(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_sentence = 0; - PyObject *__pyx_v_bos = 0; - PyObject *__pyx_v_eos = 0; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("score (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_sentence,&__pyx_n_s_bos,&__pyx_n_s_eos,0}; - PyObject* values[3] = {0,0,0}; - values[1] = ((PyObject *)Py_True); - values[2] = ((PyObject *)Py_True); - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_sentence)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_bos); - if (value) { values[1] = value; kw_args--; } - } - case 2: - if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_eos); - if (value) { values[2] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "score") < 0)) __PYX_ERR(0, 128, __pyx_L3_error) + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_tuple__10, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 92, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + if (likely(PyList_CheckExact(__pyx_t_8)) || PyTuple_CheckExact(__pyx_t_8)) { + __pyx_t_9 = __pyx_t_8; __Pyx_INCREF(__pyx_t_9); __pyx_t_10 = 0; + __pyx_t_11 = NULL; + } else { + __pyx_t_10 = -1; __pyx_t_9 = PyObject_GetIter(__pyx_t_8); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 92, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_11 = Py_TYPE(__pyx_t_9)->tp_iternext; if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 92, __pyx_L1_error) + } + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + for (;;) { + if (likely(!__pyx_t_11)) { + if (likely(PyList_CheckExact(__pyx_t_9))) { + if (__pyx_t_10 >= PyList_GET_SIZE(__pyx_t_9)) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = PyList_GET_ITEM(__pyx_t_9, __pyx_t_10); __Pyx_INCREF(__pyx_t_8); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 92, __pyx_L1_error) + #else + __pyx_t_8 = PySequence_ITEM(__pyx_t_9, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 92, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + #endif + } else { + if (__pyx_t_10 >= PyTuple_GET_SIZE(__pyx_t_9)) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = PyTuple_GET_ITEM(__pyx_t_9, __pyx_t_10); __Pyx_INCREF(__pyx_t_8); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 92, __pyx_L1_error) + #else + __pyx_t_8 = PySequence_ITEM(__pyx_t_9, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 92, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + #endif } } else { - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + __pyx_t_8 = __pyx_t_11(__pyx_t_9); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); + else __PYX_ERR(0, 92, __pyx_L1_error) + } break; - default: goto __pyx_L5_argtuple_error; } + __Pyx_GOTREF(__pyx_t_8); } - __pyx_v_sentence = values[0]; - __pyx_v_bos = values[1]; - __pyx_v_eos = values[2]; - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("score", 0, 1, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 128, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("kenlm.Model.score", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_5kenlm_5Model_4score(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), __pyx_v_sentence, __pyx_v_bos, __pyx_v_eos); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} + __Pyx_XDECREF_SET(__pyx_v_i, __pyx_t_8); + __pyx_t_8 = 0; -static PyObject *__pyx_pf_5kenlm_5Model_4score(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_sentence, PyObject *__pyx_v_bos, PyObject *__pyx_v_eos) { - PyObject *__pyx_v_words = 0; - lm::ngram::State __pyx_v_state; - lm::ngram::State __pyx_v_out_state; - float __pyx_v_total; - PyObject *__pyx_v_word = NULL; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - int __pyx_t_4; - Py_ssize_t __pyx_t_5; - char *__pyx_t_6; - __Pyx_RefNannySetupContext("score", 0); + /* "kenlm.pyx":93 + * + * for i in range(4): + * pipeline.discount.fallback.amount[i] = 0.0 # <<<<<<<<<<<<<< + * + * if discount_fallback is None: + */ + __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_v_i); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 93, __pyx_L1_error) + (__pyx_v_pipeline.discount.fallback.amount[__pyx_t_12]) = 0.0; - /* "kenlm.pyx":167 - * model.score("a fragment ", bos = False, eos = False) - * """ - * cdef list words = as_str(sentence).split() # <<<<<<<<<<<<<< - * cdef _kenlm.State state - * if bos: + /* "kenlm.pyx":92 + * pipeline.disallowed_symbol_action = _kenlm.THROW_UP + * + * for i in range(4): # <<<<<<<<<<<<<< + * pipeline.discount.fallback.amount[i] = 0.0 + * */ - __pyx_t_2 = __pyx_f_5kenlm_as_str(__pyx_v_sentence); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 167, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_split); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 167, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = NULL; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { - __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); - if (likely(__pyx_t_2)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); - __Pyx_INCREF(__pyx_t_2); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_3, function); - } - } - if (__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 167, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 167, __pyx_L1_error) } - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (!(likely(PyList_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "list", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 167, __pyx_L1_error) - __pyx_v_words = ((PyObject*)__pyx_t_1); - __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - /* "kenlm.pyx":169 - * cdef list words = as_str(sentence).split() - * cdef _kenlm.State state - * if bos: # <<<<<<<<<<<<<< - * self.model.BeginSentenceWrite(&state) - * else: + /* "kenlm.pyx":95 + * pipeline.discount.fallback.amount[i] = 0.0 + * + * if discount_fallback is None: # <<<<<<<<<<<<<< + * pipeline.discount.bad_action = _kenlm.THROW_UP + * else: */ - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_bos); if (unlikely(__pyx_t_4 < 0)) __PYX_ERR(0, 169, __pyx_L1_error) + __pyx_t_3 = (__pyx_v_discount_fallback == Py_None); + __pyx_t_4 = (__pyx_t_3 != 0); if (__pyx_t_4) { - /* "kenlm.pyx":170 - * cdef _kenlm.State state - * if bos: - * self.model.BeginSentenceWrite(&state) # <<<<<<<<<<<<<< - * else: - * self.model.NullContextWrite(&state) + /* "kenlm.pyx":96 + * + * if discount_fallback is None: + * pipeline.discount.bad_action = _kenlm.THROW_UP # <<<<<<<<<<<<<< + * else: + * if len(discount_fallback) > 3: */ - __pyx_v_self->model->BeginSentenceWrite((&__pyx_v_state)); + __pyx_v_pipeline.discount.bad_action = lm::THROW_UP; - /* "kenlm.pyx":169 - * cdef list words = as_str(sentence).split() - * cdef _kenlm.State state - * if bos: # <<<<<<<<<<<<<< - * self.model.BeginSentenceWrite(&state) - * else: + /* "kenlm.pyx":95 + * pipeline.discount.fallback.amount[i] = 0.0 + * + * if discount_fallback is None: # <<<<<<<<<<<<<< + * pipeline.discount.bad_action = _kenlm.THROW_UP + * else: */ - goto __pyx_L3; + goto __pyx_L11; } - /* "kenlm.pyx":172 - * self.model.BeginSentenceWrite(&state) - * else: - * self.model.NullContextWrite(&state) # <<<<<<<<<<<<<< - * cdef _kenlm.State out_state - * cdef float total = 0 + /* "kenlm.pyx":98 + * pipeline.discount.bad_action = _kenlm.THROW_UP + * else: + * if len(discount_fallback) > 3: # <<<<<<<<<<<<<< + * raise RuntimeError("Specify at most three fallback discounts: 1, 2, and 3+") + * */ /*else*/ { - __pyx_v_self->model->NullContextWrite((&__pyx_v_state)); - } - __pyx_L3:; - - /* "kenlm.pyx":174 - * self.model.NullContextWrite(&state) - * cdef _kenlm.State out_state - * cdef float total = 0 # <<<<<<<<<<<<<< - * for word in words: - * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) - */ - __pyx_v_total = 0.0; + __pyx_t_10 = PyObject_Length(__pyx_v_discount_fallback); if (unlikely(__pyx_t_10 == ((Py_ssize_t)-1))) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_t_4 = ((__pyx_t_10 > 3) != 0); + if (__pyx_t_4) { - /* "kenlm.pyx":175 - * cdef _kenlm.State out_state - * cdef float total = 0 - * for word in words: # <<<<<<<<<<<<<< - * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) - * state = out_state + /* "kenlm.pyx":99 + * else: + * if len(discount_fallback) > 3: + * raise RuntimeError("Specify at most three fallback discounts: 1, 2, and 3+") # <<<<<<<<<<<<<< + * + * if len(discount_fallback) == 0: */ - if (unlikely(__pyx_v_words == Py_None)) { - PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); - __PYX_ERR(0, 175, __pyx_L1_error) - } - __pyx_t_1 = __pyx_v_words; __Pyx_INCREF(__pyx_t_1); __pyx_t_5 = 0; - for (;;) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_3 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_5); __Pyx_INCREF(__pyx_t_3); __pyx_t_5++; if (unlikely(0 < 0)) __PYX_ERR(0, 175, __pyx_L1_error) - #else - __pyx_t_3 = PySequence_ITEM(__pyx_t_1, __pyx_t_5); __pyx_t_5++; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 175, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - #endif - __Pyx_XDECREF_SET(__pyx_v_word, __pyx_t_3); - __pyx_t_3 = 0; + __pyx_t_9 = __Pyx_PyObject_Call(__pyx_builtin_RuntimeError, __pyx_tuple__11, NULL); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 99, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_Raise(__pyx_t_9, 0, 0, 0); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __PYX_ERR(0, 99, __pyx_L1_error) - /* "kenlm.pyx":176 - * cdef float total = 0 - * for word in words: - * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) # <<<<<<<<<<<<<< - * state = out_state - * if eos: + /* "kenlm.pyx":98 + * pipeline.discount.bad_action = _kenlm.THROW_UP + * else: + * if len(discount_fallback) > 3: # <<<<<<<<<<<<<< + * raise RuntimeError("Specify at most three fallback discounts: 1, 2, and 3+") + * */ - __pyx_t_6 = __Pyx_PyObject_AsString(__pyx_v_word); if (unlikely((!__pyx_t_6) && PyErr_Occurred())) __PYX_ERR(0, 176, __pyx_L1_error) - __pyx_v_total = (__pyx_v_total + __pyx_v_self->model->BaseScore((&__pyx_v_state), __pyx_v_self->vocab->Index(__pyx_t_6), (&__pyx_v_out_state))); + } - /* "kenlm.pyx":177 - * for word in words: - * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) - * state = out_state # <<<<<<<<<<<<<< - * if eos: - * total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state) + /* "kenlm.pyx":101 + * raise RuntimeError("Specify at most three fallback discounts: 1, 2, and 3+") + * + * if len(discount_fallback) == 0: # <<<<<<<<<<<<<< + * raise RuntimeError("Fallback discounting enabled, but no discount specified") + * */ - __pyx_v_state = __pyx_v_out_state; + __pyx_t_10 = PyObject_Length(__pyx_v_discount_fallback); if (unlikely(__pyx_t_10 == ((Py_ssize_t)-1))) __PYX_ERR(0, 101, __pyx_L1_error) + __pyx_t_4 = ((__pyx_t_10 == 0) != 0); + if (__pyx_t_4) { - /* "kenlm.pyx":175 - * cdef _kenlm.State out_state - * cdef float total = 0 - * for word in words: # <<<<<<<<<<<<<< - * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) - * state = out_state + /* "kenlm.pyx":102 + * + * if len(discount_fallback) == 0: + * raise RuntimeError("Fallback discounting enabled, but no discount specified") # <<<<<<<<<<<<<< + * + * pipeline.discount.fallback.amount[0] = 0.0 */ - } - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_9 = __Pyx_PyObject_Call(__pyx_builtin_RuntimeError, __pyx_tuple__12, NULL); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 102, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_Raise(__pyx_t_9, 0, 0, 0); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __PYX_ERR(0, 102, __pyx_L1_error) - /* "kenlm.pyx":178 - * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) - * state = out_state - * if eos: # <<<<<<<<<<<<<< - * total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state) - * return total + /* "kenlm.pyx":101 + * raise RuntimeError("Specify at most three fallback discounts: 1, 2, and 3+") + * + * if len(discount_fallback) == 0: # <<<<<<<<<<<<<< + * raise RuntimeError("Fallback discounting enabled, but no discount specified") + * */ - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_eos); if (unlikely(__pyx_t_4 < 0)) __PYX_ERR(0, 178, __pyx_L1_error) - if (__pyx_t_4) { + } - /* "kenlm.pyx":179 - * state = out_state - * if eos: - * total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state) # <<<<<<<<<<<<<< - * return total + /* "kenlm.pyx":104 + * raise RuntimeError("Fallback discounting enabled, but no discount specified") * + * pipeline.discount.fallback.amount[0] = 0.0 # <<<<<<<<<<<<<< + * + * for i in range(3): */ - __pyx_v_total = (__pyx_v_total + __pyx_v_self->model->BaseScore((&__pyx_v_state), __pyx_v_self->vocab->EndSentence(), (&__pyx_v_out_state))); + (__pyx_v_pipeline.discount.fallback.amount[0]) = 0.0; - /* "kenlm.pyx":178 - * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) - * state = out_state - * if eos: # <<<<<<<<<<<<<< - * total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state) - * return total + /* "kenlm.pyx":106 + * pipeline.discount.fallback.amount[0] = 0.0 + * + * for i in range(3): # <<<<<<<<<<<<<< + * discount = discount_fallback[len(discount_fallback) - 1] + * if i < len(discount_fallback): */ + __pyx_t_9 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_tuple__13, NULL); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 106, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + if (likely(PyList_CheckExact(__pyx_t_9)) || PyTuple_CheckExact(__pyx_t_9)) { + __pyx_t_8 = __pyx_t_9; __Pyx_INCREF(__pyx_t_8); __pyx_t_10 = 0; + __pyx_t_11 = NULL; + } else { + __pyx_t_10 = -1; __pyx_t_8 = PyObject_GetIter(__pyx_t_9); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 106, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_11 = Py_TYPE(__pyx_t_8)->tp_iternext; if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 106, __pyx_L1_error) + } + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + for (;;) { + if (likely(!__pyx_t_11)) { + if (likely(PyList_CheckExact(__pyx_t_8))) { + if (__pyx_t_10 >= PyList_GET_SIZE(__pyx_t_8)) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_9 = PyList_GET_ITEM(__pyx_t_8, __pyx_t_10); __Pyx_INCREF(__pyx_t_9); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 106, __pyx_L1_error) + #else + __pyx_t_9 = PySequence_ITEM(__pyx_t_8, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 106, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + #endif + } else { + if (__pyx_t_10 >= PyTuple_GET_SIZE(__pyx_t_8)) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_9 = PyTuple_GET_ITEM(__pyx_t_8, __pyx_t_10); __Pyx_INCREF(__pyx_t_9); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 106, __pyx_L1_error) + #else + __pyx_t_9 = PySequence_ITEM(__pyx_t_8, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 106, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + #endif + } + } else { + __pyx_t_9 = __pyx_t_11(__pyx_t_8); + if (unlikely(!__pyx_t_9)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); + else __PYX_ERR(0, 106, __pyx_L1_error) + } + break; + } + __Pyx_GOTREF(__pyx_t_9); + } + __Pyx_XDECREF_SET(__pyx_v_i, __pyx_t_9); + __pyx_t_9 = 0; + + /* "kenlm.pyx":107 + * + * for i in range(3): + * discount = discount_fallback[len(discount_fallback) - 1] # <<<<<<<<<<<<<< + * if i < len(discount_fallback): + * discount = discount_fallback[i] + */ + __pyx_t_12 = PyObject_Length(__pyx_v_discount_fallback); if (unlikely(__pyx_t_12 == ((Py_ssize_t)-1))) __PYX_ERR(0, 107, __pyx_L1_error) + __pyx_t_13 = (__pyx_t_12 - 1); + __pyx_t_9 = __Pyx_GetItemInt(__pyx_v_discount_fallback, __pyx_t_13, Py_ssize_t, 1, PyInt_FromSsize_t, 0, 1, 1); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 107, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_XDECREF_SET(__pyx_v_discount, __pyx_t_9); + __pyx_t_9 = 0; + + /* "kenlm.pyx":108 + * for i in range(3): + * discount = discount_fallback[len(discount_fallback) - 1] + * if i < len(discount_fallback): # <<<<<<<<<<<<<< + * discount = discount_fallback[i] + * discount = float(discount) + */ + __pyx_t_13 = PyObject_Length(__pyx_v_discount_fallback); if (unlikely(__pyx_t_13 == ((Py_ssize_t)-1))) __PYX_ERR(0, 108, __pyx_L1_error) + __pyx_t_9 = PyInt_FromSsize_t(__pyx_t_13); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 108, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_14 = PyObject_RichCompare(__pyx_v_i, __pyx_t_9, Py_LT); __Pyx_XGOTREF(__pyx_t_14); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 108, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_14); if (unlikely(__pyx_t_4 < 0)) __PYX_ERR(0, 108, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + if (__pyx_t_4) { + + /* "kenlm.pyx":109 + * discount = discount_fallback[len(discount_fallback) - 1] + * if i < len(discount_fallback): + * discount = discount_fallback[i] # <<<<<<<<<<<<<< + * discount = float(discount) + * + */ + __pyx_t_14 = PyObject_GetItem(__pyx_v_discount_fallback, __pyx_v_i); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 109, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF_SET(__pyx_v_discount, __pyx_t_14); + __pyx_t_14 = 0; + + /* "kenlm.pyx":108 + * for i in range(3): + * discount = discount_fallback[len(discount_fallback) - 1] + * if i < len(discount_fallback): # <<<<<<<<<<<<<< + * discount = discount_fallback[i] + * discount = float(discount) + */ + } + + /* "kenlm.pyx":110 + * if i < len(discount_fallback): + * discount = discount_fallback[i] + * discount = float(discount) # <<<<<<<<<<<<<< + * + * if (discount < 0.0 or discount > (i + 1)): + */ + __pyx_t_14 = __Pyx_PyNumber_Float(__pyx_v_discount); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 110, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF_SET(__pyx_v_discount, __pyx_t_14); + __pyx_t_14 = 0; + + /* "kenlm.pyx":112 + * discount = float(discount) + * + * if (discount < 0.0 or discount > (i + 1)): # <<<<<<<<<<<<<< + * raise RuntimeError("The discount for count " + str(i+1) + " was parsed as " + discount + " which is not in the range [0, " + str(i+1) + "].") + * + */ + __pyx_t_14 = PyObject_RichCompare(__pyx_v_discount, __pyx_float_0_0, Py_LT); __Pyx_XGOTREF(__pyx_t_14); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 112, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_14); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 112, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + if (!__pyx_t_3) { + } else { + __pyx_t_4 = __pyx_t_3; + goto __pyx_L18_bool_binop_done; + } + __pyx_t_14 = __Pyx_PyInt_AddObjC(__pyx_v_i, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 112, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_9 = PyObject_RichCompare(__pyx_v_discount, __pyx_t_14, Py_GT); __Pyx_XGOTREF(__pyx_t_9); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 112, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_9); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(0, 112, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_4 = __pyx_t_3; + __pyx_L18_bool_binop_done:; + if (__pyx_t_4) { + + /* "kenlm.pyx":113 + * + * if (discount < 0.0 or discount > (i + 1)): + * raise RuntimeError("The discount for count " + str(i+1) + " was parsed as " + discount + " which is not in the range [0, " + str(i+1) + "].") # <<<<<<<<<<<<<< + * + * pipeline.discount.fallback.amount[i+1] = discount + */ + __pyx_t_9 = __Pyx_PyInt_AddObjC(__pyx_v_i, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_14 = PyTuple_New(1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_9); + PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_9); + __pyx_t_9 = 0; + __pyx_t_9 = __Pyx_PyObject_Call(((PyObject *)(&PyString_Type)), __pyx_t_14, NULL); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = PyNumber_Add(__pyx_kp_s_The_discount_for_count, __pyx_t_9); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_9 = PyNumber_Add(__pyx_t_14, __pyx_kp_s_was_parsed_as); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = PyNumber_Add(__pyx_t_9, __pyx_v_discount); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_9 = PyNumber_Add(__pyx_t_14, __pyx_kp_s_which_is_not_in_the_range_0); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = __Pyx_PyInt_AddObjC(__pyx_v_i, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_15 = PyTuple_New(1); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __Pyx_GIVEREF(__pyx_t_14); + PyTuple_SET_ITEM(__pyx_t_15, 0, __pyx_t_14); + __pyx_t_14 = 0; + __pyx_t_14 = __Pyx_PyObject_Call(((PyObject *)(&PyString_Type)), __pyx_t_15, NULL); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_15 = PyNumber_Add(__pyx_t_9, __pyx_t_14); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = PyNumber_Add(__pyx_t_15, __pyx_kp_s__14); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_15 = PyTuple_New(1); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __Pyx_GIVEREF(__pyx_t_14); + PyTuple_SET_ITEM(__pyx_t_15, 0, __pyx_t_14); + __pyx_t_14 = 0; + __pyx_t_14 = __Pyx_PyObject_Call(__pyx_builtin_RuntimeError, __pyx_t_15, NULL); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __Pyx_Raise(__pyx_t_14, 0, 0, 0); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __PYX_ERR(0, 113, __pyx_L1_error) + + /* "kenlm.pyx":112 + * discount = float(discount) + * + * if (discount < 0.0 or discount > (i + 1)): # <<<<<<<<<<<<<< + * raise RuntimeError("The discount for count " + str(i+1) + " was parsed as " + discount + " which is not in the range [0, " + str(i+1) + "].") + * + */ + } + + /* "kenlm.pyx":115 + * raise RuntimeError("The discount for count " + str(i+1) + " was parsed as " + discount + " which is not in the range [0, " + str(i+1) + "].") + * + * pipeline.discount.fallback.amount[i+1] = discount # <<<<<<<<<<<<<< + * + * pipeline.discount.bad_action = _kenlm.COMPLAIN + */ + __pyx_t_16 = __pyx_PyFloat_AsFloat(__pyx_v_discount); if (unlikely((__pyx_t_16 == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 115, __pyx_L1_error) + __pyx_t_14 = __Pyx_PyInt_AddObjC(__pyx_v_i, __pyx_int_1, 1, 0); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 115, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_13 = __Pyx_PyIndex_AsSsize_t(__pyx_t_14); if (unlikely((__pyx_t_13 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 115, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + (__pyx_v_pipeline.discount.fallback.amount[__pyx_t_13]) = __pyx_t_16; + + /* "kenlm.pyx":106 + * pipeline.discount.fallback.amount[0] = 0.0 + * + * for i in range(3): # <<<<<<<<<<<<<< + * discount = discount_fallback[len(discount_fallback) - 1] + * if i < len(discount_fallback): + */ + } + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + + /* "kenlm.pyx":117 + * pipeline.discount.fallback.amount[i+1] = discount + * + * pipeline.discount.bad_action = _kenlm.COMPLAIN # <<<<<<<<<<<<<< + * + * if len(pruning) > 0: + */ + __pyx_v_pipeline.discount.bad_action = lm::COMPLAIN; } + __pyx_L11:; - /* "kenlm.pyx":180 - * if eos: - * total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state) - * return total # <<<<<<<<<<<<<< + /* "kenlm.pyx":119 + * pipeline.discount.bad_action = _kenlm.COMPLAIN * - * def perplexity(self, sentence): + * if len(pruning) > 0: # <<<<<<<<<<<<<< + * + * pipeline.prune_thresholds.reserve(len(pruning)) */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyFloat_FromDouble(__pyx_v_total); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 180, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; + __pyx_t_10 = PyObject_Length(__pyx_v_pruning); if (unlikely(__pyx_t_10 == ((Py_ssize_t)-1))) __PYX_ERR(0, 119, __pyx_L1_error) + __pyx_t_4 = ((__pyx_t_10 > 0) != 0); + if (__pyx_t_4) { - /* "kenlm.pyx":128 - * return self.model.Order() + /* "kenlm.pyx":121 + * if len(pruning) > 0: * - * def score(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<< - * """ - * Return the log10 probability of a string. By default, the string is + * pipeline.prune_thresholds.reserve(len(pruning)) # <<<<<<<<<<<<<< + * + * for e in pruning: */ + __pyx_t_10 = PyObject_Length(__pyx_v_pruning); if (unlikely(__pyx_t_10 == ((Py_ssize_t)-1))) __PYX_ERR(0, 121, __pyx_L1_error) + __pyx_v_pipeline.prune_thresholds.reserve(__pyx_t_10); - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("kenlm.Model.score", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_words); - __Pyx_XDECREF(__pyx_v_word); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} + /* "kenlm.pyx":123 + * pipeline.prune_thresholds.reserve(len(pruning)) + * + * for e in pruning: # <<<<<<<<<<<<<< + * pipeline.prune_thresholds.push_back(int(e)) + * + */ + if (likely(PyList_CheckExact(__pyx_v_pruning)) || PyTuple_CheckExact(__pyx_v_pruning)) { + __pyx_t_8 = __pyx_v_pruning; __Pyx_INCREF(__pyx_t_8); __pyx_t_10 = 0; + __pyx_t_11 = NULL; + } else { + __pyx_t_10 = -1; __pyx_t_8 = PyObject_GetIter(__pyx_v_pruning); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 123, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_11 = Py_TYPE(__pyx_t_8)->tp_iternext; if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 123, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_11)) { + if (likely(PyList_CheckExact(__pyx_t_8))) { + if (__pyx_t_10 >= PyList_GET_SIZE(__pyx_t_8)) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_14 = PyList_GET_ITEM(__pyx_t_8, __pyx_t_10); __Pyx_INCREF(__pyx_t_14); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 123, __pyx_L1_error) + #else + __pyx_t_14 = PySequence_ITEM(__pyx_t_8, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 123, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #endif + } else { + if (__pyx_t_10 >= PyTuple_GET_SIZE(__pyx_t_8)) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_14 = PyTuple_GET_ITEM(__pyx_t_8, __pyx_t_10); __Pyx_INCREF(__pyx_t_14); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 123, __pyx_L1_error) + #else + __pyx_t_14 = PySequence_ITEM(__pyx_t_8, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 123, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #endif + } + } else { + __pyx_t_14 = __pyx_t_11(__pyx_t_8); + if (unlikely(!__pyx_t_14)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); + else __PYX_ERR(0, 123, __pyx_L1_error) + } + break; + } + __Pyx_GOTREF(__pyx_t_14); + } + __Pyx_XDECREF_SET(__pyx_v_e, __pyx_t_14); + __pyx_t_14 = 0; -/* "kenlm.pyx":182 - * return total + /* "kenlm.pyx":124 * - * def perplexity(self, sentence): # <<<<<<<<<<<<<< - * """ - * Compute perplexity of a sentence. + * for e in pruning: + * pipeline.prune_thresholds.push_back(int(e)) # <<<<<<<<<<<<<< + * + * if len(pruning) > order: */ + __pyx_t_14 = __Pyx_PyNumber_Int(__pyx_v_e); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 124, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_6 = __Pyx_PyInt_As_uint64_t(__pyx_t_14); if (unlikely((__pyx_t_6 == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 124, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + try { + __pyx_v_pipeline.prune_thresholds.push_back(__pyx_t_6); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 124, __pyx_L1_error) + } -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_7perplexity(PyObject *__pyx_v_self, PyObject *__pyx_v_sentence); /*proto*/ -static char __pyx_doc_5kenlm_5Model_6perplexity[] = "\n Compute perplexity of a sentence.\n @param sentence One full sentence to score. Do not include or .\n "; -static PyObject *__pyx_pw_5kenlm_5Model_7perplexity(PyObject *__pyx_v_self, PyObject *__pyx_v_sentence) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("perplexity (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5Model_6perplexity(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), ((PyObject *)__pyx_v_sentence)); + /* "kenlm.pyx":123 + * pipeline.prune_thresholds.reserve(len(pruning)) + * + * for e in pruning: # <<<<<<<<<<<<<< + * pipeline.prune_thresholds.push_back(int(e)) + * + */ + } + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} + /* "kenlm.pyx":126 + * pipeline.prune_thresholds.push_back(int(e)) + * + * if len(pruning) > order: # <<<<<<<<<<<<<< + * raise RuntimeError( + * "You specified pruning thresholds for orders 1 through " + len(pruning) + + */ + __pyx_t_10 = PyObject_Length(__pyx_v_pruning); if (unlikely(__pyx_t_10 == ((Py_ssize_t)-1))) __PYX_ERR(0, 126, __pyx_L1_error) + __pyx_t_8 = PyInt_FromSsize_t(__pyx_t_10); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 126, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_14 = PyObject_RichCompare(__pyx_t_8, __pyx_v_order, Py_GT); __Pyx_XGOTREF(__pyx_t_14); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 126, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_14); if (unlikely(__pyx_t_4 < 0)) __PYX_ERR(0, 126, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + if (__pyx_t_4) { + + /* "kenlm.pyx":128 + * if len(pruning) > order: + * raise RuntimeError( + * "You specified pruning thresholds for orders 1 through " + len(pruning) + # <<<<<<<<<<<<<< + * " but the model only has order " + order + * ) + */ + __pyx_t_10 = PyObject_Length(__pyx_v_pruning); if (unlikely(__pyx_t_10 == ((Py_ssize_t)-1))) __PYX_ERR(0, 128, __pyx_L1_error) + __pyx_t_14 = PyInt_FromSsize_t(__pyx_t_10); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 128, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_8 = PyNumber_Add(__pyx_kp_s_You_specified_pruning_thresholds, __pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 128, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = PyNumber_Add(__pyx_t_8, __pyx_kp_s_but_the_model_only_has_order); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 128, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + + /* "kenlm.pyx":129 + * raise RuntimeError( + * "You specified pruning thresholds for orders 1 through " + len(pruning) + + * " but the model only has order " + order # <<<<<<<<<<<<<< + * ) + * + */ + __pyx_t_8 = PyNumber_Add(__pyx_t_14, __pyx_v_order); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 129, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "kenlm.pyx":127 + * + * if len(pruning) > order: + * raise RuntimeError( # <<<<<<<<<<<<<< + * "You specified pruning thresholds for orders 1 through " + len(pruning) + + * " but the model only has order " + order + */ + __pyx_t_14 = PyTuple_New(1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 127, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_8); + PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_8); + __pyx_t_8 = 0; + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_builtin_RuntimeError, __pyx_t_14, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 127, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_Raise(__pyx_t_8, 0, 0, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __PYX_ERR(0, 127, __pyx_L1_error) + + /* "kenlm.pyx":126 + * pipeline.prune_thresholds.push_back(int(e)) + * + * if len(pruning) > order: # <<<<<<<<<<<<<< + * raise RuntimeError( + * "You specified pruning thresholds for orders 1 through " + len(pruning) + + */ + } -static PyObject *__pyx_pf_5kenlm_5Model_6perplexity(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_sentence) { - PyObject *__pyx_v_words = NULL; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - Py_ssize_t __pyx_t_4; - PyObject *__pyx_t_5 = NULL; - __Pyx_RefNannySetupContext("perplexity", 0); + /* "kenlm.pyx":132 + * ) + * + * pipeline.prune_thresholds.resize(order, pipeline.prune_thresholds.back()); # <<<<<<<<<<<<<< + * + * else: + */ + __pyx_t_17 = __Pyx_PyInt_As_size_t(__pyx_v_order); if (unlikely((__pyx_t_17 == (size_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 132, __pyx_L1_error) + try { + __pyx_v_pipeline.prune_thresholds.resize(__pyx_t_17, __pyx_v_pipeline.prune_thresholds.back()); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 132, __pyx_L1_error) + } - /* "kenlm.pyx":187 - * @param sentence One full sentence to score. Do not include or . - * """ - * words = len(as_str(sentence).split()) + 1 # For # <<<<<<<<<<<<<< - * return 10.0**(-self.score(sentence) / words) + /* "kenlm.pyx":119 + * pipeline.discount.bad_action = _kenlm.COMPLAIN + * + * if len(pruning) > 0: # <<<<<<<<<<<<<< * + * pipeline.prune_thresholds.reserve(len(pruning)) */ - __pyx_t_2 = __pyx_f_5kenlm_as_str(__pyx_v_sentence); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 187, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_split); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 187, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = NULL; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { - __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); - if (likely(__pyx_t_2)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); - __Pyx_INCREF(__pyx_t_2); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_3, function); + goto __pyx_L20; + } + + /* "kenlm.pyx":135 + * + * else: + * pipeline.prune_thresholds.resize(order, 0) # <<<<<<<<<<<<<< + * + * if len(limit_vocab_file) == 0: + */ + /*else*/ { + __pyx_t_17 = __Pyx_PyInt_As_size_t(__pyx_v_order); if (unlikely((__pyx_t_17 == (size_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 135, __pyx_L1_error) + try { + __pyx_v_pipeline.prune_thresholds.resize(__pyx_t_17, 0); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 135, __pyx_L1_error) } } - if (__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 187, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 187, __pyx_L1_error) + __pyx_L20:; + + /* "kenlm.pyx":137 + * pipeline.prune_thresholds.resize(order, 0) + * + * if len(limit_vocab_file) == 0: # <<<<<<<<<<<<<< + * pipeline.prune_vocab = True + * + */ + __pyx_t_10 = PyObject_Length(__pyx_v_limit_vocab_file); if (unlikely(__pyx_t_10 == ((Py_ssize_t)-1))) __PYX_ERR(0, 137, __pyx_L1_error) + __pyx_t_4 = ((__pyx_t_10 == 0) != 0); + if (__pyx_t_4) { + + /* "kenlm.pyx":138 + * + * if len(limit_vocab_file) == 0: + * pipeline.prune_vocab = True # <<<<<<<<<<<<<< + * + * else: + */ + __pyx_v_pipeline.prune_vocab = 1; + + /* "kenlm.pyx":137 + * pipeline.prune_thresholds.resize(order, 0) + * + * if len(limit_vocab_file) == 0: # <<<<<<<<<<<<<< + * pipeline.prune_vocab = True + * + */ + goto __pyx_L24; } - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_4 = PyObject_Length(__pyx_t_1); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(0, 187, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyInt_FromSsize_t((__pyx_t_4 + 1)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 187, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_v_words = __pyx_t_1; - __pyx_t_1 = 0; - /* "kenlm.pyx":188 - * """ - * words = len(as_str(sentence).split()) + 1 # For - * return 10.0**(-self.score(sentence) / words) # <<<<<<<<<<<<<< + /* "kenlm.pyx":141 + * + * else: + * pipeline.prune_vocab = False # <<<<<<<<<<<<<< + * pipeline.prune_vocab_file = limit_vocab_file * - * def full_scores(self, sentence, bos = True, eos = True): */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_score); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 188, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_2 = NULL; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { - __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); - if (likely(__pyx_t_2)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); - __Pyx_INCREF(__pyx_t_2); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_3, function); - } + /*else*/ { + __pyx_v_pipeline.prune_vocab = 0; } - if (!__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_v_sentence); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 188, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - } else { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_3)) { - PyObject *__pyx_temp[2] = {__pyx_t_2, __pyx_v_sentence}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 188, __pyx_L1_error) - __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_GOTREF(__pyx_t_1); - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_3)) { - PyObject *__pyx_temp[2] = {__pyx_t_2, __pyx_v_sentence}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 188, __pyx_L1_error) - __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_GOTREF(__pyx_t_1); - } else - #endif - { - __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 188, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_2); __pyx_t_2 = NULL; - __Pyx_INCREF(__pyx_v_sentence); - __Pyx_GIVEREF(__pyx_v_sentence); - PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_v_sentence); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 188, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - } + __pyx_L24:; + + /* "kenlm.pyx":142 + * else: + * pipeline.prune_vocab = False + * pipeline.prune_vocab_file = limit_vocab_file # <<<<<<<<<<<<<< + * + * _kenlm.NormalizeTempPrefix(pipeline.sort.temp_prefix) + */ + __pyx_t_5 = __pyx_convert_string_from_py_std__in_string(__pyx_v_limit_vocab_file); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 142, __pyx_L1_error) + __pyx_v_pipeline.prune_vocab_file = __pyx_t_5; + + /* "kenlm.pyx":144 + * pipeline.prune_vocab_file = limit_vocab_file + * + * _kenlm.NormalizeTempPrefix(pipeline.sort.temp_prefix) # <<<<<<<<<<<<<< + * + * pipeline.initial_probs.adder_in.total_memory = 32768; + */ + try { + util::NormalizeTempPrefix(__pyx_v_pipeline.sort.temp_prefix); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 144, __pyx_L1_error) } - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyNumber_Negative(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 188, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyNumber_Divide(__pyx_t_3, __pyx_v_words); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 188, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyNumber_Power(__pyx_float_10_0, __pyx_t_1, Py_None); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 188, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_r = __pyx_t_3; - __pyx_t_3 = 0; - goto __pyx_L0; - /* "kenlm.pyx":182 - * return total + /* "kenlm.pyx":146 + * _kenlm.NormalizeTempPrefix(pipeline.sort.temp_prefix) * - * def perplexity(self, sentence): # <<<<<<<<<<<<<< - * """ - * Compute perplexity of a sentence. + * pipeline.initial_probs.adder_in.total_memory = 32768; # <<<<<<<<<<<<<< + * pipeline.initial_probs.adder_in.block_count = 2; + * pipeline.initial_probs.adder_out.total_memory = 32768; */ + __pyx_v_pipeline.initial_probs.adder_in.total_memory = 0x8000; - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_5); - __Pyx_AddTraceback("kenlm.Model.perplexity", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_words); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} -static PyObject *__pyx_gb_5kenlm_5Model_10generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ + /* "kenlm.pyx":147 + * + * pipeline.initial_probs.adder_in.total_memory = 32768; + * pipeline.initial_probs.adder_in.block_count = 2; # <<<<<<<<<<<<<< + * pipeline.initial_probs.adder_out.total_memory = 32768; + * pipeline.initial_probs.adder_out.block_count = 2; + */ + __pyx_v_pipeline.initial_probs.adder_in.block_count = 2; -/* "kenlm.pyx":190 - * return 10.0**(-self.score(sentence) / words) + /* "kenlm.pyx":148 + * pipeline.initial_probs.adder_in.total_memory = 32768; + * pipeline.initial_probs.adder_in.block_count = 2; + * pipeline.initial_probs.adder_out.total_memory = 32768; # <<<<<<<<<<<<<< + * pipeline.initial_probs.adder_out.block_count = 2; + * pipeline.read_backoffs.total_memory = 32768; + */ + __pyx_v_pipeline.initial_probs.adder_out.total_memory = 0x8000; + + /* "kenlm.pyx":149 + * pipeline.initial_probs.adder_in.block_count = 2; + * pipeline.initial_probs.adder_out.total_memory = 32768; + * pipeline.initial_probs.adder_out.block_count = 2; # <<<<<<<<<<<<<< + * pipeline.read_backoffs.total_memory = 32768; + * pipeline.read_backoffs.block_count = 2; + */ + __pyx_v_pipeline.initial_probs.adder_out.block_count = 2; + + /* "kenlm.pyx":150 + * pipeline.initial_probs.adder_out.total_memory = 32768; + * pipeline.initial_probs.adder_out.block_count = 2; + * pipeline.read_backoffs.total_memory = 32768; # <<<<<<<<<<<<<< + * pipeline.read_backoffs.block_count = 2; * - * def full_scores(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<< - * """ - * full_scores(sentence, bos = True, eos = Ture) -> generate full scores (prob, ngram length, oov) */ + __pyx_v_pipeline.read_backoffs.total_memory = 0x8000; -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_9full_scores(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_5kenlm_5Model_8full_scores[] = "\n full_scores(sentence, bos = True, eos = Ture) -> generate full scores (prob, ngram length, oov)\n @param sentence is a string (do not use boundary symbols)\n @param bos should kenlm add a bos state\n @param eos should kenlm add an eos state\n "; -static PyObject *__pyx_pw_5kenlm_5Model_9full_scores(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_sentence = 0; - PyObject *__pyx_v_bos = 0; - PyObject *__pyx_v_eos = 0; - PyObject *__pyx_r = 0; + /* "kenlm.pyx":151 + * pipeline.initial_probs.adder_out.block_count = 2; + * pipeline.read_backoffs.total_memory = 32768; + * pipeline.read_backoffs.block_count = 2; # <<<<<<<<<<<<<< + * + * cdef _kenlm.scoped_fd _in + */ + __pyx_v_pipeline.read_backoffs.block_count = 2; + + /* "kenlm.pyx":155 + * cdef _kenlm.scoped_fd _in + * cdef _kenlm.scoped_fd _out + * _in.reset(_kenlm.OpenReadOrThrow(path_text_file)) # <<<<<<<<<<<<<< + * _out.reset(_kenlm.CreateOrThrow(path_arpa_file)) + * + */ + __pyx_t_18 = __Pyx_PyObject_AsWritableString(__pyx_v_path_text_file); if (unlikely((!__pyx_t_18) && PyErr_Occurred())) __PYX_ERR(0, 155, __pyx_L1_error) + try { + __pyx_t_19 = util::OpenReadOrThrow(__pyx_t_18); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 155, __pyx_L1_error) + } + try { + __pyx_v__in.reset(__pyx_t_19); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 155, __pyx_L1_error) + } + + /* "kenlm.pyx":156 + * cdef _kenlm.scoped_fd _out + * _in.reset(_kenlm.OpenReadOrThrow(path_text_file)) + * _out.reset(_kenlm.CreateOrThrow(path_arpa_file)) # <<<<<<<<<<<<<< + * + * if intermediate is None: + */ + __pyx_t_18 = __Pyx_PyObject_AsWritableString(__pyx_v_path_arpa_file); if (unlikely((!__pyx_t_18) && PyErr_Occurred())) __PYX_ERR(0, 156, __pyx_L1_error) + try { + __pyx_t_19 = util::CreateOrThrow(__pyx_t_18); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 156, __pyx_L1_error) + } + try { + __pyx_v__out.reset(__pyx_t_19); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 156, __pyx_L1_error) + } + + /* "kenlm.pyx":158 + * _out.reset(_kenlm.CreateOrThrow(path_arpa_file)) + * + * if intermediate is None: # <<<<<<<<<<<<<< + * pipeline.renumber_vocabulary = False + * output = Output(pipeline.sort.temp_prefix, False, False) + */ + __pyx_t_4 = (__pyx_v_intermediate == Py_None); + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "kenlm.pyx":159 + * + * if intermediate is None: + * pipeline.renumber_vocabulary = False # <<<<<<<<<<<<<< + * output = Output(pipeline.sort.temp_prefix, False, False) + * else: + */ + __pyx_v_pipeline.renumber_vocabulary = 0; + + /* "kenlm.pyx":160 + * if intermediate is None: + * pipeline.renumber_vocabulary = False + * output = Output(pipeline.sort.temp_prefix, False, False) # <<<<<<<<<<<<<< + * else: + * pipeline.renumber_vocabulary = True + */ + __pyx_t_8 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_pipeline.sort.temp_prefix); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 160, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_14 = PyTuple_New(3); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 160, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_8); + PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_8); + __Pyx_INCREF(Py_False); + __Pyx_GIVEREF(Py_False); + PyTuple_SET_ITEM(__pyx_t_14, 1, Py_False); + __Pyx_INCREF(Py_False); + __Pyx_GIVEREF(Py_False); + PyTuple_SET_ITEM(__pyx_t_14, 2, Py_False); + __pyx_t_8 = 0; + __pyx_t_8 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5kenlm_Output), __pyx_t_14, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 160, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_v_output = ((struct __pyx_obj_5kenlm_Output *)__pyx_t_8); + __pyx_t_8 = 0; + + /* "kenlm.pyx":158 + * _out.reset(_kenlm.CreateOrThrow(path_arpa_file)) + * + * if intermediate is None: # <<<<<<<<<<<<<< + * pipeline.renumber_vocabulary = False + * output = Output(pipeline.sort.temp_prefix, False, False) + */ + goto __pyx_L25; + } + + /* "kenlm.pyx":162 + * output = Output(pipeline.sort.temp_prefix, False, False) + * else: + * pipeline.renumber_vocabulary = True # <<<<<<<<<<<<<< + * output = Output(intermediate, False, False) + * + */ + /*else*/ { + __pyx_v_pipeline.renumber_vocabulary = 1; + + /* "kenlm.pyx":163 + * else: + * pipeline.renumber_vocabulary = True + * output = Output(intermediate, False, False) # <<<<<<<<<<<<<< + * + * output.Add(_out.release(), verbose_header) + */ + __pyx_t_8 = PyTuple_New(3); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 163, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_INCREF(__pyx_v_intermediate); + __Pyx_GIVEREF(__pyx_v_intermediate); + PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_v_intermediate); + __Pyx_INCREF(Py_False); + __Pyx_GIVEREF(Py_False); + PyTuple_SET_ITEM(__pyx_t_8, 1, Py_False); + __Pyx_INCREF(Py_False); + __Pyx_GIVEREF(Py_False); + PyTuple_SET_ITEM(__pyx_t_8, 2, Py_False); + __pyx_t_14 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5kenlm_Output), __pyx_t_8, NULL); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 163, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_output = ((struct __pyx_obj_5kenlm_Output *)__pyx_t_14); + __pyx_t_14 = 0; + } + __pyx_L25:; + + /* "kenlm.pyx":165 + * output = Output(intermediate, False, False) + * + * output.Add(_out.release(), verbose_header) # <<<<<<<<<<<<<< + * + * Pipeline(pipeline, _in.release(), output) + */ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_output), __pyx_n_s_Add); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + try { + __pyx_t_19 = __pyx_v__out.release(); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 165, __pyx_L1_error) + } + __pyx_t_15 = __Pyx_PyInt_From_int(__pyx_t_19); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __pyx_t_9 = NULL; + __pyx_t_19 = 0; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_8))) { + __pyx_t_9 = PyMethod_GET_SELF(__pyx_t_8); + if (likely(__pyx_t_9)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_8); + __Pyx_INCREF(__pyx_t_9); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_8, function); + __pyx_t_19 = 1; + } + } + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_8)) { + PyObject *__pyx_temp[3] = {__pyx_t_9, __pyx_t_15, __pyx_v_verbose_header}; + __pyx_t_14 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_19, 2+__pyx_t_19); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_8)) { + PyObject *__pyx_temp[3] = {__pyx_t_9, __pyx_t_15, __pyx_v_verbose_header}; + __pyx_t_14 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_19, 2+__pyx_t_19); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + } else + #endif + { + __pyx_t_20 = PyTuple_New(2+__pyx_t_19); if (unlikely(!__pyx_t_20)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_20); + if (__pyx_t_9) { + __Pyx_GIVEREF(__pyx_t_9); PyTuple_SET_ITEM(__pyx_t_20, 0, __pyx_t_9); __pyx_t_9 = NULL; + } + __Pyx_GIVEREF(__pyx_t_15); + PyTuple_SET_ITEM(__pyx_t_20, 0+__pyx_t_19, __pyx_t_15); + __Pyx_INCREF(__pyx_v_verbose_header); + __Pyx_GIVEREF(__pyx_v_verbose_header); + PyTuple_SET_ITEM(__pyx_t_20, 1+__pyx_t_19, __pyx_v_verbose_header); + __pyx_t_15 = 0; + __pyx_t_14 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_20, NULL); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_20); __pyx_t_20 = 0; + } + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "kenlm.pyx":167 + * output.Add(_out.release(), verbose_header) + * + * Pipeline(pipeline, _in.release(), output) # <<<<<<<<<<<<<< + * + * + */ + try { + __pyx_t_19 = __pyx_v__in.release(); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 167, __pyx_L1_error) + } + __pyx_t_14 = __Pyx_PyInt_From_int(__pyx_t_19); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 167, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_8 = __pyx_f_5kenlm_Pipeline(__pyx_v_pipeline, __pyx_t_14, __pyx_v_output); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 167, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + + /* "kenlm.pyx":42 + * + * + * def lmplz( # <<<<<<<<<<<<<< + * path_text_file, path_arpa_file, + * order=3, + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_XDECREF(__pyx_t_14); + __Pyx_XDECREF(__pyx_t_15); + __Pyx_XDECREF(__pyx_t_20); + __Pyx_AddTraceback("kenlm.lmplz", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_i); + __Pyx_XDECREF(__pyx_v_discount); + __Pyx_XDECREF(__pyx_v_e); + __Pyx_XDECREF((PyObject *)__pyx_v_output); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "kenlm.pyx":183 + * cdef bint oov + * + * def __cinit__(self, log_prob, ngram_length, oov): # <<<<<<<<<<<<<< + * self.log_prob = log_prob + * self.ngram_length = ngram_length + */ + +/* Python wrapper */ +static int __pyx_pw_5kenlm_15FullScoreReturn_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5kenlm_15FullScoreReturn_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_log_prob = 0; + PyObject *__pyx_v_ngram_length = 0; + PyObject *__pyx_v_oov = 0; + int __pyx_r; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("full_scores (wrapper)", 0); + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_sentence,&__pyx_n_s_bos,&__pyx_n_s_eos,0}; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_log_prob,&__pyx_n_s_ngram_length,&__pyx_n_s_oov,0}; PyObject* values[3] = {0,0,0}; - values[1] = ((PyObject *)Py_True); - values[2] = ((PyObject *)Py_True); if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_sentence)) != 0)) kw_args--; + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_log_prob)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; case 1: - if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_bos); - if (value) { values[1] = value; kw_args--; } + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_ngram_length)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, 1); __PYX_ERR(0, 183, __pyx_L3_error) } + CYTHON_FALLTHROUGH; case 2: - if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_eos); - if (value) { values[2] = value; kw_args--; } + if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_oov)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, 2); __PYX_ERR(0, 183, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "full_scores") < 0)) __PYX_ERR(0, 190, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(0, 183, __pyx_L3_error) } + } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { + goto __pyx_L5_argtuple_error; } else { - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - break; - default: goto __pyx_L5_argtuple_error; - } + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); } - __pyx_v_sentence = values[0]; - __pyx_v_bos = values[1]; - __pyx_v_eos = values[2]; + __pyx_v_log_prob = values[0]; + __pyx_v_ngram_length = values[1]; + __pyx_v_oov = values[2]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("full_scores", 0, 1, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 190, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 183, __pyx_L3_error) __pyx_L3_error:; - __Pyx_AddTraceback("kenlm.Model.full_scores", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("kenlm.FullScoreReturn.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); - return NULL; + return -1; __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_5kenlm_5Model_8full_scores(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), __pyx_v_sentence, __pyx_v_bos, __pyx_v_eos); + __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn___cinit__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self), __pyx_v_log_prob, __pyx_v_ngram_length, __pyx_v_oov); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5kenlm_5Model_8full_scores(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_sentence, PyObject *__pyx_v_bos, PyObject *__pyx_v_eos) { - struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *__pyx_cur_scope; - PyObject *__pyx_r = NULL; +static int __pyx_pf_5kenlm_15FullScoreReturn___cinit__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self, PyObject *__pyx_v_log_prob, PyObject *__pyx_v_ngram_length, PyObject *__pyx_v_oov) { + int __pyx_r; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("full_scores", 0); - __pyx_cur_scope = (struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)__pyx_tp_new_5kenlm___pyx_scope_struct__full_scores(__pyx_ptype_5kenlm___pyx_scope_struct__full_scores, __pyx_empty_tuple, NULL); - if (unlikely(!__pyx_cur_scope)) { - __pyx_cur_scope = ((struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)Py_None); - __Pyx_INCREF(Py_None); - __PYX_ERR(0, 190, __pyx_L1_error) - } else { - __Pyx_GOTREF(__pyx_cur_scope); - } - __pyx_cur_scope->__pyx_v_self = __pyx_v_self; - __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - __pyx_cur_scope->__pyx_v_sentence = __pyx_v_sentence; - __Pyx_INCREF(__pyx_cur_scope->__pyx_v_sentence); - __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_sentence); - __pyx_cur_scope->__pyx_v_bos = __pyx_v_bos; - __Pyx_INCREF(__pyx_cur_scope->__pyx_v_bos); - __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_bos); - __pyx_cur_scope->__pyx_v_eos = __pyx_v_eos; - __Pyx_INCREF(__pyx_cur_scope->__pyx_v_eos); - __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_eos); - { - __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5kenlm_5Model_10generator, (PyObject *) __pyx_cur_scope, __pyx_n_s_full_scores, __pyx_n_s_Model_full_scores, __pyx_n_s_kenlm); if (unlikely(!gen)) __PYX_ERR(0, 190, __pyx_L1_error) - __Pyx_DECREF(__pyx_cur_scope); - __Pyx_RefNannyFinishContext(); - return (PyObject *) gen; - } + float __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + __Pyx_RefNannySetupContext("__cinit__", 0); + + /* "kenlm.pyx":184 + * + * def __cinit__(self, log_prob, ngram_length, oov): + * self.log_prob = log_prob # <<<<<<<<<<<<<< + * self.ngram_length = ngram_length + * self.oov = oov + */ + __pyx_t_1 = __pyx_PyFloat_AsFloat(__pyx_v_log_prob); if (unlikely((__pyx_t_1 == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 184, __pyx_L1_error) + __pyx_v_self->log_prob = __pyx_t_1; + + /* "kenlm.pyx":185 + * def __cinit__(self, log_prob, ngram_length, oov): + * self.log_prob = log_prob + * self.ngram_length = ngram_length # <<<<<<<<<<<<<< + * self.oov = oov + * + */ + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_ngram_length); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 185, __pyx_L1_error) + __pyx_v_self->ngram_length = __pyx_t_2; + + /* "kenlm.pyx":186 + * self.log_prob = log_prob + * self.ngram_length = ngram_length + * self.oov = oov # <<<<<<<<<<<<<< + * + * def __repr__(self): + */ + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v_oov); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 186, __pyx_L1_error) + __pyx_v_self->oov = __pyx_t_3; + + /* "kenlm.pyx":183 + * cdef bint oov + * + * def __cinit__(self, log_prob, ngram_length, oov): # <<<<<<<<<<<<<< + * self.log_prob = log_prob + * self.ngram_length = ngram_length + */ /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; __pyx_L1_error:; - __Pyx_AddTraceback("kenlm.Model.full_scores", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); - __Pyx_XGIVEREF(__pyx_r); + __Pyx_AddTraceback("kenlm.FullScoreReturn.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_gb_5kenlm_5Model_10generator(__pyx_CoroutineObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ -{ - struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *__pyx_cur_scope = ((struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)__pyx_generator->closure); +/* "kenlm.pyx":188 + * self.oov = oov + * + * def __repr__(self): # <<<<<<<<<<<<<< + * return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov)) + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3__repr__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3__repr__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_2__repr__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_2__repr__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) { PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; - int __pyx_t_4; - Py_ssize_t __pyx_t_5; - char *__pyx_t_6; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; PyObject *__pyx_t_7 = NULL; - PyObject *__pyx_t_8 = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("None", 0); - switch (__pyx_generator->resume_label) { - case 0: goto __pyx_L3_first_run; - case 1: goto __pyx_L7_resume_from_yield; - case 2: goto __pyx_L9_resume_from_yield; - default: /* CPython raises the right error here */ - __Pyx_RefNannyFinishContext(); - return NULL; - } - __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 190, __pyx_L1_error) + int __pyx_t_8; + PyObject *__pyx_t_9 = NULL; + __Pyx_RefNannySetupContext("__repr__", 0); - /* "kenlm.pyx":197 - * @param eos should kenlm add an eos state - * """ - * cdef list words = as_str(sentence).split() # <<<<<<<<<<<<<< - * cdef _kenlm.State state - * if bos: + /* "kenlm.pyx":189 + * + * def __repr__(self): + * return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov)) # <<<<<<<<<<<<<< + * + * property log_prob: */ - __pyx_t_2 = __pyx_f_5kenlm_as_str(__pyx_cur_scope->__pyx_v_sentence); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 197, __pyx_L1_error) + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_kp_s_0_1_2_3, __pyx_n_s_format); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 189, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_split); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 197, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_class); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 189, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = NULL; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { - __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); - if (likely(__pyx_t_2)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); - __Pyx_INCREF(__pyx_t_2); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_name); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 189, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = PyFloat_FromDouble(__pyx_v_self->log_prob); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 189, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = PyObject_Repr(__pyx_t_3); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 189, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_self->ngram_length); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 189, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = PyObject_Repr(__pyx_t_3); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 189, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_PyBool_FromLong(__pyx_v_self->oov); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 189, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_7 = PyObject_Repr(__pyx_t_3); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 189, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = NULL; + __pyx_t_8 = 0; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); + if (likely(__pyx_t_3)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_3); __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_3, function); + __Pyx_DECREF_SET(__pyx_t_2, function); + __pyx_t_8 = 1; } } - if (__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 197, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 197, __pyx_L1_error) + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[5] = {__pyx_t_3, __pyx_t_4, __pyx_t_5, __pyx_t_6, __pyx_t_7}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_8, 4+__pyx_t_8); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 189, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[5] = {__pyx_t_3, __pyx_t_4, __pyx_t_5, __pyx_t_6, __pyx_t_7}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_8, 4+__pyx_t_8); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 189, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + } else + #endif + { + __pyx_t_9 = PyTuple_New(4+__pyx_t_8); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 189, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + if (__pyx_t_3) { + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_3); __pyx_t_3 = NULL; + } + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_9, 0+__pyx_t_8, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_9, 1+__pyx_t_8, __pyx_t_5); + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_9, 2+__pyx_t_8, __pyx_t_6); + __Pyx_GIVEREF(__pyx_t_7); + PyTuple_SET_ITEM(__pyx_t_9, 3+__pyx_t_8, __pyx_t_7); + __pyx_t_4 = 0; + __pyx_t_5 = 0; + __pyx_t_6 = 0; + __pyx_t_7 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_9, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 189, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; } - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (!(likely(PyList_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "list", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 197, __pyx_L1_error) - __Pyx_GIVEREF(__pyx_t_1); - __pyx_cur_scope->__pyx_v_words = ((PyObject*)__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_1; __pyx_t_1 = 0; + goto __pyx_L0; - /* "kenlm.pyx":199 - * cdef list words = as_str(sentence).split() - * cdef _kenlm.State state - * if bos: # <<<<<<<<<<<<<< - * self.model.BeginSentenceWrite(&state) - * else: + /* "kenlm.pyx":188 + * self.oov = oov + * + * def __repr__(self): # <<<<<<<<<<<<<< + * return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov)) + * */ - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_cur_scope->__pyx_v_bos); if (unlikely(__pyx_t_4 < 0)) __PYX_ERR(0, 199, __pyx_L1_error) - if (__pyx_t_4) { - /* "kenlm.pyx":200 - * cdef _kenlm.State state - * if bos: - * self.model.BeginSentenceWrite(&state) # <<<<<<<<<<<<<< - * else: - * self.model.NullContextWrite(&state) - */ - __pyx_cur_scope->__pyx_v_self->model->BeginSentenceWrite((&__pyx_cur_scope->__pyx_v_state)); + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("kenlm.FullScoreReturn.__repr__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} - /* "kenlm.pyx":199 - * cdef list words = as_str(sentence).split() - * cdef _kenlm.State state - * if bos: # <<<<<<<<<<<<<< - * self.model.BeginSentenceWrite(&state) - * else: +/* "kenlm.pyx":192 + * + * property log_prob: + * def __get__(self): # <<<<<<<<<<<<<< + * return self.log_prob + * */ - goto __pyx_L4; - } - /* "kenlm.pyx":202 - * self.model.BeginSentenceWrite(&state) - * else: - * self.model.NullContextWrite(&state) # <<<<<<<<<<<<<< - * cdef _kenlm.State out_state - * cdef _kenlm.FullScoreReturn ret - */ - /*else*/ { - __pyx_cur_scope->__pyx_v_self->model->NullContextWrite((&__pyx_cur_scope->__pyx_v_state)); - } - __pyx_L4:; +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_8log_prob_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_8log_prob_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_8log_prob___get__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self)); - /* "kenlm.pyx":205 - * cdef _kenlm.State out_state - * cdef _kenlm.FullScoreReturn ret - * cdef float total = 0 # <<<<<<<<<<<<<< - * cdef _kenlm.WordIndex wid - * for word in words: - */ - __pyx_cur_scope->__pyx_v_total = 0.0; + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} - /* "kenlm.pyx":207 - * cdef float total = 0 - * cdef _kenlm.WordIndex wid - * for word in words: # <<<<<<<<<<<<<< - * wid = self.vocab.Index(word) - * ret = self.model.BaseFullScore(&state, wid, &out_state) +static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_8log_prob___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "kenlm.pyx":193 + * property log_prob: + * def __get__(self): + * return self.log_prob # <<<<<<<<<<<<<< + * + * property ngram_length: */ - if (unlikely(__pyx_cur_scope->__pyx_v_words == Py_None)) { - PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); - __PYX_ERR(0, 207, __pyx_L1_error) - } - __pyx_t_1 = __pyx_cur_scope->__pyx_v_words; __Pyx_INCREF(__pyx_t_1); __pyx_t_5 = 0; - for (;;) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_3 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_5); __Pyx_INCREF(__pyx_t_3); __pyx_t_5++; if (unlikely(0 < 0)) __PYX_ERR(0, 207, __pyx_L1_error) - #else - __pyx_t_3 = PySequence_ITEM(__pyx_t_1, __pyx_t_5); __pyx_t_5++; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 207, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - #endif - __Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_word); - __Pyx_XDECREF_SET(__pyx_cur_scope->__pyx_v_word, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_3); - __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->log_prob); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; - /* "kenlm.pyx":208 - * cdef _kenlm.WordIndex wid - * for word in words: - * wid = self.vocab.Index(word) # <<<<<<<<<<<<<< - * ret = self.model.BaseFullScore(&state, wid, &out_state) - * yield (ret.prob, ret.ngram_length, wid == 0) + /* "kenlm.pyx":192 + * + * property log_prob: + * def __get__(self): # <<<<<<<<<<<<<< + * return self.log_prob + * */ - __pyx_t_6 = __Pyx_PyObject_AsString(__pyx_cur_scope->__pyx_v_word); if (unlikely((!__pyx_t_6) && PyErr_Occurred())) __PYX_ERR(0, 208, __pyx_L1_error) - __pyx_cur_scope->__pyx_v_wid = __pyx_cur_scope->__pyx_v_self->vocab->Index(__pyx_t_6); - /* "kenlm.pyx":209 - * for word in words: - * wid = self.vocab.Index(word) - * ret = self.model.BaseFullScore(&state, wid, &out_state) # <<<<<<<<<<<<<< - * yield (ret.prob, ret.ngram_length, wid == 0) - * state = out_state + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("kenlm.FullScoreReturn.log_prob.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "kenlm.pyx":196 + * + * property ngram_length: + * def __get__(self): # <<<<<<<<<<<<<< + * return self.ngram_length + * */ - __pyx_cur_scope->__pyx_v_ret = __pyx_cur_scope->__pyx_v_self->model->BaseFullScore((&__pyx_cur_scope->__pyx_v_state), __pyx_cur_scope->__pyx_v_wid, (&__pyx_cur_scope->__pyx_v_out_state)); - /* "kenlm.pyx":210 - * wid = self.vocab.Index(word) - * ret = self.model.BaseFullScore(&state, wid, &out_state) - * yield (ret.prob, ret.ngram_length, wid == 0) # <<<<<<<<<<<<<< - * state = out_state - * if eos: - */ - __pyx_t_3 = PyFloat_FromDouble(__pyx_cur_scope->__pyx_v_ret.prob); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 210, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_2 = __Pyx_PyInt_From_unsigned_char(__pyx_cur_scope->__pyx_v_ret.ngram_length); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 210, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_PyBool_FromLong((__pyx_cur_scope->__pyx_v_wid == 0)); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 210, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_8 = PyTuple_New(3); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 210, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_GIVEREF(__pyx_t_3); - PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_2); - PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_2); - __Pyx_GIVEREF(__pyx_t_7); - PyTuple_SET_ITEM(__pyx_t_8, 2, __pyx_t_7); - __pyx_t_3 = 0; - __pyx_t_2 = 0; - __pyx_t_7 = 0; - __pyx_r = __pyx_t_8; - __pyx_t_8 = 0; - __Pyx_XGIVEREF(__pyx_t_1); - __pyx_cur_scope->__pyx_t_0 = __pyx_t_1; - __pyx_cur_scope->__pyx_t_1 = __pyx_t_5; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - /* return from generator, yielding value */ - __pyx_generator->resume_label = 1; - return __pyx_r; - __pyx_L7_resume_from_yield:; - __pyx_t_1 = __pyx_cur_scope->__pyx_t_0; - __pyx_cur_scope->__pyx_t_0 = 0; - __Pyx_XGOTREF(__pyx_t_1); - __pyx_t_5 = __pyx_cur_scope->__pyx_t_1; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 210, __pyx_L1_error) - - /* "kenlm.pyx":211 - * ret = self.model.BaseFullScore(&state, wid, &out_state) - * yield (ret.prob, ret.ngram_length, wid == 0) - * state = out_state # <<<<<<<<<<<<<< - * if eos: - * ret = self.model.BaseFullScore(&state, - */ - __pyx_cur_scope->__pyx_v_state = __pyx_cur_scope->__pyx_v_out_state; - - /* "kenlm.pyx":207 - * cdef float total = 0 - * cdef _kenlm.WordIndex wid - * for word in words: # <<<<<<<<<<<<<< - * wid = self.vocab.Index(word) - * ret = self.model.BaseFullScore(&state, wid, &out_state) - */ - } - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_12ngram_length_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_12ngram_length_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_12ngram_length___get__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self)); - /* "kenlm.pyx":212 - * yield (ret.prob, ret.ngram_length, wid == 0) - * state = out_state - * if eos: # <<<<<<<<<<<<<< - * ret = self.model.BaseFullScore(&state, - * self.vocab.EndSentence(), &out_state) - */ - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_cur_scope->__pyx_v_eos); if (unlikely(__pyx_t_4 < 0)) __PYX_ERR(0, 212, __pyx_L1_error) - if (__pyx_t_4) { + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} - /* "kenlm.pyx":213 - * state = out_state - * if eos: - * ret = self.model.BaseFullScore(&state, # <<<<<<<<<<<<<< - * self.vocab.EndSentence(), &out_state) - * yield (ret.prob, ret.ngram_length, False) - */ - __pyx_cur_scope->__pyx_v_ret = __pyx_cur_scope->__pyx_v_self->model->BaseFullScore((&__pyx_cur_scope->__pyx_v_state), __pyx_cur_scope->__pyx_v_self->vocab->EndSentence(), (&__pyx_cur_scope->__pyx_v_out_state)); +static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_12ngram_length___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); - /* "kenlm.pyx":215 - * ret = self.model.BaseFullScore(&state, - * self.vocab.EndSentence(), &out_state) - * yield (ret.prob, ret.ngram_length, False) # <<<<<<<<<<<<<< - * + /* "kenlm.pyx":197 + * property ngram_length: + * def __get__(self): + * return self.ngram_length # <<<<<<<<<<<<<< * + * property oov: */ - __pyx_t_1 = PyFloat_FromDouble(__pyx_cur_scope->__pyx_v_ret.prob); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 215, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_8 = __Pyx_PyInt_From_unsigned_char(__pyx_cur_scope->__pyx_v_ret.ngram_length); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 215, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_8); - __pyx_t_7 = PyTuple_New(3); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 215, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_1); - __Pyx_GIVEREF(__pyx_t_8); - PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_8); - __Pyx_INCREF(Py_False); - __Pyx_GIVEREF(Py_False); - PyTuple_SET_ITEM(__pyx_t_7, 2, Py_False); - __pyx_t_1 = 0; - __pyx_t_8 = 0; - __pyx_r = __pyx_t_7; - __pyx_t_7 = 0; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - /* return from generator, yielding value */ - __pyx_generator->resume_label = 2; - return __pyx_r; - __pyx_L9_resume_from_yield:; - if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 215, __pyx_L1_error) - - /* "kenlm.pyx":212 - * yield (ret.prob, ret.ngram_length, wid == 0) - * state = out_state - * if eos: # <<<<<<<<<<<<<< - * ret = self.model.BaseFullScore(&state, - * self.vocab.EndSentence(), &out_state) - */ - } - CYTHON_MAYBE_UNUSED_VAR(__pyx_cur_scope); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_self->ngram_length); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 197, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; - /* "kenlm.pyx":190 - * return 10.0**(-self.score(sentence) / words) + /* "kenlm.pyx":196 + * + * property ngram_length: + * def __get__(self): # <<<<<<<<<<<<<< + * return self.ngram_length * - * def full_scores(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<< - * """ - * full_scores(sentence, bos = True, eos = Ture) -> generate full scores (prob, ngram length, oov) */ /* function exit code */ - PyErr_SetNone(PyExc_StopIteration); - goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_XDECREF(__pyx_t_8); - __Pyx_AddTraceback("full_scores", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("kenlm.FullScoreReturn.ngram_length.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; __pyx_L0:; - __Pyx_XDECREF(__pyx_r); __pyx_r = 0; - __pyx_generator->resume_label = -1; - __Pyx_Coroutine_clear((PyObject*)__pyx_generator); + __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "kenlm.pyx":218 +/* "kenlm.pyx":200 * + * property oov: + * def __get__(self): # <<<<<<<<<<<<<< + * return self.oov * - * def BeginSentenceWrite(self, State state): # <<<<<<<<<<<<<< - * """Change the given state to a BOS state.""" - * self.model.BeginSentenceWrite(&state._c_state) */ /* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_12BeginSentenceWrite(PyObject *__pyx_v_self, PyObject *__pyx_v_state); /*proto*/ -static char __pyx_doc_5kenlm_5Model_11BeginSentenceWrite[] = "Change the given state to a BOS state."; -static PyObject *__pyx_pw_5kenlm_5Model_12BeginSentenceWrite(PyObject *__pyx_v_self, PyObject *__pyx_v_state) { +static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3oov_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3oov_1__get__(PyObject *__pyx_v_self) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("BeginSentenceWrite (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_state), __pyx_ptype_5kenlm_State, 1, "state", 0))) __PYX_ERR(0, 218, __pyx_L1_error) - __pyx_r = __pyx_pf_5kenlm_5Model_11BeginSentenceWrite(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), ((struct __pyx_obj_5kenlm_State *)__pyx_v_state)); + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_3oov___get__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self)); /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = NULL; - __pyx_L0:; __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5kenlm_5Model_11BeginSentenceWrite(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_state) { +static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_3oov___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("BeginSentenceWrite", 0); + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); - /* "kenlm.pyx":220 - * def BeginSentenceWrite(self, State state): - * """Change the given state to a BOS state.""" - * self.model.BeginSentenceWrite(&state._c_state) # <<<<<<<<<<<<<< + /* "kenlm.pyx":201 + * property oov: + * def __get__(self): + * return self.oov # <<<<<<<<<<<<<< * - * def NullContextWrite(self, State state): + * cdef class State: */ - __pyx_v_self->model->BeginSentenceWrite((&__pyx_v_state->_c_state)); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->oov); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 201, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; - /* "kenlm.pyx":218 + /* "kenlm.pyx":200 * + * property oov: + * def __get__(self): # <<<<<<<<<<<<<< + * return self.oov * - * def BeginSentenceWrite(self, State state): # <<<<<<<<<<<<<< - * """Change the given state to a BOS state.""" - * self.model.BeginSentenceWrite(&state._c_state) */ /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("kenlm.FullScoreReturn.oov.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "kenlm.pyx":222 - * self.model.BeginSentenceWrite(&state._c_state) - * - * def NullContextWrite(self, State state): # <<<<<<<<<<<<<< - * """Change the given state to a NULL state.""" - * self.model.NullContextWrite(&state._c_state) +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): */ /* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_14NullContextWrite(PyObject *__pyx_v_self, PyObject *__pyx_v_state); /*proto*/ -static char __pyx_doc_5kenlm_5Model_13NullContextWrite[] = "Change the given state to a NULL state."; -static PyObject *__pyx_pw_5kenlm_5Model_14NullContextWrite(PyObject *__pyx_v_self, PyObject *__pyx_v_state) { +static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_5__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_5__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("NullContextWrite (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_state), __pyx_ptype_5kenlm_State, 1, "state", 0))) __PYX_ERR(0, 222, __pyx_L1_error) - __pyx_r = __pyx_pf_5kenlm_5Model_13NullContextWrite(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), ((struct __pyx_obj_5kenlm_State *)__pyx_v_state)); + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_4__reduce_cython__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self)); /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = NULL; - __pyx_L0:; __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5kenlm_5Model_13NullContextWrite(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_state) { +static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_4__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("NullContextWrite", 0); + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); - /* "kenlm.pyx":224 - * def NullContextWrite(self, State state): - * """Change the given state to a NULL state.""" - * self.model.NullContextWrite(&state._c_state) # <<<<<<<<<<<<<< - * - * def BaseScore(self, State in_state, str word, State out_state): + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") */ - __pyx_v_self->model->NullContextWrite((&__pyx_v_state->_c_state)); + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__15, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 2, __pyx_L1_error) - /* "kenlm.pyx":222 - * self.model.BeginSentenceWrite(&state._c_state) - * - * def NullContextWrite(self, State state): # <<<<<<<<<<<<<< - * """Change the given state to a NULL state.""" - * self.model.NullContextWrite(&state._c_state) + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): */ /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("kenlm.FullScoreReturn.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "kenlm.pyx":226 - * self.model.NullContextWrite(&state._c_state) - * - * def BaseScore(self, State in_state, str word, State out_state): # <<<<<<<<<<<<<< - * """ - * Return p(word|in_state) and update the output state. +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") */ /* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_16BaseScore(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_5kenlm_5Model_15BaseScore[] = "\n Return p(word|in_state) and update the output state.\n Wrapper around model.BaseScore(in_state, Index(word), out_state)\n\n :param word: the suffix\n :param state: the context (defaults to NullContext)\n :returns: p(word|state)\n "; -static PyObject *__pyx_pw_5kenlm_5Model_16BaseScore(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - struct __pyx_obj_5kenlm_State *__pyx_v_in_state = 0; - PyObject *__pyx_v_word = 0; - struct __pyx_obj_5kenlm_State *__pyx_v_out_state = 0; +static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_7__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state); /*proto*/ +static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_7__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("BaseScore (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_in_state,&__pyx_n_s_word,&__pyx_n_s_out_state,0}; - PyObject* values[3] = {0,0,0}; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_in_state)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_word)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("BaseScore", 1, 3, 3, 1); __PYX_ERR(0, 226, __pyx_L3_error) - } - case 2: - if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_out_state)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("BaseScore", 1, 3, 3, 2); __PYX_ERR(0, 226, __pyx_L3_error) - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "BaseScore") < 0)) __PYX_ERR(0, 226, __pyx_L3_error) - } - } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { - goto __pyx_L5_argtuple_error; - } else { - values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - } - __pyx_v_in_state = ((struct __pyx_obj_5kenlm_State *)values[0]); - __pyx_v_word = ((PyObject*)values[1]); - __pyx_v_out_state = ((struct __pyx_obj_5kenlm_State *)values[2]); - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("BaseScore", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 226, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("kenlm.Model.BaseScore", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_in_state), __pyx_ptype_5kenlm_State, 1, "in_state", 0))) __PYX_ERR(0, 226, __pyx_L1_error) - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_word), (&PyString_Type), 1, "word", 1))) __PYX_ERR(0, 226, __pyx_L1_error) - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_out_state), __pyx_ptype_5kenlm_State, 1, "out_state", 0))) __PYX_ERR(0, 226, __pyx_L1_error) - __pyx_r = __pyx_pf_5kenlm_5Model_15BaseScore(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), __pyx_v_in_state, __pyx_v_word, __pyx_v_out_state); + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_6__setstate_cython__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self), ((PyObject *)__pyx_v___pyx_state)); /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = NULL; - __pyx_L0:; __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5kenlm_5Model_15BaseScore(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_in_state, PyObject *__pyx_v_word, struct __pyx_obj_5kenlm_State *__pyx_v_out_state) { - float __pyx_v_total; +static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_6__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; - char *__pyx_t_2; - __Pyx_RefNannySetupContext("BaseScore", 0); + __Pyx_RefNannySetupContext("__setstate_cython__", 0); - /* "kenlm.pyx":235 - * :returns: p(word|state) - * """ - * cdef float total = self.model.BaseScore(&in_state._c_state, self.vocab.Index(as_str(word)), &out_state._c_state) # <<<<<<<<<<<<<< - * return total - * + /* "(tree fragment)":4 + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< */ - __pyx_t_1 = __pyx_f_5kenlm_as_str(__pyx_v_word); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 235, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__16, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_AsString(__pyx_t_1); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) __PYX_ERR(0, 235, __pyx_L1_error) - __pyx_v_total = __pyx_v_self->model->BaseScore((&__pyx_v_in_state->_c_state), __pyx_v_self->vocab->Index(__pyx_t_2), (&__pyx_v_out_state->_c_state)); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 4, __pyx_L1_error) - /* "kenlm.pyx":236 - * """ - * cdef float total = self.model.BaseScore(&in_state._c_state, self.vocab.Index(as_str(word)), &out_state._c_state) - * return total # <<<<<<<<<<<<<< - * - * def BaseFullScore(self, State in_state, str word, State out_state): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyFloat_FromDouble(__pyx_v_total); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 236, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":226 - * self.model.NullContextWrite(&state._c_state) - * - * def BaseScore(self, State in_state, str word, State out_state): # <<<<<<<<<<<<<< - * """ - * Return p(word|in_state) and update the output state. + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.Model.BaseScore", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("kenlm.FullScoreReturn.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; - __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "kenlm.pyx":238 - * return total +/* "kenlm.pyx":214 + * cdef _kenlm.State _c_state * - * def BaseFullScore(self, State in_state, str word, State out_state): # <<<<<<<<<<<<<< - * """ - * Wrapper around model.BaseScore(in_state, Index(word), out_state) + * def __richcmp__(State qa, State qb, int op): # <<<<<<<<<<<<<< + * r = qa._c_state.Compare(qb._c_state) + * if op == 0: # < */ /* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_18BaseFullScore(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_5kenlm_5Model_17BaseFullScore[] = "\n Wrapper around model.BaseScore(in_state, Index(word), out_state)\n\n :param word: the suffix\n :param state: the context (defaults to NullContext)\n :returns: FullScoreReturn(word|state)\n "; -static PyObject *__pyx_pw_5kenlm_5Model_18BaseFullScore(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - struct __pyx_obj_5kenlm_State *__pyx_v_in_state = 0; - PyObject *__pyx_v_word = 0; - struct __pyx_obj_5kenlm_State *__pyx_v_out_state = 0; +static PyObject *__pyx_pw_5kenlm_5State_1__richcmp__(PyObject *__pyx_v_qa, PyObject *__pyx_v_qb, int __pyx_v_op); /*proto*/ +static PyObject *__pyx_pw_5kenlm_5State_1__richcmp__(PyObject *__pyx_v_qa, PyObject *__pyx_v_qb, int __pyx_v_op) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("BaseFullScore (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_in_state,&__pyx_n_s_word,&__pyx_n_s_out_state,0}; - PyObject* values[3] = {0,0,0}; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_in_state)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_word)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("BaseFullScore", 1, 3, 3, 1); __PYX_ERR(0, 238, __pyx_L3_error) - } - case 2: - if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_out_state)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("BaseFullScore", 1, 3, 3, 2); __PYX_ERR(0, 238, __pyx_L3_error) - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "BaseFullScore") < 0)) __PYX_ERR(0, 238, __pyx_L3_error) - } - } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { - goto __pyx_L5_argtuple_error; - } else { - values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - } - __pyx_v_in_state = ((struct __pyx_obj_5kenlm_State *)values[0]); - __pyx_v_word = ((PyObject*)values[1]); - __pyx_v_out_state = ((struct __pyx_obj_5kenlm_State *)values[2]); - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("BaseFullScore", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 238, __pyx_L3_error) - __pyx_L3_error:; - __Pyx_AddTraceback("kenlm.Model.BaseFullScore", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_in_state), __pyx_ptype_5kenlm_State, 1, "in_state", 0))) __PYX_ERR(0, 238, __pyx_L1_error) - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_word), (&PyString_Type), 1, "word", 1))) __PYX_ERR(0, 238, __pyx_L1_error) - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_out_state), __pyx_ptype_5kenlm_State, 1, "out_state", 0))) __PYX_ERR(0, 238, __pyx_L1_error) - __pyx_r = __pyx_pf_5kenlm_5Model_17BaseFullScore(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), __pyx_v_in_state, __pyx_v_word, __pyx_v_out_state); + __Pyx_RefNannySetupContext("__richcmp__ (wrapper)", 0); + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_qa), __pyx_ptype_5kenlm_State, 1, "qa", 0))) __PYX_ERR(0, 214, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_qb), __pyx_ptype_5kenlm_State, 1, "qb", 0))) __PYX_ERR(0, 214, __pyx_L1_error) + __pyx_r = __pyx_pf_5kenlm_5State___richcmp__(((struct __pyx_obj_5kenlm_State *)__pyx_v_qa), ((struct __pyx_obj_5kenlm_State *)__pyx_v_qb), ((int)__pyx_v_op)); /* function exit code */ goto __pyx_L0; @@ -4027,398 +4806,494 @@ static PyObject *__pyx_pw_5kenlm_5Model_18BaseFullScore(PyObject *__pyx_v_self, return __pyx_r; } -static PyObject *__pyx_pf_5kenlm_5Model_17BaseFullScore(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_in_state, PyObject *__pyx_v_word, struct __pyx_obj_5kenlm_State *__pyx_v_out_state) { - lm::WordIndex __pyx_v_wid; - struct lm::FullScoreReturn __pyx_v_ret; +static PyObject *__pyx_pf_5kenlm_5State___richcmp__(struct __pyx_obj_5kenlm_State *__pyx_v_qa, struct __pyx_obj_5kenlm_State *__pyx_v_qb, int __pyx_v_op) { + int __pyx_v_r; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; - char *__pyx_t_2; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - PyObject *__pyx_t_5 = NULL; - __Pyx_RefNannySetupContext("BaseFullScore", 0); + __Pyx_RefNannySetupContext("__richcmp__", 0); - /* "kenlm.pyx":246 - * :returns: FullScoreReturn(word|state) - * """ - * cdef _kenlm.WordIndex wid = self.vocab.Index(as_str(word)) # <<<<<<<<<<<<<< - * cdef _kenlm.FullScoreReturn ret = self.model.BaseFullScore(&in_state._c_state, wid, &out_state._c_state) - * return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0) + /* "kenlm.pyx":215 + * + * def __richcmp__(State qa, State qb, int op): + * r = qa._c_state.Compare(qb._c_state) # <<<<<<<<<<<<<< + * if op == 0: # < + * return r < 0 */ - __pyx_t_1 = __pyx_f_5kenlm_as_str(__pyx_v_word); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 246, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_AsString(__pyx_t_1); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) __PYX_ERR(0, 246, __pyx_L1_error) - __pyx_v_wid = __pyx_v_self->vocab->Index(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_r = __pyx_v_qa->_c_state.Compare(__pyx_v_qb->_c_state); - /* "kenlm.pyx":247 - * """ - * cdef _kenlm.WordIndex wid = self.vocab.Index(as_str(word)) - * cdef _kenlm.FullScoreReturn ret = self.model.BaseFullScore(&in_state._c_state, wid, &out_state._c_state) # <<<<<<<<<<<<<< - * return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0) - * + /* "kenlm.pyx":216 + * def __richcmp__(State qa, State qb, int op): + * r = qa._c_state.Compare(qb._c_state) + * if op == 0: # < # <<<<<<<<<<<<<< + * return r < 0 + * elif op == 1: # <= */ - __pyx_v_ret = __pyx_v_self->model->BaseFullScore((&__pyx_v_in_state->_c_state), __pyx_v_wid, (&__pyx_v_out_state->_c_state)); + switch (__pyx_v_op) { + case 0: - /* "kenlm.pyx":248 - * cdef _kenlm.WordIndex wid = self.vocab.Index(as_str(word)) - * cdef _kenlm.FullScoreReturn ret = self.model.BaseFullScore(&in_state._c_state, wid, &out_state._c_state) - * return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0) # <<<<<<<<<<<<<< - * - * def __contains__(self, word): + /* "kenlm.pyx":217 + * r = qa._c_state.Compare(qb._c_state) + * if op == 0: # < + * return r < 0 # <<<<<<<<<<<<<< + * elif op == 1: # <= + * return r <= 0 */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyFloat_FromDouble(__pyx_v_ret.prob); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 248, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_PyInt_From_unsigned_char(__pyx_v_ret.ngram_length); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 248, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyBool_FromLong((__pyx_v_wid == 0)); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 248, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = PyTuple_New(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 248, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_1); - __Pyx_GIVEREF(__pyx_t_3); - PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_4); - PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_t_4); - __pyx_t_1 = 0; - __pyx_t_3 = 0; - __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5kenlm_FullScoreReturn), __pyx_t_5, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 248, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_r = __pyx_t_4; - __pyx_t_4 = 0; - goto __pyx_L0; + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r < 0)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 217, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; - /* "kenlm.pyx":238 - * return total - * - * def BaseFullScore(self, State in_state, str word, State out_state): # <<<<<<<<<<<<<< - * """ - * Wrapper around model.BaseScore(in_state, Index(word), out_state) + /* "kenlm.pyx":216 + * def __richcmp__(State qa, State qb, int op): + * r = qa._c_state.Compare(qb._c_state) + * if op == 0: # < # <<<<<<<<<<<<<< + * return r < 0 + * elif op == 1: # <= */ + break; - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_XDECREF(__pyx_t_5); - __Pyx_AddTraceback("kenlm.Model.BaseFullScore", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} + /* "kenlm.pyx":218 + * if op == 0: # < + * return r < 0 + * elif op == 1: # <= # <<<<<<<<<<<<<< + * return r <= 0 + * elif op == 2: # == + */ + case 1: -/* "kenlm.pyx":250 - * return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0) - * - * def __contains__(self, word): # <<<<<<<<<<<<<< - * cdef bytes w = as_str(word) - * return (self.vocab.Index(w) != 0) + /* "kenlm.pyx":219 + * return r < 0 + * elif op == 1: # <= + * return r <= 0 # <<<<<<<<<<<<<< + * elif op == 2: # == + * return r == 0 */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r <= 0)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 219, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; -/* Python wrapper */ -static int __pyx_pw_5kenlm_5Model_20__contains__(PyObject *__pyx_v_self, PyObject *__pyx_v_word); /*proto*/ -static int __pyx_pw_5kenlm_5Model_20__contains__(PyObject *__pyx_v_self, PyObject *__pyx_v_word) { - int __pyx_r; + /* "kenlm.pyx":218 + * if op == 0: # < + * return r < 0 + * elif op == 1: # <= # <<<<<<<<<<<<<< + * return r <= 0 + * elif op == 2: # == + */ + break; + + /* "kenlm.pyx":220 + * elif op == 1: # <= + * return r <= 0 + * elif op == 2: # == # <<<<<<<<<<<<<< + * return r == 0 + * elif op == 3: # != + */ + case 2: + + /* "kenlm.pyx":221 + * return r <= 0 + * elif op == 2: # == + * return r == 0 # <<<<<<<<<<<<<< + * elif op == 3: # != + * return r != 0 + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r == 0)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 221, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "kenlm.pyx":220 + * elif op == 1: # <= + * return r <= 0 + * elif op == 2: # == # <<<<<<<<<<<<<< + * return r == 0 + * elif op == 3: # != + */ + break; + + /* "kenlm.pyx":222 + * elif op == 2: # == + * return r == 0 + * elif op == 3: # != # <<<<<<<<<<<<<< + * return r != 0 + * elif op == 4: # > + */ + case 3: + + /* "kenlm.pyx":223 + * return r == 0 + * elif op == 3: # != + * return r != 0 # <<<<<<<<<<<<<< + * elif op == 4: # > + * return r > 0 + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r != 0)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 223, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "kenlm.pyx":222 + * elif op == 2: # == + * return r == 0 + * elif op == 3: # != # <<<<<<<<<<<<<< + * return r != 0 + * elif op == 4: # > + */ + break; + + /* "kenlm.pyx":224 + * elif op == 3: # != + * return r != 0 + * elif op == 4: # > # <<<<<<<<<<<<<< + * return r > 0 + * else: # >= + */ + case 4: + + /* "kenlm.pyx":225 + * return r != 0 + * elif op == 4: # > + * return r > 0 # <<<<<<<<<<<<<< + * else: # >= + * return r >= 0 + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r > 0)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 225, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "kenlm.pyx":224 + * elif op == 3: # != + * return r != 0 + * elif op == 4: # > # <<<<<<<<<<<<<< + * return r > 0 + * else: # >= + */ + break; + default: + + /* "kenlm.pyx":227 + * return r > 0 + * else: # >= + * return r >= 0 # <<<<<<<<<<<<<< + * + * def __hash__(self): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r >= 0)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 227, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + break; + } + + /* "kenlm.pyx":214 + * cdef _kenlm.State _c_state + * + * def __richcmp__(State qa, State qb, int op): # <<<<<<<<<<<<<< + * r = qa._c_state.Compare(qb._c_state) + * if op == 0: # < + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("kenlm.State.__richcmp__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "kenlm.pyx":229 + * return r >= 0 + * + * def __hash__(self): # <<<<<<<<<<<<<< + * return _kenlm.hash_value(self._c_state) + * + */ + +/* Python wrapper */ +static Py_hash_t __pyx_pw_5kenlm_5State_3__hash__(PyObject *__pyx_v_self); /*proto*/ +static Py_hash_t __pyx_pw_5kenlm_5State_3__hash__(PyObject *__pyx_v_self) { + Py_hash_t __pyx_r; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__contains__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5Model_19__contains__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), ((PyObject *)__pyx_v_word)); + __Pyx_RefNannySetupContext("__hash__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_5State_2__hash__(((struct __pyx_obj_5kenlm_State *)__pyx_v_self)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static int __pyx_pf_5kenlm_5Model_19__contains__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_word) { - PyObject *__pyx_v_w = 0; - int __pyx_r; +static Py_hash_t __pyx_pf_5kenlm_5State_2__hash__(struct __pyx_obj_5kenlm_State *__pyx_v_self) { + Py_hash_t __pyx_r; __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - char *__pyx_t_2; - __Pyx_RefNannySetupContext("__contains__", 0); + __Pyx_RefNannySetupContext("__hash__", 0); - /* "kenlm.pyx":251 + /* "kenlm.pyx":230 * - * def __contains__(self, word): - * cdef bytes w = as_str(word) # <<<<<<<<<<<<<< - * return (self.vocab.Index(w) != 0) + * def __hash__(self): + * return _kenlm.hash_value(self._c_state) # <<<<<<<<<<<<<< * + * class LoadMethod: */ - __pyx_t_1 = __pyx_f_5kenlm_as_str(__pyx_v_word); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 251, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __pyx_v_w = ((PyObject*)__pyx_t_1); - __pyx_t_1 = 0; + __pyx_r = lm::ngram::hash_value(__pyx_v_self->_c_state); + goto __pyx_L0; - /* "kenlm.pyx":252 - * def __contains__(self, word): - * cdef bytes w = as_str(word) - * return (self.vocab.Index(w) != 0) # <<<<<<<<<<<<<< + /* "kenlm.pyx":229 + * return r >= 0 + * + * def __hash__(self): # <<<<<<<<<<<<<< + * return _kenlm.hash_value(self._c_state) * - * def __repr__(self): */ - __pyx_t_2 = __Pyx_PyObject_AsString(__pyx_v_w); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) __PYX_ERR(0, 252, __pyx_L1_error) - __pyx_r = (__pyx_v_self->vocab->Index(__pyx_t_2) != 0); - goto __pyx_L0; - /* "kenlm.pyx":250 - * return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0) - * - * def __contains__(self, word): # <<<<<<<<<<<<<< - * cdef bytes w = as_str(word) - * return (self.vocab.Index(w) != 0) + /* function exit code */ + __pyx_L0:; + if (unlikely(__pyx_r == -1) && !PyErr_Occurred()) __pyx_r = -2; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("self._c_state cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_5State_5__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5kenlm_5State_5__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_5State_4__reduce_cython__(((struct __pyx_obj_5kenlm_State *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5kenlm_5State_4__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_State *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("self._c_state cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self._c_state cannot be converted to a Python object for pickling") + */ + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__17, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("self._c_state cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.Model.__contains__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_w); + __Pyx_AddTraceback("kenlm.State.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "kenlm.pyx":254 - * return (self.vocab.Index(w) != 0) - * - * def __repr__(self): # <<<<<<<<<<<<<< - * return ''.format(os.path.basename(self.path)) - * +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("self._c_state cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("self._c_state cannot be converted to a Python object for pickling") */ /* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_22__repr__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5kenlm_5Model_22__repr__(PyObject *__pyx_v_self) { +static PyObject *__pyx_pw_5kenlm_5State_7__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state); /*proto*/ +static PyObject *__pyx_pw_5kenlm_5State_7__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5Model_21__repr__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_5State_6__setstate_cython__(((struct __pyx_obj_5kenlm_State *)__pyx_v_self), ((PyObject *)__pyx_v___pyx_state)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5kenlm_5Model_21__repr__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { +static PyObject *__pyx_pf_5kenlm_5State_6__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_State *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; - __Pyx_RefNannySetupContext("__repr__", 0); + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":4 + * raise TypeError("self._c_state cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self._c_state cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + */ + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__18, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("self._c_state cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("self._c_state cannot be converted to a Python object for pickling") + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("kenlm.State.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} - /* "kenlm.pyx":255 +/* "kenlm.pyx":246 + * cdef _kenlm.Config _c_config * - * def __repr__(self): - * return ''.format(os.path.basename(self.path)) # <<<<<<<<<<<<<< + * def __init__(self): # <<<<<<<<<<<<<< + * self._c_config = _kenlm.Config() * - * def __reduce__(self): */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_kp_s_Model_from_0, __pyx_n_s_format); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 255, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_os); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 255, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_path); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 255, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_basename); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 255, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = NULL; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_4))) { - __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_4); - if (likely(__pyx_t_5)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); - __Pyx_INCREF(__pyx_t_5); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_4, function); - } - } - if (!__pyx_t_5) { - __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_v_self->path); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 255, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - } else { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_4)) { - PyObject *__pyx_temp[2] = {__pyx_t_5, __pyx_v_self->path}; - __pyx_t_3 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 255, __pyx_L1_error) - __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_GOTREF(__pyx_t_3); - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { - PyObject *__pyx_temp[2] = {__pyx_t_5, __pyx_v_self->path}; - __pyx_t_3 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 255, __pyx_L1_error) - __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_GOTREF(__pyx_t_3); - } else - #endif - { - __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 255, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_5); __pyx_t_5 = NULL; - __Pyx_INCREF(__pyx_v_self->path); - __Pyx_GIVEREF(__pyx_v_self->path); - PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_v_self->path); - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_6, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 255, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - } - } - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = NULL; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_2))) { - __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_2); - if (likely(__pyx_t_4)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); - __Pyx_INCREF(__pyx_t_4); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_2, function); - } - } - if (!__pyx_t_4) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 255, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_GOTREF(__pyx_t_1); - } else { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_t_3}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 255, __pyx_L1_error) - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_t_3}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 255, __pyx_L1_error) - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - } else - #endif - { - __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 255, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; - __Pyx_GIVEREF(__pyx_t_3); - PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_3); - __pyx_t_3 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 255, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - } + +/* Python wrapper */ +static int __pyx_pw_5kenlm_6Config_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5kenlm_6Config_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); + if (unlikely(PyTuple_GET_SIZE(__pyx_args) > 0)) { + __Pyx_RaiseArgtupleInvalid("__init__", 1, 0, 0, PyTuple_GET_SIZE(__pyx_args)); return -1;} + if (unlikely(__pyx_kwds) && unlikely(PyDict_Size(__pyx_kwds) > 0) && unlikely(!__Pyx_CheckKeywordStrings(__pyx_kwds, "__init__", 0))) return -1; + __pyx_r = __pyx_pf_5kenlm_6Config___init__(((struct __pyx_obj_5kenlm_Config *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5kenlm_6Config___init__(struct __pyx_obj_5kenlm_Config *__pyx_v_self) { + int __pyx_r; + __Pyx_RefNannyDeclarations + lm::ngram::Config __pyx_t_1; + __Pyx_RefNannySetupContext("__init__", 0); + + /* "kenlm.pyx":247 + * + * def __init__(self): + * self._c_config = _kenlm.Config() # <<<<<<<<<<<<<< + * + * property load_method: + */ + try { + __pyx_t_1 = lm::ngram::Config(); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 247, __pyx_L1_error) } - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; + __pyx_v_self->_c_config = __pyx_t_1; - /* "kenlm.pyx":254 - * return (self.vocab.Index(w) != 0) + /* "kenlm.pyx":246 + * cdef _kenlm.Config _c_config * - * def __repr__(self): # <<<<<<<<<<<<<< - * return ''.format(os.path.basename(self.path)) + * def __init__(self): # <<<<<<<<<<<<<< + * self._c_config = _kenlm.Config() * */ /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_XDECREF(__pyx_t_5); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_AddTraceback("kenlm.Model.__repr__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; + __Pyx_AddTraceback("kenlm.Config.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "kenlm.pyx":257 - * return ''.format(os.path.basename(self.path)) - * - * def __reduce__(self): # <<<<<<<<<<<<<< - * return (Model, (self.path,)) +/* "kenlm.pyx":250 * + * property load_method: + * def __get__(self): # <<<<<<<<<<<<<< + * return self._c_config.load_method + * def __set__(self, to): */ /* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_24__reduce__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static PyObject *__pyx_pw_5kenlm_5Model_24__reduce__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { +static PyObject *__pyx_pw_5kenlm_6Config_11load_method_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5kenlm_6Config_11load_method_1__get__(PyObject *__pyx_v_self) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__reduce__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5Model_23__reduce__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_6Config_11load_method___get__(((struct __pyx_obj_5kenlm_Config *)__pyx_v_self)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5kenlm_5Model_23__reduce__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { +static PyObject *__pyx_pf_5kenlm_6Config_11load_method___get__(struct __pyx_obj_5kenlm_Config *__pyx_v_self) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - __Pyx_RefNannySetupContext("__reduce__", 0); + __Pyx_RefNannySetupContext("__get__", 0); - /* "kenlm.pyx":258 - * - * def __reduce__(self): - * return (Model, (self.path,)) # <<<<<<<<<<<<<< - * - * class LanguageModel(Model): + /* "kenlm.pyx":251 + * property load_method: + * def __get__(self): + * return self._c_config.load_method # <<<<<<<<<<<<<< + * def __set__(self, to): + * self._c_config.load_method = to */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 258, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_From_enum__util_3a__3a_LoadMethod(__pyx_v_self->_c_config.load_method); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 251, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_INCREF(__pyx_v_self->path); - __Pyx_GIVEREF(__pyx_v_self->path); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_self->path); - __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 258, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_INCREF(((PyObject *)__pyx_ptype_5kenlm_Model)); - __Pyx_GIVEREF(((PyObject *)__pyx_ptype_5kenlm_Model)); - PyTuple_SET_ITEM(__pyx_t_2, 0, ((PyObject *)__pyx_ptype_5kenlm_Model)); - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_t_1); + __pyx_r = __pyx_t_1; __pyx_t_1 = 0; - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; goto __pyx_L0; - /* "kenlm.pyx":257 - * return ''.format(os.path.basename(self.path)) - * - * def __reduce__(self): # <<<<<<<<<<<<<< - * return (Model, (self.path,)) + /* "kenlm.pyx":250 * + * property load_method: + * def __get__(self): # <<<<<<<<<<<<<< + * return self._c_config.load_method + * def __set__(self, to): */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("kenlm.Model.__reduce__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("kenlm.Config.load_method.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); @@ -4426,163 +5301,2959 @@ static PyObject *__pyx_pf_5kenlm_5Model_23__reduce__(struct __pyx_obj_5kenlm_Mod return __pyx_r; } -/* "kenlm.pyx":102 - * - * cdef _kenlm.Model* model - * cdef public bytes path # <<<<<<<<<<<<<< - * cdef _kenlm.const_Vocabulary* vocab +/* "kenlm.pyx":252 + * def __get__(self): + * return self._c_config.load_method + * def __set__(self, to): # <<<<<<<<<<<<<< + * self._c_config.load_method = to * */ /* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_4path_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5kenlm_5Model_4path_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; +static int __pyx_pw_5kenlm_6Config_11load_method_3__set__(PyObject *__pyx_v_self, PyObject *__pyx_v_to); /*proto*/ +static int __pyx_pw_5kenlm_6Config_11load_method_3__set__(PyObject *__pyx_v_self, PyObject *__pyx_v_to) { + int __pyx_r; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5Model_4path___get__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); + __Pyx_RefNannySetupContext("__set__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_6Config_11load_method_2__set__(((struct __pyx_obj_5kenlm_Config *)__pyx_v_self), ((PyObject *)__pyx_v_to)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5kenlm_5Model_4path___get__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { - PyObject *__pyx_r = NULL; +static int __pyx_pf_5kenlm_6Config_11load_method_2__set__(struct __pyx_obj_5kenlm_Config *__pyx_v_self, PyObject *__pyx_v_to) { + int __pyx_r; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_self->path); - __pyx_r = __pyx_v_self->path; - goto __pyx_L0; + enum util::LoadMethod __pyx_t_1; + __Pyx_RefNannySetupContext("__set__", 0); + + /* "kenlm.pyx":253 + * return self._c_config.load_method + * def __set__(self, to): + * self._c_config.load_method = to # <<<<<<<<<<<<<< + * + * cdef class Model: + */ + __pyx_t_1 = ((enum util::LoadMethod)__Pyx_PyInt_As_enum__util_3a__3a_LoadMethod(__pyx_v_to)); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 253, __pyx_L1_error) + __pyx_v_self->_c_config.load_method = __pyx_t_1; + + /* "kenlm.pyx":252 + * def __get__(self): + * return self._c_config.load_method + * def __set__(self, to): # <<<<<<<<<<<<<< + * self._c_config.load_method = to + * + */ /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("kenlm.Config.load_method.__set__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("self._c_config cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + */ + /* Python wrapper */ -static int __pyx_pw_5kenlm_5Model_4path_3__set__(PyObject *__pyx_v_self, PyObject *__pyx_v_value); /*proto*/ -static int __pyx_pw_5kenlm_5Model_4path_3__set__(PyObject *__pyx_v_self, PyObject *__pyx_v_value) { - int __pyx_r; +static PyObject *__pyx_pw_5kenlm_6Config_3__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5kenlm_6Config_3__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__set__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5Model_4path_2__set__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), ((PyObject *)__pyx_v_value)); + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_6Config_2__reduce_cython__(((struct __pyx_obj_5kenlm_Config *)__pyx_v_self)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static int __pyx_pf_5kenlm_5Model_4path_2__set__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_value) { - int __pyx_r; +static PyObject *__pyx_pf_5kenlm_6Config_2__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_Config *__pyx_v_self) { + PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; - __Pyx_RefNannySetupContext("__set__", 0); - if (!(likely(PyBytes_CheckExact(__pyx_v_value))||((__pyx_v_value) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_v_value)->tp_name), 0))) __PYX_ERR(0, 102, __pyx_L1_error) - __pyx_t_1 = __pyx_v_value; - __Pyx_INCREF(__pyx_t_1); - __Pyx_GIVEREF(__pyx_t_1); - __Pyx_GOTREF(__pyx_v_self->path); - __Pyx_DECREF(__pyx_v_self->path); - __pyx_v_self->path = ((PyObject*)__pyx_t_1); - __pyx_t_1 = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("self._c_config cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self._c_config cannot be converted to a Python object for pickling") + */ + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__19, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("self._c_config cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + */ /* function exit code */ - __pyx_r = 0; - goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.Model.path.__set__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; + __Pyx_AddTraceback("kenlm.Config.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("self._c_config cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("self._c_config cannot be converted to a Python object for pickling") + */ + /* Python wrapper */ -static int __pyx_pw_5kenlm_5Model_4path_5__del__(PyObject *__pyx_v_self); /*proto*/ -static int __pyx_pw_5kenlm_5Model_4path_5__del__(PyObject *__pyx_v_self) { - int __pyx_r; +static PyObject *__pyx_pw_5kenlm_6Config_5__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state); /*proto*/ +static PyObject *__pyx_pw_5kenlm_6Config_5__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__del__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5Model_4path_4__del__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_6Config_4__setstate_cython__(((struct __pyx_obj_5kenlm_Config *)__pyx_v_self), ((PyObject *)__pyx_v___pyx_state)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static int __pyx_pf_5kenlm_5Model_4path_4__del__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { - int __pyx_r; +static PyObject *__pyx_pf_5kenlm_6Config_4__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5kenlm_Config *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__del__", 0); - __Pyx_INCREF(Py_None); - __Pyx_GIVEREF(Py_None); - __Pyx_GOTREF(__pyx_v_self->path); - __Pyx_DECREF(__pyx_v_self->path); - __pyx_v_self->path = ((PyObject*)Py_None); + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":4 + * raise TypeError("self._c_config cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self._c_config cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + */ + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__20, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("self._c_config cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("self._c_config cannot be converted to a Python object for pickling") + */ /* function exit code */ - __pyx_r = 0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("kenlm.Config.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_tp_new_5kenlm_FullScoreReturn(PyTypeObject *t, PyObject *a, PyObject *k) { - PyObject *o; - if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { - o = (*t->tp_alloc)(t, 0); - } else { - o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); - } - if (unlikely(!o)) return 0; - if (unlikely(__pyx_pw_5kenlm_15FullScoreReturn_1__cinit__(o, a, k) < 0)) goto bad; - return o; - bad: - Py_DECREF(o); o = 0; - return NULL; -} +/* "kenlm.pyx":264 + * cdef _kenlm.const_Vocabulary* vocab + * + * def __init__(self, path, Config config = Config()): # <<<<<<<<<<<<<< + * """ + * Load the language model. + */ -static void __pyx_tp_dealloc_5kenlm_FullScoreReturn(PyObject *o) { - #if PY_VERSION_HEX >= 0x030400a1 - if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { - if (PyObject_CallFinalizerFromDealloc(o)) return; +/* Python wrapper */ +static int __pyx_pw_5kenlm_5Model_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_5kenlm_5Model___init__[] = "\n Load the language model.\n\n :param path: path to an arpa file or a kenlm binary file.\n :param config: configuration options (see lm/config.hh for documentation)\n "; +#if CYTHON_COMPILING_IN_CPYTHON +struct wrapperbase __pyx_wrapperbase_5kenlm_5Model___init__; +#endif +static int __pyx_pw_5kenlm_5Model_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_path = 0; + struct __pyx_obj_5kenlm_Config *__pyx_v_config = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_path,&__pyx_n_s_config,0}; + PyObject* values[2] = {0,0}; + values[1] = (PyObject *)__pyx_k__21; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_path)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_config); + if (value) { values[1] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(0, 264, __pyx_L3_error) + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_path = values[0]; + __pyx_v_config = ((struct __pyx_obj_5kenlm_Config *)values[1]); } - #endif - (*Py_TYPE(o)->tp_free)(o); -} + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 264, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("kenlm.Model.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_config), __pyx_ptype_5kenlm_Config, 1, "config", 0))) __PYX_ERR(0, 264, __pyx_L1_error) + __pyx_r = __pyx_pf_5kenlm_5Model___init__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), __pyx_v_path, __pyx_v_config); -static PyObject *__pyx_getprop_5kenlm_15FullScoreReturn_log_prob(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5kenlm_15FullScoreReturn_8log_prob_1__get__(o); + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; } -static PyObject *__pyx_getprop_5kenlm_15FullScoreReturn_ngram_length(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5kenlm_15FullScoreReturn_12ngram_length_1__get__(o); -} +static int __pyx_pf_5kenlm_5Model___init__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_path, struct __pyx_obj_5kenlm_Config *__pyx_v_config) { + PyObject *__pyx_v_exception = NULL; + PyObject *__pyx_v_exception_message = NULL; + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + char *__pyx_t_9; + lm::base::Model *__pyx_t_10; + int __pyx_t_11; + PyObject *__pyx_t_12 = NULL; + PyObject *__pyx_t_13 = NULL; + __Pyx_RefNannySetupContext("__init__", 0); -static PyObject *__pyx_getprop_5kenlm_15FullScoreReturn_oov(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5kenlm_15FullScoreReturn_3oov_1__get__(o); -} + /* "kenlm.pyx":271 + * :param config: configuration options (see lm/config.hh for documentation) + * """ + * self.path = os.path.abspath(as_str(path)) # <<<<<<<<<<<<<< + * try: + * self.model = _kenlm.LoadVirtual(self.path, config._c_config) + */ + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_os); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_path); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_abspath); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __pyx_f_5kenlm_as_str(__pyx_v_path); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_2); + if (likely(__pyx_t_4)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_2, function); + } + } + if (!__pyx_t_4) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 271, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_t_3}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 271, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_t_3}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 271, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + { + __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4); __pyx_t_4 = NULL; + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 271, __pyx_L1_error) + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v_self->path); + __Pyx_DECREF(__pyx_v_self->path); + __pyx_v_self->path = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; -static PyMethodDef __pyx_methods_5kenlm_FullScoreReturn[] = { - {0, 0, 0, 0} -}; + /* "kenlm.pyx":272 + * """ + * self.path = os.path.abspath(as_str(path)) + * try: # <<<<<<<<<<<<<< + * self.model = _kenlm.LoadVirtual(self.path, config._c_config) + * except RuntimeError as exception: + */ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_6, &__pyx_t_7, &__pyx_t_8); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_7); + __Pyx_XGOTREF(__pyx_t_8); + /*try:*/ { -static struct PyGetSetDef __pyx_getsets_5kenlm_FullScoreReturn[] = { - {(char *)"log_prob", __pyx_getprop_5kenlm_15FullScoreReturn_log_prob, 0, (char *)0, 0}, - {(char *)"ngram_length", __pyx_getprop_5kenlm_15FullScoreReturn_ngram_length, 0, (char *)0, 0}, - {(char *)"oov", __pyx_getprop_5kenlm_15FullScoreReturn_oov, 0, (char *)0, 0}, - {0, 0, 0, 0, 0} -}; + /* "kenlm.pyx":273 + * self.path = os.path.abspath(as_str(path)) + * try: + * self.model = _kenlm.LoadVirtual(self.path, config._c_config) # <<<<<<<<<<<<<< + * except RuntimeError as exception: + * exception_message = str(exception).replace('\n', ' ') + */ + if (unlikely(__pyx_v_self->path == Py_None)) { + PyErr_SetString(PyExc_TypeError, "expected bytes, NoneType found"); + __PYX_ERR(0, 273, __pyx_L3_error) + } + __pyx_t_9 = __Pyx_PyBytes_AsWritableString(__pyx_v_self->path); if (unlikely((!__pyx_t_9) && PyErr_Occurred())) __PYX_ERR(0, 273, __pyx_L3_error) + try { + __pyx_t_10 = lm::ngram::LoadVirtual(__pyx_t_9, __pyx_v_config->_c_config); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 273, __pyx_L3_error) + } + __pyx_v_self->model = __pyx_t_10; -static PyTypeObject __pyx_type_5kenlm_FullScoreReturn = { - PyVarObject_HEAD_INIT(0, 0) - "kenlm.FullScoreReturn", /*tp_name*/ - sizeof(struct __pyx_obj_5kenlm_FullScoreReturn), /*tp_basicsize*/ - 0, /*tp_itemsize*/ + /* "kenlm.pyx":272 + * """ + * self.path = os.path.abspath(as_str(path)) + * try: # <<<<<<<<<<<<<< + * self.model = _kenlm.LoadVirtual(self.path, config._c_config) + * except RuntimeError as exception: + */ + } + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L8_try_end; + __pyx_L3_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "kenlm.pyx":274 + * try: + * self.model = _kenlm.LoadVirtual(self.path, config._c_config) + * except RuntimeError as exception: # <<<<<<<<<<<<<< + * exception_message = str(exception).replace('\n', ' ') + * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ + */ + __pyx_t_11 = __Pyx_PyErr_ExceptionMatches(__pyx_builtin_RuntimeError); + if (__pyx_t_11) { + __Pyx_AddTraceback("kenlm.Model.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_1, &__pyx_t_2, &__pyx_t_5) < 0) __PYX_ERR(0, 274, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GOTREF(__pyx_t_5); + __Pyx_INCREF(__pyx_t_2); + __pyx_v_exception = __pyx_t_2; + + /* "kenlm.pyx":275 + * self.model = _kenlm.LoadVirtual(self.path, config._c_config) + * except RuntimeError as exception: + * exception_message = str(exception).replace('\n', ' ') # <<<<<<<<<<<<<< + * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ + * from exception + */ + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 275, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(__pyx_v_exception); + __Pyx_GIVEREF(__pyx_v_exception); + PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_exception); + __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)(&PyString_Type)), __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 275, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_replace); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 275, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_tuple__24, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 275, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_exception_message = __pyx_t_4; + __pyx_t_4 = 0; + + /* "kenlm.pyx":276 + * except RuntimeError as exception: + * exception_message = str(exception).replace('\n', ' ') + * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ # <<<<<<<<<<<<<< + * from exception + * self.vocab = &self.model.BaseVocabulary() + */ + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_kp_s_Cannot_read_model, __pyx_n_s_format); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 276, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_12 = NULL; + __pyx_t_11 = 0; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_12 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_12)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_12); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + __pyx_t_11 = 1; + } + } + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_3)) { + PyObject *__pyx_temp[3] = {__pyx_t_12, __pyx_v_path, __pyx_v_exception_message}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_11, 2+__pyx_t_11); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 276, __pyx_L5_except_error) + __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0; + __Pyx_GOTREF(__pyx_t_4); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_3)) { + PyObject *__pyx_temp[3] = {__pyx_t_12, __pyx_v_path, __pyx_v_exception_message}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_11, 2+__pyx_t_11); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 276, __pyx_L5_except_error) + __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0; + __Pyx_GOTREF(__pyx_t_4); + } else + #endif + { + __pyx_t_13 = PyTuple_New(2+__pyx_t_11); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 276, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_13); + if (__pyx_t_12) { + __Pyx_GIVEREF(__pyx_t_12); PyTuple_SET_ITEM(__pyx_t_13, 0, __pyx_t_12); __pyx_t_12 = NULL; + } + __Pyx_INCREF(__pyx_v_path); + __Pyx_GIVEREF(__pyx_v_path); + PyTuple_SET_ITEM(__pyx_t_13, 0+__pyx_t_11, __pyx_v_path); + __Pyx_INCREF(__pyx_v_exception_message); + __Pyx_GIVEREF(__pyx_v_exception_message); + PyTuple_SET_ITEM(__pyx_t_13, 1+__pyx_t_11, __pyx_v_exception_message); + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_13, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 276, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 276, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_IOError, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 276, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "kenlm.pyx":277 + * exception_message = str(exception).replace('\n', ' ') + * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ + * from exception # <<<<<<<<<<<<<< + * self.vocab = &self.model.BaseVocabulary() + * + */ + __Pyx_Raise(__pyx_t_4, 0, 0, __pyx_v_exception); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __PYX_ERR(0, 276, __pyx_L5_except_error) + } + goto __pyx_L5_except_error; + __pyx_L5_except_error:; + + /* "kenlm.pyx":272 + * """ + * self.path = os.path.abspath(as_str(path)) + * try: # <<<<<<<<<<<<<< + * self.model = _kenlm.LoadVirtual(self.path, config._c_config) + * except RuntimeError as exception: + */ + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L1_error; + __pyx_L8_try_end:; + } + + /* "kenlm.pyx":278 + * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ + * from exception + * self.vocab = &self.model.BaseVocabulary() # <<<<<<<<<<<<<< + * + * def __dealloc__(self): + */ + __pyx_v_self->vocab = (&__pyx_v_self->model->BaseVocabulary()); + + /* "kenlm.pyx":264 + * cdef _kenlm.const_Vocabulary* vocab + * + * def __init__(self, path, Config config = Config()): # <<<<<<<<<<<<<< + * """ + * Load the language model. + */ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_12); + __Pyx_XDECREF(__pyx_t_13); + __Pyx_AddTraceback("kenlm.Model.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_exception); + __Pyx_XDECREF(__pyx_v_exception_message); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "kenlm.pyx":280 + * self.vocab = &self.model.BaseVocabulary() + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * del self.model + * + */ + +/* Python wrapper */ +static void __pyx_pw_5kenlm_5Model_3__dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_pw_5kenlm_5Model_3__dealloc__(PyObject *__pyx_v_self) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_pf_5kenlm_5Model_2__dealloc__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +static void __pyx_pf_5kenlm_5Model_2__dealloc__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__", 0); + + /* "kenlm.pyx":281 + * + * def __dealloc__(self): + * del self.model # <<<<<<<<<<<<<< + * + * property order: + */ + delete __pyx_v_self->model; + + /* "kenlm.pyx":280 + * self.vocab = &self.model.BaseVocabulary() + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * del self.model + * + */ + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +/* "kenlm.pyx":284 + * + * property order: + * def __get__(self): # <<<<<<<<<<<<<< + * return self.model.Order() + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_5Model_5order_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5kenlm_5Model_5order_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_5Model_5order___get__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5kenlm_5Model_5order___get__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "kenlm.pyx":285 + * property order: + * def __get__(self): + * return self.model.Order() # <<<<<<<<<<<<<< + * + * def score(self, sentence, bos = True, eos = True): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyInt_From_unsigned_int(__pyx_v_self->model->Order()); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 285, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "kenlm.pyx":284 + * + * property order: + * def __get__(self): # <<<<<<<<<<<<<< + * return self.model.Order() + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("kenlm.Model.order.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "kenlm.pyx":287 + * return self.model.Order() + * + * def score(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<< + * """ + * Return the log10 probability of a string. By default, the string is + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_5Model_5score(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_5kenlm_5Model_4score[] = "\n Return the log10 probability of a string. By default, the string is\n treated as a sentence. \n return log10 p(sentence | )\n\n If you do not want to condition on the beginning of sentence, pass\n bos = False\n Never include as part of the string. That would be predicting the\n beginning of sentence. Language models are only supposed to condition\n on it as context.\n\n Similarly, the end of sentence token can be omitted with\n eos = False\n Since language models explicitly predict , it can be part of the\n string.\n\n Examples:\n\n #Good: returns log10 p(this is a sentence . | )\n model.score(\"this is a sentence .\")\n #Good: same as the above but more explicit\n model.score(\"this is a sentence .\", bos = True, eos = True)\n\n #Bad: never include \n model.score(\" this is a sentence\")\n #Bad: never include , even if bos = False.\n model.score(\" this is a sentence\", bos = False)\n\n #Good: returns log10 p(a fragment)\n model.score(\"a fragment\", bos = False, eos = False)\n\n #Good: returns log10 p(a fragment )\n model.score(\"a fragment\", bos = False, eos = True)\n\n #Ok, but bad practice: returns log10 p(a fragment )\n #Unlike , the end of sentence token can appear explicitly.\n model.score(\"a fragment \", bos = False, eos = False)\n "; +static PyObject *__pyx_pw_5kenlm_5Model_5score(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_sentence = 0; + PyObject *__pyx_v_bos = 0; + PyObject *__pyx_v_eos = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("score (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_sentence,&__pyx_n_s_bos,&__pyx_n_s_eos,0}; + PyObject* values[3] = {0,0,0}; + values[1] = ((PyObject *)Py_True); + values[2] = ((PyObject *)Py_True); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_sentence)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_bos); + if (value) { values[1] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 2: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_eos); + if (value) { values[2] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "score") < 0)) __PYX_ERR(0, 287, __pyx_L3_error) + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_sentence = values[0]; + __pyx_v_bos = values[1]; + __pyx_v_eos = values[2]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("score", 0, 1, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 287, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("kenlm.Model.score", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5kenlm_5Model_4score(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), __pyx_v_sentence, __pyx_v_bos, __pyx_v_eos); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5kenlm_5Model_4score(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_sentence, PyObject *__pyx_v_bos, PyObject *__pyx_v_eos) { + PyObject *__pyx_v_words = 0; + lm::ngram::State __pyx_v_state; + lm::ngram::State __pyx_v_out_state; + float __pyx_v_total; + PyObject *__pyx_v_word = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + Py_ssize_t __pyx_t_5; + char *__pyx_t_6; + __Pyx_RefNannySetupContext("score", 0); + + /* "kenlm.pyx":326 + * model.score("a fragment ", bos = False, eos = False) + * """ + * cdef list words = as_str(sentence).split() # <<<<<<<<<<<<<< + * cdef _kenlm.State state + * if bos: + */ + __pyx_t_2 = __pyx_f_5kenlm_as_str(__pyx_v_sentence); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 326, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_split); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 326, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_2)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + } + } + if (__pyx_t_2) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 326, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else { + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 326, __pyx_L1_error) + } + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (!(likely(PyList_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "list", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 326, __pyx_L1_error) + __pyx_v_words = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "kenlm.pyx":328 + * cdef list words = as_str(sentence).split() + * cdef _kenlm.State state + * if bos: # <<<<<<<<<<<<<< + * self.model.BeginSentenceWrite(&state) + * else: + */ + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_bos); if (unlikely(__pyx_t_4 < 0)) __PYX_ERR(0, 328, __pyx_L1_error) + if (__pyx_t_4) { + + /* "kenlm.pyx":329 + * cdef _kenlm.State state + * if bos: + * self.model.BeginSentenceWrite(&state) # <<<<<<<<<<<<<< + * else: + * self.model.NullContextWrite(&state) + */ + __pyx_v_self->model->BeginSentenceWrite((&__pyx_v_state)); + + /* "kenlm.pyx":328 + * cdef list words = as_str(sentence).split() + * cdef _kenlm.State state + * if bos: # <<<<<<<<<<<<<< + * self.model.BeginSentenceWrite(&state) + * else: + */ + goto __pyx_L3; + } + + /* "kenlm.pyx":331 + * self.model.BeginSentenceWrite(&state) + * else: + * self.model.NullContextWrite(&state) # <<<<<<<<<<<<<< + * cdef _kenlm.State out_state + * cdef float total = 0 + */ + /*else*/ { + __pyx_v_self->model->NullContextWrite((&__pyx_v_state)); + } + __pyx_L3:; + + /* "kenlm.pyx":333 + * self.model.NullContextWrite(&state) + * cdef _kenlm.State out_state + * cdef float total = 0 # <<<<<<<<<<<<<< + * for word in words: + * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) + */ + __pyx_v_total = 0.0; + + /* "kenlm.pyx":334 + * cdef _kenlm.State out_state + * cdef float total = 0 + * for word in words: # <<<<<<<<<<<<<< + * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) + * state = out_state + */ + if (unlikely(__pyx_v_words == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(0, 334, __pyx_L1_error) + } + __pyx_t_1 = __pyx_v_words; __Pyx_INCREF(__pyx_t_1); __pyx_t_5 = 0; + for (;;) { + if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_1)) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_3 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_5); __Pyx_INCREF(__pyx_t_3); __pyx_t_5++; if (unlikely(0 < 0)) __PYX_ERR(0, 334, __pyx_L1_error) + #else + __pyx_t_3 = PySequence_ITEM(__pyx_t_1, __pyx_t_5); __pyx_t_5++; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 334, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + __Pyx_XDECREF_SET(__pyx_v_word, __pyx_t_3); + __pyx_t_3 = 0; + + /* "kenlm.pyx":335 + * cdef float total = 0 + * for word in words: + * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) # <<<<<<<<<<<<<< + * state = out_state + * if eos: + */ + __pyx_t_6 = __Pyx_PyObject_AsWritableString(__pyx_v_word); if (unlikely((!__pyx_t_6) && PyErr_Occurred())) __PYX_ERR(0, 335, __pyx_L1_error) + __pyx_v_total = (__pyx_v_total + __pyx_v_self->model->BaseScore((&__pyx_v_state), __pyx_v_self->vocab->Index(__pyx_t_6), (&__pyx_v_out_state))); + + /* "kenlm.pyx":336 + * for word in words: + * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) + * state = out_state # <<<<<<<<<<<<<< + * if eos: + * total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state) + */ + __pyx_v_state = __pyx_v_out_state; + + /* "kenlm.pyx":334 + * cdef _kenlm.State out_state + * cdef float total = 0 + * for word in words: # <<<<<<<<<<<<<< + * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) + * state = out_state + */ + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "kenlm.pyx":337 + * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) + * state = out_state + * if eos: # <<<<<<<<<<<<<< + * total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state) + * return total + */ + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_eos); if (unlikely(__pyx_t_4 < 0)) __PYX_ERR(0, 337, __pyx_L1_error) + if (__pyx_t_4) { + + /* "kenlm.pyx":338 + * state = out_state + * if eos: + * total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state) # <<<<<<<<<<<<<< + * return total + * + */ + __pyx_v_total = (__pyx_v_total + __pyx_v_self->model->BaseScore((&__pyx_v_state), __pyx_v_self->vocab->EndSentence(), (&__pyx_v_out_state))); + + /* "kenlm.pyx":337 + * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) + * state = out_state + * if eos: # <<<<<<<<<<<<<< + * total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state) + * return total + */ + } + + /* "kenlm.pyx":339 + * if eos: + * total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state) + * return total # <<<<<<<<<<<<<< + * + * def perplexity(self, sentence): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyFloat_FromDouble(__pyx_v_total); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 339, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "kenlm.pyx":287 + * return self.model.Order() + * + * def score(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<< + * """ + * Return the log10 probability of a string. By default, the string is + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("kenlm.Model.score", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_words); + __Pyx_XDECREF(__pyx_v_word); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "kenlm.pyx":341 + * return total + * + * def perplexity(self, sentence): # <<<<<<<<<<<<<< + * """ + * Compute perplexity of a sentence. + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_5Model_7perplexity(PyObject *__pyx_v_self, PyObject *__pyx_v_sentence); /*proto*/ +static char __pyx_doc_5kenlm_5Model_6perplexity[] = "\n Compute perplexity of a sentence.\n @param sentence One full sentence to score. Do not include or .\n "; +static PyObject *__pyx_pw_5kenlm_5Model_7perplexity(PyObject *__pyx_v_self, PyObject *__pyx_v_sentence) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("perplexity (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_5Model_6perplexity(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), ((PyObject *)__pyx_v_sentence)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5kenlm_5Model_6perplexity(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_sentence) { + PyObject *__pyx_v_words = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + Py_ssize_t __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + __Pyx_RefNannySetupContext("perplexity", 0); + + /* "kenlm.pyx":346 + * @param sentence One full sentence to score. Do not include or . + * """ + * words = len(as_str(sentence).split()) + 1 # For # <<<<<<<<<<<<<< + * return 10.0**(-self.score(sentence) / words) + * + */ + __pyx_t_2 = __pyx_f_5kenlm_as_str(__pyx_v_sentence); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 346, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_split); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 346, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_2)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + } + } + if (__pyx_t_2) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 346, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else { + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 346, __pyx_L1_error) + } + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_4 = PyObject_Length(__pyx_t_1); if (unlikely(__pyx_t_4 == ((Py_ssize_t)-1))) __PYX_ERR(0, 346, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyInt_FromSsize_t((__pyx_t_4 + 1)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 346, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_words = __pyx_t_1; + __pyx_t_1 = 0; + + /* "kenlm.pyx":347 + * """ + * words = len(as_str(sentence).split()) + 1 # For + * return 10.0**(-self.score(sentence) / words) # <<<<<<<<<<<<<< + * + * def full_scores(self, sentence, bos = True, eos = True): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_score); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_2)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + } + } + if (!__pyx_t_2) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_v_sentence); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_3)) { + PyObject *__pyx_temp[2] = {__pyx_t_2, __pyx_v_sentence}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_3)) { + PyObject *__pyx_temp[2] = {__pyx_t_2, __pyx_v_sentence}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else + #endif + { + __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_2); __pyx_t_2 = NULL; + __Pyx_INCREF(__pyx_v_sentence); + __Pyx_GIVEREF(__pyx_v_sentence); + PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_v_sentence); + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = PyNumber_Negative(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyNumber_Divide(__pyx_t_3, __pyx_v_words); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = PyNumber_Power(__pyx_float_10_0, __pyx_t_1, Py_None); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "kenlm.pyx":341 + * return total + * + * def perplexity(self, sentence): # <<<<<<<<<<<<<< + * """ + * Compute perplexity of a sentence. + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("kenlm.Model.perplexity", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_words); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +static PyObject *__pyx_gb_5kenlm_5Model_10generator(__pyx_CoroutineObject *__pyx_generator, CYTHON_UNUSED PyThreadState *__pyx_tstate, PyObject *__pyx_sent_value); /* proto */ + +/* "kenlm.pyx":349 + * return 10.0**(-self.score(sentence) / words) + * + * def full_scores(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<< + * """ + * full_scores(sentence, bos = True, eos = Ture) -> generate full scores (prob, ngram length, oov) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_5Model_9full_scores(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_5kenlm_5Model_8full_scores[] = "\n full_scores(sentence, bos = True, eos = Ture) -> generate full scores (prob, ngram length, oov)\n @param sentence is a string (do not use boundary symbols)\n @param bos should kenlm add a bos state\n @param eos should kenlm add an eos state\n "; +static PyObject *__pyx_pw_5kenlm_5Model_9full_scores(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_sentence = 0; + PyObject *__pyx_v_bos = 0; + PyObject *__pyx_v_eos = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("full_scores (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_sentence,&__pyx_n_s_bos,&__pyx_n_s_eos,0}; + PyObject* values[3] = {0,0,0}; + values[1] = ((PyObject *)Py_True); + values[2] = ((PyObject *)Py_True); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_sentence)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_bos); + if (value) { values[1] = value; kw_args--; } + } + CYTHON_FALLTHROUGH; + case 2: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_eos); + if (value) { values[2] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "full_scores") < 0)) __PYX_ERR(0, 349, __pyx_L3_error) + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_sentence = values[0]; + __pyx_v_bos = values[1]; + __pyx_v_eos = values[2]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("full_scores", 0, 1, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 349, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("kenlm.Model.full_scores", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5kenlm_5Model_8full_scores(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), __pyx_v_sentence, __pyx_v_bos, __pyx_v_eos); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5kenlm_5Model_8full_scores(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_sentence, PyObject *__pyx_v_bos, PyObject *__pyx_v_eos) { + struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *__pyx_cur_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("full_scores", 0); + __pyx_cur_scope = (struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)__pyx_tp_new_5kenlm___pyx_scope_struct__full_scores(__pyx_ptype_5kenlm___pyx_scope_struct__full_scores, __pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(0, 349, __pyx_L1_error) + } else { + __Pyx_GOTREF(__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_v_self = __pyx_v_self; + __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + __pyx_cur_scope->__pyx_v_sentence = __pyx_v_sentence; + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_sentence); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_sentence); + __pyx_cur_scope->__pyx_v_bos = __pyx_v_bos; + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_bos); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_bos); + __pyx_cur_scope->__pyx_v_eos = __pyx_v_eos; + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_eos); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_eos); + { + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5kenlm_5Model_10generator, (PyObject *) __pyx_cur_scope, __pyx_n_s_full_scores, __pyx_n_s_Model_full_scores, __pyx_n_s_kenlm); if (unlikely(!gen)) __PYX_ERR(0, 349, __pyx_L1_error) + __Pyx_DECREF(__pyx_cur_scope); + __Pyx_RefNannyFinishContext(); + return (PyObject *) gen; + } + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("kenlm.Model.full_scores", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_gb_5kenlm_5Model_10generator(__pyx_CoroutineObject *__pyx_generator, CYTHON_UNUSED PyThreadState *__pyx_tstate, PyObject *__pyx_sent_value) /* generator body */ +{ + struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *__pyx_cur_scope = ((struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)__pyx_generator->closure); + PyObject *__pyx_r = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + Py_ssize_t __pyx_t_5; + char *__pyx_t_6; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("full_scores", 0); + switch (__pyx_generator->resume_label) { + case 0: goto __pyx_L3_first_run; + case 1: goto __pyx_L7_resume_from_yield; + case 2: goto __pyx_L9_resume_from_yield; + default: /* CPython raises the right error here */ + __Pyx_RefNannyFinishContext(); + return NULL; + } + __pyx_L3_first_run:; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 349, __pyx_L1_error) + + /* "kenlm.pyx":356 + * @param eos should kenlm add an eos state + * """ + * cdef list words = as_str(sentence).split() # <<<<<<<<<<<<<< + * cdef _kenlm.State state + * if bos: + */ + __pyx_t_2 = __pyx_f_5kenlm_as_str(__pyx_cur_scope->__pyx_v_sentence); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_split); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_2)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + } + } + if (__pyx_t_2) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else { + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 356, __pyx_L1_error) + } + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (!(likely(PyList_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "list", Py_TYPE(__pyx_t_1)->tp_name), 0))) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_GIVEREF(__pyx_t_1); + __pyx_cur_scope->__pyx_v_words = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "kenlm.pyx":358 + * cdef list words = as_str(sentence).split() + * cdef _kenlm.State state + * if bos: # <<<<<<<<<<<<<< + * self.model.BeginSentenceWrite(&state) + * else: + */ + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_cur_scope->__pyx_v_bos); if (unlikely(__pyx_t_4 < 0)) __PYX_ERR(0, 358, __pyx_L1_error) + if (__pyx_t_4) { + + /* "kenlm.pyx":359 + * cdef _kenlm.State state + * if bos: + * self.model.BeginSentenceWrite(&state) # <<<<<<<<<<<<<< + * else: + * self.model.NullContextWrite(&state) + */ + __pyx_cur_scope->__pyx_v_self->model->BeginSentenceWrite((&__pyx_cur_scope->__pyx_v_state)); + + /* "kenlm.pyx":358 + * cdef list words = as_str(sentence).split() + * cdef _kenlm.State state + * if bos: # <<<<<<<<<<<<<< + * self.model.BeginSentenceWrite(&state) + * else: + */ + goto __pyx_L4; + } + + /* "kenlm.pyx":361 + * self.model.BeginSentenceWrite(&state) + * else: + * self.model.NullContextWrite(&state) # <<<<<<<<<<<<<< + * cdef _kenlm.State out_state + * cdef _kenlm.FullScoreReturn ret + */ + /*else*/ { + __pyx_cur_scope->__pyx_v_self->model->NullContextWrite((&__pyx_cur_scope->__pyx_v_state)); + } + __pyx_L4:; + + /* "kenlm.pyx":364 + * cdef _kenlm.State out_state + * cdef _kenlm.FullScoreReturn ret + * cdef float total = 0 # <<<<<<<<<<<<<< + * cdef _kenlm.WordIndex wid + * for word in words: + */ + __pyx_cur_scope->__pyx_v_total = 0.0; + + /* "kenlm.pyx":366 + * cdef float total = 0 + * cdef _kenlm.WordIndex wid + * for word in words: # <<<<<<<<<<<<<< + * wid = self.vocab.Index(word) + * ret = self.model.BaseFullScore(&state, wid, &out_state) + */ + if (unlikely(__pyx_cur_scope->__pyx_v_words == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(0, 366, __pyx_L1_error) + } + __pyx_t_1 = __pyx_cur_scope->__pyx_v_words; __Pyx_INCREF(__pyx_t_1); __pyx_t_5 = 0; + for (;;) { + if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_1)) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_3 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_5); __Pyx_INCREF(__pyx_t_3); __pyx_t_5++; if (unlikely(0 < 0)) __PYX_ERR(0, 366, __pyx_L1_error) + #else + __pyx_t_3 = PySequence_ITEM(__pyx_t_1, __pyx_t_5); __pyx_t_5++; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 366, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + __Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_word); + __Pyx_XDECREF_SET(__pyx_cur_scope->__pyx_v_word, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + __pyx_t_3 = 0; + + /* "kenlm.pyx":367 + * cdef _kenlm.WordIndex wid + * for word in words: + * wid = self.vocab.Index(word) # <<<<<<<<<<<<<< + * ret = self.model.BaseFullScore(&state, wid, &out_state) + * yield (ret.prob, ret.ngram_length, wid == 0) + */ + __pyx_t_6 = __Pyx_PyObject_AsWritableString(__pyx_cur_scope->__pyx_v_word); if (unlikely((!__pyx_t_6) && PyErr_Occurred())) __PYX_ERR(0, 367, __pyx_L1_error) + __pyx_cur_scope->__pyx_v_wid = __pyx_cur_scope->__pyx_v_self->vocab->Index(__pyx_t_6); + + /* "kenlm.pyx":368 + * for word in words: + * wid = self.vocab.Index(word) + * ret = self.model.BaseFullScore(&state, wid, &out_state) # <<<<<<<<<<<<<< + * yield (ret.prob, ret.ngram_length, wid == 0) + * state = out_state + */ + __pyx_cur_scope->__pyx_v_ret = __pyx_cur_scope->__pyx_v_self->model->BaseFullScore((&__pyx_cur_scope->__pyx_v_state), __pyx_cur_scope->__pyx_v_wid, (&__pyx_cur_scope->__pyx_v_out_state)); + + /* "kenlm.pyx":369 + * wid = self.vocab.Index(word) + * ret = self.model.BaseFullScore(&state, wid, &out_state) + * yield (ret.prob, ret.ngram_length, wid == 0) # <<<<<<<<<<<<<< + * state = out_state + * if eos: + */ + __pyx_t_3 = PyFloat_FromDouble(__pyx_cur_scope->__pyx_v_ret.prob); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 369, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __Pyx_PyInt_From_unsigned_char(__pyx_cur_scope->__pyx_v_ret.ngram_length); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 369, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_7 = __Pyx_PyBool_FromLong((__pyx_cur_scope->__pyx_v_wid == 0)); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 369, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = PyTuple_New(3); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 369, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_2); + __Pyx_GIVEREF(__pyx_t_7); + PyTuple_SET_ITEM(__pyx_t_8, 2, __pyx_t_7); + __pyx_t_3 = 0; + __pyx_t_2 = 0; + __pyx_t_7 = 0; + __pyx_r = __pyx_t_8; + __pyx_t_8 = 0; + __Pyx_XGIVEREF(__pyx_t_1); + __pyx_cur_scope->__pyx_t_0 = __pyx_t_1; + __pyx_cur_scope->__pyx_t_1 = __pyx_t_5; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + __Pyx_Coroutine_ResetAndClearException(__pyx_generator); + /* return from generator, yielding value */ + __pyx_generator->resume_label = 1; + return __pyx_r; + __pyx_L7_resume_from_yield:; + __pyx_t_1 = __pyx_cur_scope->__pyx_t_0; + __pyx_cur_scope->__pyx_t_0 = 0; + __Pyx_XGOTREF(__pyx_t_1); + __pyx_t_5 = __pyx_cur_scope->__pyx_t_1; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 369, __pyx_L1_error) + + /* "kenlm.pyx":370 + * ret = self.model.BaseFullScore(&state, wid, &out_state) + * yield (ret.prob, ret.ngram_length, wid == 0) + * state = out_state # <<<<<<<<<<<<<< + * if eos: + * ret = self.model.BaseFullScore(&state, + */ + __pyx_cur_scope->__pyx_v_state = __pyx_cur_scope->__pyx_v_out_state; + + /* "kenlm.pyx":366 + * cdef float total = 0 + * cdef _kenlm.WordIndex wid + * for word in words: # <<<<<<<<<<<<<< + * wid = self.vocab.Index(word) + * ret = self.model.BaseFullScore(&state, wid, &out_state) + */ + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "kenlm.pyx":371 + * yield (ret.prob, ret.ngram_length, wid == 0) + * state = out_state + * if eos: # <<<<<<<<<<<<<< + * ret = self.model.BaseFullScore(&state, + * self.vocab.EndSentence(), &out_state) + */ + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_cur_scope->__pyx_v_eos); if (unlikely(__pyx_t_4 < 0)) __PYX_ERR(0, 371, __pyx_L1_error) + if (__pyx_t_4) { + + /* "kenlm.pyx":372 + * state = out_state + * if eos: + * ret = self.model.BaseFullScore(&state, # <<<<<<<<<<<<<< + * self.vocab.EndSentence(), &out_state) + * yield (ret.prob, ret.ngram_length, False) + */ + __pyx_cur_scope->__pyx_v_ret = __pyx_cur_scope->__pyx_v_self->model->BaseFullScore((&__pyx_cur_scope->__pyx_v_state), __pyx_cur_scope->__pyx_v_self->vocab->EndSentence(), (&__pyx_cur_scope->__pyx_v_out_state)); + + /* "kenlm.pyx":374 + * ret = self.model.BaseFullScore(&state, + * self.vocab.EndSentence(), &out_state) + * yield (ret.prob, ret.ngram_length, False) # <<<<<<<<<<<<<< + * + * + */ + __pyx_t_1 = PyFloat_FromDouble(__pyx_cur_scope->__pyx_v_ret.prob); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 374, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_8 = __Pyx_PyInt_From_unsigned_char(__pyx_cur_scope->__pyx_v_ret.ngram_length); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 374, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_7 = PyTuple_New(3); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 374, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_8); + PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_8); + __Pyx_INCREF(Py_False); + __Pyx_GIVEREF(Py_False); + PyTuple_SET_ITEM(__pyx_t_7, 2, Py_False); + __pyx_t_1 = 0; + __pyx_t_8 = 0; + __pyx_r = __pyx_t_7; + __pyx_t_7 = 0; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + __Pyx_Coroutine_ResetAndClearException(__pyx_generator); + /* return from generator, yielding value */ + __pyx_generator->resume_label = 2; + return __pyx_r; + __pyx_L9_resume_from_yield:; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 374, __pyx_L1_error) + + /* "kenlm.pyx":371 + * yield (ret.prob, ret.ngram_length, wid == 0) + * state = out_state + * if eos: # <<<<<<<<<<<<<< + * ret = self.model.BaseFullScore(&state, + * self.vocab.EndSentence(), &out_state) + */ + } + CYTHON_MAYBE_UNUSED_VAR(__pyx_cur_scope); + + /* "kenlm.pyx":349 + * return 10.0**(-self.score(sentence) / words) + * + * def full_scores(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<< + * """ + * full_scores(sentence, bos = True, eos = Ture) -> generate full scores (prob, ngram length, oov) + */ + + /* function exit code */ + PyErr_SetNone(PyExc_StopIteration); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("full_scores", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_L0:; + __Pyx_XDECREF(__pyx_r); __pyx_r = 0; + __Pyx_Coroutine_ResetAndClearException(__pyx_generator); + __pyx_generator->resume_label = -1; + __Pyx_Coroutine_clear((PyObject*)__pyx_generator); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "kenlm.pyx":377 + * + * + * def BeginSentenceWrite(self, State state): # <<<<<<<<<<<<<< + * """Change the given state to a BOS state.""" + * self.model.BeginSentenceWrite(&state._c_state) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_5Model_12BeginSentenceWrite(PyObject *__pyx_v_self, PyObject *__pyx_v_state); /*proto*/ +static char __pyx_doc_5kenlm_5Model_11BeginSentenceWrite[] = "Change the given state to a BOS state."; +static PyObject *__pyx_pw_5kenlm_5Model_12BeginSentenceWrite(PyObject *__pyx_v_self, PyObject *__pyx_v_state) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("BeginSentenceWrite (wrapper)", 0); + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_state), __pyx_ptype_5kenlm_State, 1, "state", 0))) __PYX_ERR(0, 377, __pyx_L1_error) + __pyx_r = __pyx_pf_5kenlm_5Model_11BeginSentenceWrite(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), ((struct __pyx_obj_5kenlm_State *)__pyx_v_state)); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5kenlm_5Model_11BeginSentenceWrite(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("BeginSentenceWrite", 0); + + /* "kenlm.pyx":379 + * def BeginSentenceWrite(self, State state): + * """Change the given state to a BOS state.""" + * self.model.BeginSentenceWrite(&state._c_state) # <<<<<<<<<<<<<< + * + * def NullContextWrite(self, State state): + */ + __pyx_v_self->model->BeginSentenceWrite((&__pyx_v_state->_c_state)); + + /* "kenlm.pyx":377 + * + * + * def BeginSentenceWrite(self, State state): # <<<<<<<<<<<<<< + * """Change the given state to a BOS state.""" + * self.model.BeginSentenceWrite(&state._c_state) + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "kenlm.pyx":381 + * self.model.BeginSentenceWrite(&state._c_state) + * + * def NullContextWrite(self, State state): # <<<<<<<<<<<<<< + * """Change the given state to a NULL state.""" + * self.model.NullContextWrite(&state._c_state) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_5Model_14NullContextWrite(PyObject *__pyx_v_self, PyObject *__pyx_v_state); /*proto*/ +static char __pyx_doc_5kenlm_5Model_13NullContextWrite[] = "Change the given state to a NULL state."; +static PyObject *__pyx_pw_5kenlm_5Model_14NullContextWrite(PyObject *__pyx_v_self, PyObject *__pyx_v_state) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("NullContextWrite (wrapper)", 0); + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_state), __pyx_ptype_5kenlm_State, 1, "state", 0))) __PYX_ERR(0, 381, __pyx_L1_error) + __pyx_r = __pyx_pf_5kenlm_5Model_13NullContextWrite(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), ((struct __pyx_obj_5kenlm_State *)__pyx_v_state)); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5kenlm_5Model_13NullContextWrite(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("NullContextWrite", 0); + + /* "kenlm.pyx":383 + * def NullContextWrite(self, State state): + * """Change the given state to a NULL state.""" + * self.model.NullContextWrite(&state._c_state) # <<<<<<<<<<<<<< + * + * def BaseScore(self, State in_state, str word, State out_state): + */ + __pyx_v_self->model->NullContextWrite((&__pyx_v_state->_c_state)); + + /* "kenlm.pyx":381 + * self.model.BeginSentenceWrite(&state._c_state) + * + * def NullContextWrite(self, State state): # <<<<<<<<<<<<<< + * """Change the given state to a NULL state.""" + * self.model.NullContextWrite(&state._c_state) + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "kenlm.pyx":385 + * self.model.NullContextWrite(&state._c_state) + * + * def BaseScore(self, State in_state, str word, State out_state): # <<<<<<<<<<<<<< + * """ + * Return p(word|in_state) and update the output state. + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_5Model_16BaseScore(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_5kenlm_5Model_15BaseScore[] = "\n Return p(word|in_state) and update the output state.\n Wrapper around model.BaseScore(in_state, Index(word), out_state)\n\n :param word: the suffix\n :param state: the context (defaults to NullContext)\n :returns: p(word|state)\n "; +static PyObject *__pyx_pw_5kenlm_5Model_16BaseScore(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + struct __pyx_obj_5kenlm_State *__pyx_v_in_state = 0; + PyObject *__pyx_v_word = 0; + struct __pyx_obj_5kenlm_State *__pyx_v_out_state = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("BaseScore (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_in_state,&__pyx_n_s_word,&__pyx_n_s_out_state,0}; + PyObject* values[3] = {0,0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_in_state)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_word)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("BaseScore", 1, 3, 3, 1); __PYX_ERR(0, 385, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 2: + if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_out_state)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("BaseScore", 1, 3, 3, 2); __PYX_ERR(0, 385, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "BaseScore") < 0)) __PYX_ERR(0, 385, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + } + __pyx_v_in_state = ((struct __pyx_obj_5kenlm_State *)values[0]); + __pyx_v_word = ((PyObject*)values[1]); + __pyx_v_out_state = ((struct __pyx_obj_5kenlm_State *)values[2]); + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("BaseScore", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 385, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("kenlm.Model.BaseScore", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_in_state), __pyx_ptype_5kenlm_State, 1, "in_state", 0))) __PYX_ERR(0, 385, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_word), (&PyString_Type), 1, "word", 1))) __PYX_ERR(0, 385, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_out_state), __pyx_ptype_5kenlm_State, 1, "out_state", 0))) __PYX_ERR(0, 385, __pyx_L1_error) + __pyx_r = __pyx_pf_5kenlm_5Model_15BaseScore(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), __pyx_v_in_state, __pyx_v_word, __pyx_v_out_state); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5kenlm_5Model_15BaseScore(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_in_state, PyObject *__pyx_v_word, struct __pyx_obj_5kenlm_State *__pyx_v_out_state) { + float __pyx_v_total; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + char *__pyx_t_2; + __Pyx_RefNannySetupContext("BaseScore", 0); + + /* "kenlm.pyx":394 + * :returns: p(word|state) + * """ + * cdef float total = self.model.BaseScore(&in_state._c_state, self.vocab.Index(as_str(word)), &out_state._c_state) # <<<<<<<<<<<<<< + * return total + * + */ + __pyx_t_1 = __pyx_f_5kenlm_as_str(__pyx_v_word); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 394, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (unlikely(__pyx_t_1 == Py_None)) { + PyErr_SetString(PyExc_TypeError, "expected bytes, NoneType found"); + __PYX_ERR(0, 394, __pyx_L1_error) + } + __pyx_t_2 = __Pyx_PyBytes_AsWritableString(__pyx_t_1); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) __PYX_ERR(0, 394, __pyx_L1_error) + __pyx_v_total = __pyx_v_self->model->BaseScore((&__pyx_v_in_state->_c_state), __pyx_v_self->vocab->Index(__pyx_t_2), (&__pyx_v_out_state->_c_state)); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "kenlm.pyx":395 + * """ + * cdef float total = self.model.BaseScore(&in_state._c_state, self.vocab.Index(as_str(word)), &out_state._c_state) + * return total # <<<<<<<<<<<<<< + * + * def BaseFullScore(self, State in_state, str word, State out_state): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyFloat_FromDouble(__pyx_v_total); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 395, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "kenlm.pyx":385 + * self.model.NullContextWrite(&state._c_state) + * + * def BaseScore(self, State in_state, str word, State out_state): # <<<<<<<<<<<<<< + * """ + * Return p(word|in_state) and update the output state. + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("kenlm.Model.BaseScore", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "kenlm.pyx":397 + * return total + * + * def BaseFullScore(self, State in_state, str word, State out_state): # <<<<<<<<<<<<<< + * """ + * Wrapper around model.BaseScore(in_state, Index(word), out_state) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_5Model_18BaseFullScore(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_5kenlm_5Model_17BaseFullScore[] = "\n Wrapper around model.BaseScore(in_state, Index(word), out_state)\n\n :param word: the suffix\n :param state: the context (defaults to NullContext)\n :returns: FullScoreReturn(word|state)\n "; +static PyObject *__pyx_pw_5kenlm_5Model_18BaseFullScore(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + struct __pyx_obj_5kenlm_State *__pyx_v_in_state = 0; + PyObject *__pyx_v_word = 0; + struct __pyx_obj_5kenlm_State *__pyx_v_out_state = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("BaseFullScore (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_in_state,&__pyx_n_s_word,&__pyx_n_s_out_state,0}; + PyObject* values[3] = {0,0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_in_state)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; + case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_word)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("BaseFullScore", 1, 3, 3, 1); __PYX_ERR(0, 397, __pyx_L3_error) + } + CYTHON_FALLTHROUGH; + case 2: + if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_out_state)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("BaseFullScore", 1, 3, 3, 2); __PYX_ERR(0, 397, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "BaseFullScore") < 0)) __PYX_ERR(0, 397, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + } + __pyx_v_in_state = ((struct __pyx_obj_5kenlm_State *)values[0]); + __pyx_v_word = ((PyObject*)values[1]); + __pyx_v_out_state = ((struct __pyx_obj_5kenlm_State *)values[2]); + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("BaseFullScore", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 397, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("kenlm.Model.BaseFullScore", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_in_state), __pyx_ptype_5kenlm_State, 1, "in_state", 0))) __PYX_ERR(0, 397, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_word), (&PyString_Type), 1, "word", 1))) __PYX_ERR(0, 397, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_out_state), __pyx_ptype_5kenlm_State, 1, "out_state", 0))) __PYX_ERR(0, 397, __pyx_L1_error) + __pyx_r = __pyx_pf_5kenlm_5Model_17BaseFullScore(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), __pyx_v_in_state, __pyx_v_word, __pyx_v_out_state); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5kenlm_5Model_17BaseFullScore(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_in_state, PyObject *__pyx_v_word, struct __pyx_obj_5kenlm_State *__pyx_v_out_state) { + lm::WordIndex __pyx_v_wid; + struct lm::FullScoreReturn __pyx_v_ret; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + char *__pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + __Pyx_RefNannySetupContext("BaseFullScore", 0); + + /* "kenlm.pyx":405 + * :returns: FullScoreReturn(word|state) + * """ + * cdef _kenlm.WordIndex wid = self.vocab.Index(as_str(word)) # <<<<<<<<<<<<<< + * cdef _kenlm.FullScoreReturn ret = self.model.BaseFullScore(&in_state._c_state, wid, &out_state._c_state) + * return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0) + */ + __pyx_t_1 = __pyx_f_5kenlm_as_str(__pyx_v_word); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 405, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (unlikely(__pyx_t_1 == Py_None)) { + PyErr_SetString(PyExc_TypeError, "expected bytes, NoneType found"); + __PYX_ERR(0, 405, __pyx_L1_error) + } + __pyx_t_2 = __Pyx_PyBytes_AsWritableString(__pyx_t_1); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) __PYX_ERR(0, 405, __pyx_L1_error) + __pyx_v_wid = __pyx_v_self->vocab->Index(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "kenlm.pyx":406 + * """ + * cdef _kenlm.WordIndex wid = self.vocab.Index(as_str(word)) + * cdef _kenlm.FullScoreReturn ret = self.model.BaseFullScore(&in_state._c_state, wid, &out_state._c_state) # <<<<<<<<<<<<<< + * return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0) + * + */ + __pyx_v_ret = __pyx_v_self->model->BaseFullScore((&__pyx_v_in_state->_c_state), __pyx_v_wid, (&__pyx_v_out_state->_c_state)); + + /* "kenlm.pyx":407 + * cdef _kenlm.WordIndex wid = self.vocab.Index(as_str(word)) + * cdef _kenlm.FullScoreReturn ret = self.model.BaseFullScore(&in_state._c_state, wid, &out_state._c_state) + * return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0) # <<<<<<<<<<<<<< + * + * def __contains__(self, word): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyFloat_FromDouble(__pyx_v_ret.prob); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 407, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyInt_From_unsigned_char(__pyx_v_ret.ngram_length); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 407, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyBool_FromLong((__pyx_v_wid == 0)); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 407, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 407, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_t_4); + __pyx_t_1 = 0; + __pyx_t_3 = 0; + __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5kenlm_FullScoreReturn), __pyx_t_5, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 407, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* "kenlm.pyx":397 + * return total + * + * def BaseFullScore(self, State in_state, str word, State out_state): # <<<<<<<<<<<<<< + * """ + * Wrapper around model.BaseScore(in_state, Index(word), out_state) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("kenlm.Model.BaseFullScore", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "kenlm.pyx":409 + * return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0) + * + * def __contains__(self, word): # <<<<<<<<<<<<<< + * cdef bytes w = as_str(word) + * return (self.vocab.Index(w) != 0) + */ + +/* Python wrapper */ +static int __pyx_pw_5kenlm_5Model_20__contains__(PyObject *__pyx_v_self, PyObject *__pyx_v_word); /*proto*/ +static int __pyx_pw_5kenlm_5Model_20__contains__(PyObject *__pyx_v_self, PyObject *__pyx_v_word) { + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__contains__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_5Model_19__contains__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), ((PyObject *)__pyx_v_word)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5kenlm_5Model_19__contains__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_word) { + PyObject *__pyx_v_w = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + char *__pyx_t_2; + __Pyx_RefNannySetupContext("__contains__", 0); + + /* "kenlm.pyx":410 + * + * def __contains__(self, word): + * cdef bytes w = as_str(word) # <<<<<<<<<<<<<< + * return (self.vocab.Index(w) != 0) + * + */ + __pyx_t_1 = __pyx_f_5kenlm_as_str(__pyx_v_word); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 410, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_w = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "kenlm.pyx":411 + * def __contains__(self, word): + * cdef bytes w = as_str(word) + * return (self.vocab.Index(w) != 0) # <<<<<<<<<<<<<< + * + * def __repr__(self): + */ + if (unlikely(__pyx_v_w == Py_None)) { + PyErr_SetString(PyExc_TypeError, "expected bytes, NoneType found"); + __PYX_ERR(0, 411, __pyx_L1_error) + } + __pyx_t_2 = __Pyx_PyBytes_AsWritableString(__pyx_v_w); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) __PYX_ERR(0, 411, __pyx_L1_error) + __pyx_r = (__pyx_v_self->vocab->Index(__pyx_t_2) != 0); + goto __pyx_L0; + + /* "kenlm.pyx":409 + * return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0) + * + * def __contains__(self, word): # <<<<<<<<<<<<<< + * cdef bytes w = as_str(word) + * return (self.vocab.Index(w) != 0) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("kenlm.Model.__contains__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_w); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "kenlm.pyx":413 + * return (self.vocab.Index(w) != 0) + * + * def __repr__(self): # <<<<<<<<<<<<<< + * return ''.format(os.path.basename(self.path)) + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_5Model_22__repr__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5kenlm_5Model_22__repr__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_5Model_21__repr__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5kenlm_5Model_21__repr__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + __Pyx_RefNannySetupContext("__repr__", 0); + + /* "kenlm.pyx":414 + * + * def __repr__(self): + * return ''.format(os.path.basename(self.path)) # <<<<<<<<<<<<<< + * + * def __reduce__(self): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_kp_s_Model_from_0, __pyx_n_s_format); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_os); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_path); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_basename); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_4); + if (likely(__pyx_t_5)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_5); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_4, function); + } + } + if (!__pyx_t_5) { + __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_v_self->path); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_4)) { + PyObject *__pyx_temp[2] = {__pyx_t_5, __pyx_v_self->path}; + __pyx_t_3 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_GOTREF(__pyx_t_3); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { + PyObject *__pyx_temp[2] = {__pyx_t_5, __pyx_v_self->path}; + __pyx_t_3 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_GOTREF(__pyx_t_3); + } else + #endif + { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_5); __pyx_t_5 = NULL; + __Pyx_INCREF(__pyx_v_self->path); + __Pyx_GIVEREF(__pyx_v_self->path); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_v_self->path); + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_6, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_2); + if (likely(__pyx_t_4)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_2, function); + } + } + if (!__pyx_t_4) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_t_3}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_t_3}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "kenlm.pyx":413 + * return (self.vocab.Index(w) != 0) + * + * def __repr__(self): # <<<<<<<<<<<<<< + * return ''.format(os.path.basename(self.path)) + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("kenlm.Model.__repr__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "kenlm.pyx":416 + * return ''.format(os.path.basename(self.path)) + * + * def __reduce__(self): # <<<<<<<<<<<<<< + * return (Model, (self.path,)) + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_5Model_24__reduce__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5kenlm_5Model_24__reduce__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_5Model_23__reduce__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5kenlm_5Model_23__reduce__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannySetupContext("__reduce__", 0); + + /* "kenlm.pyx":417 + * + * def __reduce__(self): + * return (Model, (self.path,)) # <<<<<<<<<<<<<< + * + * class LanguageModel(Model): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 417, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v_self->path); + __Pyx_GIVEREF(__pyx_v_self->path); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_self->path); + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 417, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(((PyObject *)__pyx_ptype_5kenlm_Model)); + __Pyx_GIVEREF(((PyObject *)__pyx_ptype_5kenlm_Model)); + PyTuple_SET_ITEM(__pyx_t_2, 0, ((PyObject *)__pyx_ptype_5kenlm_Model)); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "kenlm.pyx":416 + * return ''.format(os.path.basename(self.path)) + * + * def __reduce__(self): # <<<<<<<<<<<<<< + * return (Model, (self.path,)) + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("kenlm.Model.__reduce__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "kenlm.pyx":261 + * + * cdef _kenlm.Model* model + * cdef public bytes path # <<<<<<<<<<<<<< + * cdef _kenlm.const_Vocabulary* vocab + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5kenlm_5Model_4path_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5kenlm_5Model_4path_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_5Model_4path___get__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5kenlm_5Model_4path___get__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->path); + __pyx_r = __pyx_v_self->path; + goto __pyx_L0; + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static int __pyx_pw_5kenlm_5Model_4path_3__set__(PyObject *__pyx_v_self, PyObject *__pyx_v_value); /*proto*/ +static int __pyx_pw_5kenlm_5Model_4path_3__set__(PyObject *__pyx_v_self, PyObject *__pyx_v_value) { + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__set__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_5Model_4path_2__set__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), ((PyObject *)__pyx_v_value)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5kenlm_5Model_4path_2__set__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_value) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__set__", 0); + if (!(likely(PyBytes_CheckExact(__pyx_v_value))||((__pyx_v_value) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_v_value)->tp_name), 0))) __PYX_ERR(0, 261, __pyx_L1_error) + __pyx_t_1 = __pyx_v_value; + __Pyx_INCREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v_self->path); + __Pyx_DECREF(__pyx_v_self->path); + __pyx_v_self->path = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("kenlm.Model.path.__set__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static int __pyx_pw_5kenlm_5Model_4path_5__del__(PyObject *__pyx_v_self); /*proto*/ +static int __pyx_pw_5kenlm_5Model_4path_5__del__(PyObject *__pyx_v_self) { + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__del__ (wrapper)", 0); + __pyx_r = __pyx_pf_5kenlm_5Model_4path_4__del__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5kenlm_5Model_4path_4__del__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__del__", 0); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + __Pyx_GOTREF(__pyx_v_self->path); + __Pyx_DECREF(__pyx_v_self->path); + __pyx_v_self->path = ((PyObject*)Py_None); + + /* function exit code */ + __pyx_r = 0; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.from_py":13 + * + * @cname("__pyx_convert_string_from_py_std__in_string") + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< + * cdef Py_ssize_t length + * cdef const char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + */ + +static std::string __pyx_convert_string_from_py_std__in_string(PyObject *__pyx_v_o) { + Py_ssize_t __pyx_v_length; + char const *__pyx_v_data; + std::string __pyx_r; + __Pyx_RefNannyDeclarations + char const *__pyx_t_1; + __Pyx_RefNannySetupContext("__pyx_convert_string_from_py_std__in_string", 0); + + /* "string.from_py":15 + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: + * cdef Py_ssize_t length + * cdef const char* data = __Pyx_PyObject_AsStringAndSize(o, &length) # <<<<<<<<<<<<<< + * return string(data, length) + * + */ + __pyx_t_1 = __Pyx_PyObject_AsStringAndSize(__pyx_v_o, (&__pyx_v_length)); if (unlikely(__pyx_t_1 == ((char const *)NULL))) __PYX_ERR(1, 15, __pyx_L1_error) + __pyx_v_data = __pyx_t_1; + + /* "string.from_py":16 + * cdef Py_ssize_t length + * cdef const char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + * return string(data, length) # <<<<<<<<<<<<<< + * + * + */ + __pyx_r = std::string(__pyx_v_data, __pyx_v_length); + goto __pyx_L0; + + /* "string.from_py":13 + * + * @cname("__pyx_convert_string_from_py_std__in_string") + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< + * cdef Py_ssize_t length + * cdef const char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("string.from_py.__pyx_convert_string_from_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_pretend_to_initialize(&__pyx_r); + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.to_py":31 + * + * @cname("__pyx_convert_PyObject_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyObject_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyObject_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + +static CYTHON_INLINE PyObject *__pyx_convert_PyObject_string_to_py_std__in_string(std::string const &__pyx_v_s) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_PyObject_string_to_py_std__in_string", 0); + + /* "string.to_py":32 + * @cname("__pyx_convert_PyObject_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyObject_string_to_py_std__in_string(const string& s): + * return __Pyx_PyObject_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< + * cdef extern from *: + * cdef object __Pyx_PyUnicode_FromStringAndSize(const char*, size_t) + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 32, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "string.to_py":31 + * + * @cname("__pyx_convert_PyObject_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyObject_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyObject_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("string.to_py.__pyx_convert_PyObject_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.to_py":37 + * + * @cname("__pyx_convert_PyUnicode_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyUnicode_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyUnicode_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + +static CYTHON_INLINE PyObject *__pyx_convert_PyUnicode_string_to_py_std__in_string(std::string const &__pyx_v_s) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_PyUnicode_string_to_py_std__in_string", 0); + + /* "string.to_py":38 + * @cname("__pyx_convert_PyUnicode_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyUnicode_string_to_py_std__in_string(const string& s): + * return __Pyx_PyUnicode_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< + * cdef extern from *: + * cdef object __Pyx_PyStr_FromStringAndSize(const char*, size_t) + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyUnicode_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 38, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "string.to_py":37 + * + * @cname("__pyx_convert_PyUnicode_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyUnicode_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyUnicode_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("string.to_py.__pyx_convert_PyUnicode_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.to_py":43 + * + * @cname("__pyx_convert_PyStr_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyStr_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyStr_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + +static CYTHON_INLINE PyObject *__pyx_convert_PyStr_string_to_py_std__in_string(std::string const &__pyx_v_s) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_PyStr_string_to_py_std__in_string", 0); + + /* "string.to_py":44 + * @cname("__pyx_convert_PyStr_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyStr_string_to_py_std__in_string(const string& s): + * return __Pyx_PyStr_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< + * cdef extern from *: + * cdef object __Pyx_PyBytes_FromStringAndSize(const char*, size_t) + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyStr_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 44, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "string.to_py":43 + * + * @cname("__pyx_convert_PyStr_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyStr_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyStr_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("string.to_py.__pyx_convert_PyStr_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.to_py":49 + * + * @cname("__pyx_convert_PyBytes_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyBytes_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyBytes_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + +static CYTHON_INLINE PyObject *__pyx_convert_PyBytes_string_to_py_std__in_string(std::string const &__pyx_v_s) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_PyBytes_string_to_py_std__in_string", 0); + + /* "string.to_py":50 + * @cname("__pyx_convert_PyBytes_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyBytes_string_to_py_std__in_string(const string& s): + * return __Pyx_PyBytes_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< + * cdef extern from *: + * cdef object __Pyx_PyByteArray_FromStringAndSize(const char*, size_t) + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 50, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "string.to_py":49 + * + * @cname("__pyx_convert_PyBytes_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyBytes_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyBytes_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("string.to_py.__pyx_convert_PyBytes_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.to_py":55 + * + * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyByteArray_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) + * + */ + +static CYTHON_INLINE PyObject *__pyx_convert_PyByteArray_string_to_py_std__in_string(std::string const &__pyx_v_s) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_PyByteArray_string_to_py_std__in_string", 0); + + /* "string.to_py":56 + * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyByteArray_string_to_py_std__in_string(const string& s): + * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< + * + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyByteArray_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 56, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "string.to_py":55 + * + * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyByteArray_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("string.to_py.__pyx_convert_PyByteArray_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_tp_new_5kenlm_Output(PyTypeObject *t, PyObject *a, PyObject *k) { + PyObject *o; + if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { + o = (*t->tp_alloc)(t, 0); + } else { + o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); + } + if (unlikely(!o)) return 0; + if (unlikely(__pyx_pw_5kenlm_6Output_1__cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_5kenlm_Output(PyObject *o) { + #if CYTHON_USE_TP_FINALIZE + if (unlikely(PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE) && Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + #endif + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + ++Py_REFCNT(o); + __pyx_pw_5kenlm_6Output_5__dealloc__(o); + --Py_REFCNT(o); + PyErr_Restore(etype, eval, etb); + } + (*Py_TYPE(o)->tp_free)(o); +} + +static PyMethodDef __pyx_methods_5kenlm_Output[] = { + {"Add", (PyCFunction)__pyx_pw_5kenlm_6Output_3Add, METH_VARARGS|METH_KEYWORDS, 0}, + {"__reduce_cython__", (PyCFunction)__pyx_pw_5kenlm_6Output_7__reduce_cython__, METH_NOARGS, 0}, + {"__setstate_cython__", (PyCFunction)__pyx_pw_5kenlm_6Output_9__setstate_cython__, METH_O, 0}, + {0, 0, 0, 0} +}; + +static PyTypeObject __pyx_type_5kenlm_Output = { + PyVarObject_HEAD_INIT(0, 0) + "kenlm.Output", /*tp_name*/ + sizeof(struct __pyx_obj_5kenlm_Output), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5kenlm_Output, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "\n Wrapper around lm::builder::Output.\n ", /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5kenlm_Output, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5kenlm_Output, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static PyObject *__pyx_tp_new_5kenlm_PrintHook(PyTypeObject *t, PyObject *a, PyObject *k) { + PyObject *o; + if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { + o = (*t->tp_alloc)(t, 0); + } else { + o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); + } + if (unlikely(!o)) return 0; + if (unlikely(__pyx_pw_5kenlm_9PrintHook_1__cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_5kenlm_PrintHook(PyObject *o) { + #if CYTHON_USE_TP_FINALIZE + if (unlikely(PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE) && Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + #endif + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + ++Py_REFCNT(o); + __pyx_pw_5kenlm_9PrintHook_3__dealloc__(o); + --Py_REFCNT(o); + PyErr_Restore(etype, eval, etb); + } + (*Py_TYPE(o)->tp_free)(o); +} + +static PyMethodDef __pyx_methods_5kenlm_PrintHook[] = { + {"__reduce_cython__", (PyCFunction)__pyx_pw_5kenlm_9PrintHook_5__reduce_cython__, METH_NOARGS, 0}, + {"__setstate_cython__", (PyCFunction)__pyx_pw_5kenlm_9PrintHook_7__setstate_cython__, METH_O, 0}, + {0, 0, 0, 0} +}; + +static PyTypeObject __pyx_type_5kenlm_PrintHook = { + PyVarObject_HEAD_INIT(0, 0) + "kenlm.PrintHook", /*tp_name*/ + sizeof(struct __pyx_obj_5kenlm_PrintHook), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5kenlm_PrintHook, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "\n Wrapper around lm::builder::Printhook\n ", /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5kenlm_PrintHook, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5kenlm_PrintHook, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static PyObject *__pyx_tp_new_5kenlm_FullScoreReturn(PyTypeObject *t, PyObject *a, PyObject *k) { + PyObject *o; + if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { + o = (*t->tp_alloc)(t, 0); + } else { + o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); + } + if (unlikely(!o)) return 0; + if (unlikely(__pyx_pw_5kenlm_15FullScoreReturn_1__cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_5kenlm_FullScoreReturn(PyObject *o) { + #if CYTHON_USE_TP_FINALIZE + if (unlikely(PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE) && Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + #endif + (*Py_TYPE(o)->tp_free)(o); +} + +static PyObject *__pyx_getprop_5kenlm_15FullScoreReturn_log_prob(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5kenlm_15FullScoreReturn_8log_prob_1__get__(o); +} + +static PyObject *__pyx_getprop_5kenlm_15FullScoreReturn_ngram_length(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5kenlm_15FullScoreReturn_12ngram_length_1__get__(o); +} + +static PyObject *__pyx_getprop_5kenlm_15FullScoreReturn_oov(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5kenlm_15FullScoreReturn_3oov_1__get__(o); +} + +static PyMethodDef __pyx_methods_5kenlm_FullScoreReturn[] = { + {"__reduce_cython__", (PyCFunction)__pyx_pw_5kenlm_15FullScoreReturn_5__reduce_cython__, METH_NOARGS, 0}, + {"__setstate_cython__", (PyCFunction)__pyx_pw_5kenlm_15FullScoreReturn_7__setstate_cython__, METH_O, 0}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_5kenlm_FullScoreReturn[] = { + {(char *)"log_prob", __pyx_getprop_5kenlm_15FullScoreReturn_log_prob, 0, (char *)0, 0}, + {(char *)"ngram_length", __pyx_getprop_5kenlm_15FullScoreReturn_ngram_length, 0, (char *)0, 0}, + {(char *)"oov", __pyx_getprop_5kenlm_15FullScoreReturn_oov, 0, (char *)0, 0}, + {0, 0, 0, 0, 0} +}; + +static PyTypeObject __pyx_type_5kenlm_FullScoreReturn = { + PyVarObject_HEAD_INIT(0, 0) + "kenlm.FullScoreReturn", /*tp_name*/ + sizeof(struct __pyx_obj_5kenlm_FullScoreReturn), /*tp_basicsize*/ + 0, /*tp_itemsize*/ __pyx_tp_dealloc_5kenlm_FullScoreReturn, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ @@ -4652,8 +8323,8 @@ static PyObject *__pyx_tp_new_5kenlm_State(PyTypeObject *t, CYTHON_UNUSED PyObje static void __pyx_tp_dealloc_5kenlm_State(PyObject *o) { struct __pyx_obj_5kenlm_State *p = (struct __pyx_obj_5kenlm_State *)o; - #if PY_VERSION_HEX >= 0x030400a1 - if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { + #if CYTHON_USE_TP_FINALIZE + if (unlikely(PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE) && Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { if (PyObject_CallFinalizerFromDealloc(o)) return; } #endif @@ -4662,6 +8333,8 @@ static void __pyx_tp_dealloc_5kenlm_State(PyObject *o) { } static PyMethodDef __pyx_methods_5kenlm_State[] = { + {"__reduce_cython__", (PyCFunction)__pyx_pw_5kenlm_5State_5__reduce_cython__, METH_NOARGS, 0}, + {"__setstate_cython__", (PyCFunction)__pyx_pw_5kenlm_5State_7__setstate_cython__, METH_O, 0}, {0, 0, 0, 0} }; @@ -4739,8 +8412,8 @@ static PyObject *__pyx_tp_new_5kenlm_Config(PyTypeObject *t, CYTHON_UNUSED PyObj static void __pyx_tp_dealloc_5kenlm_Config(PyObject *o) { struct __pyx_obj_5kenlm_Config *p = (struct __pyx_obj_5kenlm_Config *)o; - #if PY_VERSION_HEX >= 0x030400a1 - if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { + #if CYTHON_USE_TP_FINALIZE + if (unlikely(PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE) && Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { if (PyObject_CallFinalizerFromDealloc(o)) return; } #endif @@ -4763,6 +8436,8 @@ static int __pyx_setprop_5kenlm_6Config_load_method(PyObject *o, PyObject *v, CY } static PyMethodDef __pyx_methods_5kenlm_Config[] = { + {"__reduce_cython__", (PyCFunction)__pyx_pw_5kenlm_6Config_3__reduce_cython__, METH_NOARGS, 0}, + {"__setstate_cython__", (PyCFunction)__pyx_pw_5kenlm_6Config_5__setstate_cython__, METH_O, 0}, {0, 0, 0, 0} }; @@ -4845,8 +8520,8 @@ static PyObject *__pyx_tp_new_5kenlm_Model(PyTypeObject *t, CYTHON_UNUSED PyObje static void __pyx_tp_dealloc_5kenlm_Model(PyObject *o) { struct __pyx_obj_5kenlm_Model *p = (struct __pyx_obj_5kenlm_Model *)o; - #if PY_VERSION_HEX >= 0x030400a1 - if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { + #if CYTHON_USE_TP_FINALIZE + if (unlikely(PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE) && Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { if (PyObject_CallFinalizerFromDealloc(o)) return; } #endif @@ -5018,7 +8693,7 @@ static int __pyx_tp_traverse_5kenlm___pyx_scope_struct__full_scores(PyObject *o, e = (*v)(p->__pyx_v_eos, a); if (e) return e; } if (p->__pyx_v_self) { - e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; + e = (*v)(((PyObject *)p->__pyx_v_self), a); if (e) return e; } if (p->__pyx_v_sentence) { e = (*v)(p->__pyx_v_sentence, a); if (e) return e; @@ -5035,33 +8710,6 @@ static int __pyx_tp_traverse_5kenlm___pyx_scope_struct__full_scores(PyObject *o, return 0; } -static int __pyx_tp_clear_5kenlm___pyx_scope_struct__full_scores(PyObject *o) { - PyObject* tmp; - struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *p = (struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)o; - tmp = ((PyObject*)p->__pyx_v_bos); - p->__pyx_v_bos = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->__pyx_v_eos); - p->__pyx_v_eos = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->__pyx_v_self); - p->__pyx_v_self = ((struct __pyx_obj_5kenlm_Model *)Py_None); Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->__pyx_v_sentence); - p->__pyx_v_sentence = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->__pyx_v_word); - p->__pyx_v_word = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->__pyx_v_words); - p->__pyx_v_words = ((PyObject*)Py_None); Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->__pyx_t_0); - p->__pyx_t_0 = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - return 0; -} - static PyTypeObject __pyx_type_5kenlm___pyx_scope_struct__full_scores = { PyVarObject_HEAD_INIT(0, 0) "kenlm.__pyx_scope_struct__full_scores", /*tp_name*/ @@ -5090,7 +8738,7 @@ static PyTypeObject __pyx_type_5kenlm___pyx_scope_struct__full_scores = { Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ 0, /*tp_doc*/ __pyx_tp_traverse_5kenlm___pyx_scope_struct__full_scores, /*tp_traverse*/ - __pyx_tp_clear_5kenlm___pyx_scope_struct__full_scores, /*tp_clear*/ + 0, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ 0, /*tp_iter*/ @@ -5125,17 +8773,31 @@ static PyMethodDef __pyx_methods[] = { }; #if PY_MAJOR_VERSION >= 3 +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_kenlm(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_kenlm}, + {0, NULL} +}; +#endif + static struct PyModuleDef __pyx_moduledef = { - #if PY_VERSION_HEX < 0x03020000 - { PyObject_HEAD_INIT(NULL) NULL, 0, NULL }, - #else PyModuleDef_HEAD_INIT, - #endif "kenlm", 0, /* m_doc */ + #if CYTHON_PEP489_MULTI_PHASE_INIT + 0, /* m_size */ + #else -1, /* m_size */ + #endif __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else NULL, /* m_reload */ + #endif NULL, /* m_traverse */ NULL, /* m_clear */ NULL /* m_free */ @@ -5144,9 +8806,14 @@ static struct PyModuleDef __pyx_moduledef = { static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_s_0_1_2_3, __pyx_k_0_1_2_3, sizeof(__pyx_k_0_1_2_3), 0, 0, 1, 0}, + {&__pyx_kp_s_1G, __pyx_k_1G, sizeof(__pyx_k_1G), 0, 0, 1, 0}, + {&__pyx_kp_s_64M, __pyx_k_64M, sizeof(__pyx_k_64M), 0, 0, 1, 0}, + {&__pyx_kp_s_8K, __pyx_k_8K, sizeof(__pyx_k_8K), 0, 0, 1, 0}, + {&__pyx_n_s_Add, __pyx_k_Add, sizeof(__pyx_k_Add), 0, 0, 1, 1}, {&__pyx_kp_s_Backwards_compatability_stub_Use, __pyx_k_Backwards_compatability_stub_Use, sizeof(__pyx_k_Backwards_compatability_stub_Use), 0, 0, 1, 0}, {&__pyx_kp_s_Cannot_convert_s_to_string, __pyx_k_Cannot_convert_s_to_string, sizeof(__pyx_k_Cannot_convert_s_to_string), 0, 0, 1, 0}, {&__pyx_kp_s_Cannot_read_model, __pyx_k_Cannot_read_model, sizeof(__pyx_k_Cannot_read_model), 0, 0, 1, 0}, + {&__pyx_kp_s_Fallback_discounting_enabled_but, __pyx_k_Fallback_discounting_enabled_but, sizeof(__pyx_k_Fallback_discounting_enabled_but), 0, 0, 1, 0}, {&__pyx_n_s_IOError, __pyx_k_IOError, sizeof(__pyx_k_IOError), 0, 0, 1, 1}, {&__pyx_n_s_LAZY, __pyx_k_LAZY, sizeof(__pyx_k_LAZY), 0, 0, 1, 1}, {&__pyx_n_s_LanguageModel, __pyx_k_LanguageModel, sizeof(__pyx_k_LanguageModel), 0, 0, 1, 1}, @@ -5158,51 +8825,104 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_POPULATE_OR_READ, __pyx_k_POPULATE_OR_READ, sizeof(__pyx_k_POPULATE_OR_READ), 0, 0, 1, 1}, {&__pyx_n_s_READ, __pyx_k_READ, sizeof(__pyx_k_READ), 0, 0, 1, 1}, {&__pyx_n_s_RuntimeError, __pyx_k_RuntimeError, sizeof(__pyx_k_RuntimeError), 0, 0, 1, 1}, + {&__pyx_kp_s_Specify_at_most_three_fallback_d, __pyx_k_Specify_at_most_three_fallback_d, sizeof(__pyx_k_Specify_at_most_three_fallback_d), 0, 0, 1, 0}, + {&__pyx_kp_s_The_discount_for_count, __pyx_k_The_discount_for_count, sizeof(__pyx_k_The_discount_for_count), 0, 0, 1, 0}, {&__pyx_n_s_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 0, 0, 1, 1}, - {&__pyx_kp_s__3, __pyx_k__3, sizeof(__pyx_k__3), 0, 0, 1, 0}, - {&__pyx_kp_s__4, __pyx_k__4, sizeof(__pyx_k__4), 0, 0, 1, 0}, + {&__pyx_kp_s_You_specified_pruning_thresholds, __pyx_k_You_specified_pruning_thresholds, sizeof(__pyx_k_You_specified_pruning_thresholds), 0, 0, 1, 0}, + {&__pyx_kp_s__14, __pyx_k__14, sizeof(__pyx_k__14), 0, 0, 1, 0}, + {&__pyx_kp_s__22, __pyx_k__22, sizeof(__pyx_k__22), 0, 0, 1, 0}, + {&__pyx_kp_s__23, __pyx_k__23, sizeof(__pyx_k__23), 0, 0, 1, 0}, + {&__pyx_kp_s__7, __pyx_k__7, sizeof(__pyx_k__7), 0, 0, 1, 0}, {&__pyx_n_s_abspath, __pyx_k_abspath, sizeof(__pyx_k_abspath), 0, 0, 1, 1}, {&__pyx_n_s_args, __pyx_k_args, sizeof(__pyx_k_args), 0, 0, 1, 1}, {&__pyx_n_s_basename, __pyx_k_basename, sizeof(__pyx_k_basename), 0, 0, 1, 1}, + {&__pyx_n_s_block_count, __pyx_k_block_count, sizeof(__pyx_k_block_count), 0, 0, 1, 1}, {&__pyx_n_s_bos, __pyx_k_bos, sizeof(__pyx_k_bos), 0, 0, 1, 1}, + {&__pyx_kp_s_but_the_model_only_has_order, __pyx_k_but_the_model_only_has_order, sizeof(__pyx_k_but_the_model_only_has_order), 0, 0, 1, 0}, {&__pyx_n_s_class, __pyx_k_class, sizeof(__pyx_k_class), 0, 0, 1, 1}, + {&__pyx_n_s_cline_in_traceback, __pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 0, 1, 1}, {&__pyx_n_s_close, __pyx_k_close, sizeof(__pyx_k_close), 0, 0, 1, 1}, + {&__pyx_n_s_collapse_values, __pyx_k_collapse_values, sizeof(__pyx_k_collapse_values), 0, 0, 1, 1}, {&__pyx_n_s_config, __pyx_k_config, sizeof(__pyx_k_config), 0, 0, 1, 1}, + {&__pyx_n_s_discount, __pyx_k_discount, sizeof(__pyx_k_discount), 0, 0, 1, 1}, + {&__pyx_n_s_discount_fallback, __pyx_k_discount_fallback, sizeof(__pyx_k_discount_fallback), 0, 0, 1, 1}, {&__pyx_n_s_doc, __pyx_k_doc, sizeof(__pyx_k_doc), 0, 0, 1, 1}, + {&__pyx_n_s_e, __pyx_k_e, sizeof(__pyx_k_e), 0, 0, 1, 1}, {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, {&__pyx_n_s_eos, __pyx_k_eos, sizeof(__pyx_k_eos), 0, 0, 1, 1}, + {&__pyx_n_s_file_base, __pyx_k_file_base, sizeof(__pyx_k_file_base), 0, 0, 1, 1}, {&__pyx_n_s_format, __pyx_k_format, sizeof(__pyx_k_format), 0, 0, 1, 1}, {&__pyx_n_s_full_scores, __pyx_k_full_scores, sizeof(__pyx_k_full_scores), 0, 0, 1, 1}, + {&__pyx_n_s_getstate, __pyx_k_getstate, sizeof(__pyx_k_getstate), 0, 0, 1, 1}, + {&__pyx_n_s_i, __pyx_k_i, sizeof(__pyx_k_i), 0, 0, 1, 1}, {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, + {&__pyx_n_s_in, __pyx_k_in, sizeof(__pyx_k_in), 0, 0, 1, 1}, {&__pyx_n_s_in_state, __pyx_k_in_state, sizeof(__pyx_k_in_state), 0, 0, 1, 1}, + {&__pyx_n_s_intermediate, __pyx_k_intermediate, sizeof(__pyx_k_intermediate), 0, 0, 1, 1}, + {&__pyx_n_s_interpolate_unigrams, __pyx_k_interpolate_unigrams, sizeof(__pyx_k_interpolate_unigrams), 0, 0, 1, 1}, + {&__pyx_n_s_keep_buffer, __pyx_k_keep_buffer, sizeof(__pyx_k_keep_buffer), 0, 0, 1, 1}, {&__pyx_n_s_kenlm, __pyx_k_kenlm, sizeof(__pyx_k_kenlm), 0, 0, 1, 1}, + {&__pyx_n_s_limit_vocab_file, __pyx_k_limit_vocab_file, sizeof(__pyx_k_limit_vocab_file), 0, 0, 1, 1}, + {&__pyx_n_s_lmplz, __pyx_k_lmplz, sizeof(__pyx_k_lmplz), 0, 0, 1, 1}, {&__pyx_n_s_log_prob, __pyx_k_log_prob, sizeof(__pyx_k_log_prob), 0, 0, 1, 1}, {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, + {&__pyx_n_s_memory, __pyx_k_memory, sizeof(__pyx_k_memory), 0, 0, 1, 1}, {&__pyx_n_s_metaclass, __pyx_k_metaclass, sizeof(__pyx_k_metaclass), 0, 0, 1, 1}, + {&__pyx_n_s_minimum_block, __pyx_k_minimum_block, sizeof(__pyx_k_minimum_block), 0, 0, 1, 1}, {&__pyx_n_s_module, __pyx_k_module, sizeof(__pyx_k_module), 0, 0, 1, 1}, {&__pyx_n_s_name, __pyx_k_name, sizeof(__pyx_k_name), 0, 0, 1, 1}, {&__pyx_n_s_ngram_length, __pyx_k_ngram_length, sizeof(__pyx_k_ngram_length), 0, 0, 1, 1}, + {&__pyx_kp_s_no_default___reduce___due_to_non, __pyx_k_no_default___reduce___due_to_non, sizeof(__pyx_k_no_default___reduce___due_to_non), 0, 0, 1, 0}, {&__pyx_n_s_oov, __pyx_k_oov, sizeof(__pyx_k_oov), 0, 0, 1, 1}, + {&__pyx_n_s_order, __pyx_k_order, sizeof(__pyx_k_order), 0, 0, 1, 1}, {&__pyx_n_s_os, __pyx_k_os, sizeof(__pyx_k_os), 0, 0, 1, 1}, + {&__pyx_n_s_out, __pyx_k_out, sizeof(__pyx_k_out), 0, 0, 1, 1}, {&__pyx_n_s_out_state, __pyx_k_out_state, sizeof(__pyx_k_out_state), 0, 0, 1, 1}, + {&__pyx_n_s_output, __pyx_k_output, sizeof(__pyx_k_output), 0, 0, 1, 1}, + {&__pyx_n_s_output_q, __pyx_k_output_q, sizeof(__pyx_k_output_q), 0, 0, 1, 1}, {&__pyx_n_s_path, __pyx_k_path, sizeof(__pyx_k_path), 0, 0, 1, 1}, + {&__pyx_n_s_path_arpa_file, __pyx_k_path_arpa_file, sizeof(__pyx_k_path_arpa_file), 0, 0, 1, 1}, + {&__pyx_n_s_path_text_file, __pyx_k_path_text_file, sizeof(__pyx_k_path_text_file), 0, 0, 1, 1}, + {&__pyx_n_s_pipeline, __pyx_k_pipeline, sizeof(__pyx_k_pipeline), 0, 0, 1, 1}, {&__pyx_n_s_prepare, __pyx_k_prepare, sizeof(__pyx_k_prepare), 0, 0, 1, 1}, + {&__pyx_n_s_pruning, __pyx_k_pruning, sizeof(__pyx_k_pruning), 0, 0, 1, 1}, + {&__pyx_kp_s_python_kenlm_pyx, __pyx_k_python_kenlm_pyx, sizeof(__pyx_k_python_kenlm_pyx), 0, 0, 1, 0}, {&__pyx_n_s_qualname, __pyx_k_qualname, sizeof(__pyx_k_qualname), 0, 0, 1, 1}, + {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, + {&__pyx_n_s_reduce, __pyx_k_reduce, sizeof(__pyx_k_reduce), 0, 0, 1, 1}, + {&__pyx_n_s_reduce_cython, __pyx_k_reduce_cython, sizeof(__pyx_k_reduce_cython), 0, 0, 1, 1}, + {&__pyx_n_s_reduce_ex, __pyx_k_reduce_ex, sizeof(__pyx_k_reduce_ex), 0, 0, 1, 1}, + {&__pyx_n_s_renumber, __pyx_k_renumber, sizeof(__pyx_k_renumber), 0, 0, 1, 1}, {&__pyx_n_s_replace, __pyx_k_replace, sizeof(__pyx_k_replace), 0, 0, 1, 1}, {&__pyx_n_s_score, __pyx_k_score, sizeof(__pyx_k_score), 0, 0, 1, 1}, + {&__pyx_kp_s_self__c_config_cannot_be_convert, __pyx_k_self__c_config_cannot_be_convert, sizeof(__pyx_k_self__c_config_cannot_be_convert), 0, 0, 1, 0}, + {&__pyx_kp_s_self__c_state_cannot_be_converte, __pyx_k_self__c_state_cannot_be_converte, sizeof(__pyx_k_self__c_state_cannot_be_converte), 0, 0, 1, 0}, {&__pyx_n_s_send, __pyx_k_send, sizeof(__pyx_k_send), 0, 0, 1, 1}, {&__pyx_n_s_sentence, __pyx_k_sentence, sizeof(__pyx_k_sentence), 0, 0, 1, 1}, + {&__pyx_n_s_setstate, __pyx_k_setstate, sizeof(__pyx_k_setstate), 0, 0, 1, 1}, + {&__pyx_n_s_setstate_cython, __pyx_k_setstate_cython, sizeof(__pyx_k_setstate_cython), 0, 0, 1, 1}, + {&__pyx_n_s_skip_symbols, __pyx_k_skip_symbols, sizeof(__pyx_k_skip_symbols), 0, 0, 1, 1}, + {&__pyx_n_s_sort_block, __pyx_k_sort_block, sizeof(__pyx_k_sort_block), 0, 0, 1, 1}, {&__pyx_n_s_split, __pyx_k_split, sizeof(__pyx_k_split), 0, 0, 1, 1}, + {&__pyx_n_s_temp_prefix, __pyx_k_temp_prefix, sizeof(__pyx_k_temp_prefix), 0, 0, 1, 1}, {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, {&__pyx_n_s_throw, __pyx_k_throw, sizeof(__pyx_k_throw), 0, 0, 1, 1}, {&__pyx_n_s_utf8, __pyx_k_utf8, sizeof(__pyx_k_utf8), 0, 0, 1, 1}, + {&__pyx_n_s_verbose_header, __pyx_k_verbose_header, sizeof(__pyx_k_verbose_header), 0, 0, 1, 1}, + {&__pyx_n_s_vocab_estimate, __pyx_k_vocab_estimate, sizeof(__pyx_k_vocab_estimate), 0, 0, 1, 1}, + {&__pyx_n_s_vocab_pad, __pyx_k_vocab_pad, sizeof(__pyx_k_vocab_pad), 0, 0, 1, 1}, + {&__pyx_kp_s_vocab_pad_requires_interpolate, __pyx_k_vocab_pad_requires_interpolate, sizeof(__pyx_k_vocab_pad_requires_interpolate), 0, 0, 1, 0}, + {&__pyx_kp_s_was_parsed_as, __pyx_k_was_parsed_as, sizeof(__pyx_k_was_parsed_as), 0, 0, 1, 0}, + {&__pyx_kp_s_which_is_not_in_the_range_0, __pyx_k_which_is_not_in_the_range_0, sizeof(__pyx_k_which_is_not_in_the_range_0), 0, 0, 1, 0}, {&__pyx_n_s_word, __pyx_k_word, sizeof(__pyx_k_word), 0, 0, 1, 1}, + {&__pyx_n_s_write_fd, __pyx_k_write_fd, sizeof(__pyx_k_write_fd), 0, 0, 1, 1}, {0, 0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { __pyx_builtin_TypeError = __Pyx_GetBuiltinName(__pyx_n_s_TypeError); if (!__pyx_builtin_TypeError) __PYX_ERR(0, 9, __pyx_L1_error) - __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_n_s_RuntimeError); if (!__pyx_builtin_RuntimeError) __PYX_ERR(0, 115, __pyx_L1_error) - __pyx_builtin_IOError = __Pyx_GetBuiltinName(__pyx_n_s_IOError); if (!__pyx_builtin_IOError) __PYX_ERR(0, 117, __pyx_L1_error) + __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_n_s_RuntimeError); if (!__pyx_builtin_RuntimeError) __PYX_ERR(0, 85, __pyx_L1_error) + __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_builtin_IOError = __Pyx_GetBuiltinName(__pyx_n_s_IOError); if (!__pyx_builtin_IOError) __PYX_ERR(0, 276, __pyx_L1_error) return 0; __pyx_L1_error:; return -1; @@ -5223,16 +8943,178 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple_); __Pyx_GIVEREF(__pyx_tuple_); - /* "kenlm.pyx":116 + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + */ + __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_no_default___reduce___due_to_non); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__2); + __Pyx_GIVEREF(__pyx_tuple__2); + + /* "(tree fragment)":4 + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< + */ + __pyx_tuple__3 = PyTuple_Pack(1, __pyx_kp_s_no_default___reduce___due_to_non); if (unlikely(!__pyx_tuple__3)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__3); + __Pyx_GIVEREF(__pyx_tuple__3); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + */ + __pyx_tuple__4 = PyTuple_Pack(1, __pyx_kp_s_no_default___reduce___due_to_non); if (unlikely(!__pyx_tuple__4)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__4); + __Pyx_GIVEREF(__pyx_tuple__4); + + /* "(tree fragment)":4 + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< + */ + __pyx_tuple__5 = PyTuple_Pack(1, __pyx_kp_s_no_default___reduce___due_to_non); if (unlikely(!__pyx_tuple__5)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__5); + __Pyx_GIVEREF(__pyx_tuple__5); + + /* "kenlm.pyx":85 + * + * if pipeline.vocab_size_for_unk and not pipeline.initial_probs.interpolate_unigrams: + * raise RuntimeError('--vocab_pad requires --interpolate_unigrams be on') # <<<<<<<<<<<<<< + * + * if skip_symbols: + */ + __pyx_tuple__9 = PyTuple_Pack(1, __pyx_kp_s_vocab_pad_requires_interpolate); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(0, 85, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__9); + __Pyx_GIVEREF(__pyx_tuple__9); + + /* "kenlm.pyx":92 + * pipeline.disallowed_symbol_action = _kenlm.THROW_UP + * + * for i in range(4): # <<<<<<<<<<<<<< + * pipeline.discount.fallback.amount[i] = 0.0 + * + */ + __pyx_tuple__10 = PyTuple_Pack(1, __pyx_int_4); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(0, 92, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__10); + __Pyx_GIVEREF(__pyx_tuple__10); + + /* "kenlm.pyx":99 + * else: + * if len(discount_fallback) > 3: + * raise RuntimeError("Specify at most three fallback discounts: 1, 2, and 3+") # <<<<<<<<<<<<<< + * + * if len(discount_fallback) == 0: + */ + __pyx_tuple__11 = PyTuple_Pack(1, __pyx_kp_s_Specify_at_most_three_fallback_d); if (unlikely(!__pyx_tuple__11)) __PYX_ERR(0, 99, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__11); + __Pyx_GIVEREF(__pyx_tuple__11); + + /* "kenlm.pyx":102 + * + * if len(discount_fallback) == 0: + * raise RuntimeError("Fallback discounting enabled, but no discount specified") # <<<<<<<<<<<<<< + * + * pipeline.discount.fallback.amount[0] = 0.0 + */ + __pyx_tuple__12 = PyTuple_Pack(1, __pyx_kp_s_Fallback_discounting_enabled_but); if (unlikely(!__pyx_tuple__12)) __PYX_ERR(0, 102, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__12); + __Pyx_GIVEREF(__pyx_tuple__12); + + /* "kenlm.pyx":106 + * pipeline.discount.fallback.amount[0] = 0.0 + * + * for i in range(3): # <<<<<<<<<<<<<< + * discount = discount_fallback[len(discount_fallback) - 1] + * if i < len(discount_fallback): + */ + __pyx_tuple__13 = PyTuple_Pack(1, __pyx_int_3); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(0, 106, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__13); + __Pyx_GIVEREF(__pyx_tuple__13); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + */ + __pyx_tuple__15 = PyTuple_Pack(1, __pyx_kp_s_no_default___reduce___due_to_non); if (unlikely(!__pyx_tuple__15)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__15); + __Pyx_GIVEREF(__pyx_tuple__15); + + /* "(tree fragment)":4 + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("no default __reduce__ due to non-trivial __cinit__") # <<<<<<<<<<<<<< + */ + __pyx_tuple__16 = PyTuple_Pack(1, __pyx_kp_s_no_default___reduce___due_to_non); if (unlikely(!__pyx_tuple__16)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__16); + __Pyx_GIVEREF(__pyx_tuple__16); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("self._c_state cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self._c_state cannot be converted to a Python object for pickling") + */ + __pyx_tuple__17 = PyTuple_Pack(1, __pyx_kp_s_self__c_state_cannot_be_converte); if (unlikely(!__pyx_tuple__17)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__17); + __Pyx_GIVEREF(__pyx_tuple__17); + + /* "(tree fragment)":4 + * raise TypeError("self._c_state cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self._c_state cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + */ + __pyx_tuple__18 = PyTuple_Pack(1, __pyx_kp_s_self__c_state_cannot_be_converte); if (unlikely(!__pyx_tuple__18)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__18); + __Pyx_GIVEREF(__pyx_tuple__18); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("self._c_config cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self._c_config cannot be converted to a Python object for pickling") + */ + __pyx_tuple__19 = PyTuple_Pack(1, __pyx_kp_s_self__c_config_cannot_be_convert); if (unlikely(!__pyx_tuple__19)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__19); + __Pyx_GIVEREF(__pyx_tuple__19); + + /* "(tree fragment)":4 + * raise TypeError("self._c_config cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self._c_config cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + */ + __pyx_tuple__20 = PyTuple_Pack(1, __pyx_kp_s_self__c_config_cannot_be_convert); if (unlikely(!__pyx_tuple__20)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__20); + __Pyx_GIVEREF(__pyx_tuple__20); + + /* "kenlm.pyx":275 * self.model = _kenlm.LoadVirtual(self.path, config._c_config) * except RuntimeError as exception: * exception_message = str(exception).replace('\n', ' ') # <<<<<<<<<<<<<< * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ * from exception */ - __pyx_tuple__5 = PyTuple_Pack(2, __pyx_kp_s__3, __pyx_kp_s__4); if (unlikely(!__pyx_tuple__5)) __PYX_ERR(0, 116, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__5); - __Pyx_GIVEREF(__pyx_tuple__5); + __pyx_tuple__24 = PyTuple_Pack(2, __pyx_kp_s__22, __pyx_kp_s__23); if (unlikely(!__pyx_tuple__24)) __PYX_ERR(0, 275, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__24); + __Pyx_GIVEREF(__pyx_tuple__24); + + /* "kenlm.pyx":42 + * + * + * def lmplz( # <<<<<<<<<<<<<< + * path_text_file, path_arpa_file, + * order=3, + */ + __pyx_tuple__25 = PyTuple_Pack(26, __pyx_n_s_path_text_file, __pyx_n_s_path_arpa_file, __pyx_n_s_order, __pyx_n_s_interpolate_unigrams, __pyx_n_s_skip_symbols, __pyx_n_s_temp_prefix, __pyx_n_s_memory, __pyx_n_s_minimum_block, __pyx_n_s_sort_block, __pyx_n_s_block_count, __pyx_n_s_vocab_estimate, __pyx_n_s_vocab_pad, __pyx_n_s_verbose_header, __pyx_n_s_intermediate, __pyx_n_s_renumber, __pyx_n_s_collapse_values, __pyx_n_s_pruning, __pyx_n_s_limit_vocab_file, __pyx_n_s_discount_fallback, __pyx_n_s_pipeline, __pyx_n_s_i, __pyx_n_s_discount, __pyx_n_s_e, __pyx_n_s_in, __pyx_n_s_out, __pyx_n_s_output); if (unlikely(!__pyx_tuple__25)) __PYX_ERR(0, 42, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__25); + __Pyx_GIVEREF(__pyx_tuple__25); + __pyx_codeobj__26 = (PyObject*)__Pyx_PyCode_New(19, 0, 26, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__25, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_python_kenlm_pyx, __pyx_n_s_lmplz, 42, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__26)) __PYX_ERR(0, 42, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -5242,7 +9124,16 @@ static int __Pyx_InitCachedConstants(void) { static int __Pyx_InitGlobals(void) { if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + __pyx_float_0_0 = PyFloat_FromDouble(0.0); if (unlikely(!__pyx_float_0_0)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_float_0_5 = PyFloat_FromDouble(0.5); if (unlikely(!__pyx_float_0_5)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_float_1_5 = PyFloat_FromDouble(1.5); if (unlikely(!__pyx_float_1_5)) __PYX_ERR(0, 1, __pyx_L1_error) __pyx_float_10_0 = PyFloat_FromDouble(10.0); if (unlikely(!__pyx_float_10_0)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_int_0 = PyInt_FromLong(0); if (unlikely(!__pyx_int_0)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_int_1 = PyInt_FromLong(1); if (unlikely(!__pyx_int_1)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_int_2 = PyInt_FromLong(2); if (unlikely(!__pyx_int_2)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_int_3 = PyInt_FromLong(3); if (unlikely(!__pyx_int_3)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_int_4 = PyInt_FromLong(4); if (unlikely(!__pyx_int_4)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_int_1000000 = PyInt_FromLong(1000000L); if (unlikely(!__pyx_int_1000000)) __PYX_ERR(0, 1, __pyx_L1_error) return 0; __pyx_L1_error:; return -1; @@ -5254,6 +9145,47 @@ PyMODINIT_FUNC initkenlm(void) #else PyMODINIT_FUNC PyInit_kenlm(void); /*proto*/ PyMODINIT_FUNC PyInit_kenlm(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +static int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name) { + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + result = PyDict_SetItemString(moddict, to_name, value); + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static PyObject* __pyx_pymod_create(PyObject *spec, CYTHON_UNUSED PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__") < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__") < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__") < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__") < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static int __pyx_pymod_exec_kenlm(PyObject *__pyx_pyinit_module) +#endif #endif { PyObject *__pyx_t_1 = NULL; @@ -5261,6 +9193,9 @@ PyMODINIT_FUNC PyInit_kenlm(void) PyObject *__pyx_t_3 = NULL; PyObject *__pyx_t_4 = NULL; __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m && __pyx_m == __pyx_pyinit_module) return 0; + #endif #if CYTHON_REFNANNY __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); if (!__Pyx_RefNanny) { @@ -5287,6 +9222,9 @@ PyMODINIT_FUNC PyInit_kenlm(void) #ifdef __Pyx_Generator_USED if (__pyx_Generator_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_AsyncGen_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif #ifdef __Pyx_StopAsyncIteration_USED if (__pyx_StopAsyncIteration_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) #endif @@ -5298,15 +9236,21 @@ PyMODINIT_FUNC PyInit_kenlm(void) #endif #endif /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_m = __pyx_pyinit_module; + Py_INCREF(__pyx_m); + #else #if PY_MAJOR_VERSION < 3 __pyx_m = Py_InitModule4("kenlm", __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); #else __pyx_m = PyModule_Create(&__pyx_moduledef); #endif if (unlikely(!__pyx_m)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) Py_INCREF(__pyx_d); __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_cython_runtime = PyImport_AddModule((char *) "cython_runtime"); if (unlikely(!__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) #if CYTHON_COMPILING_IN_PYPY Py_INCREF(__pyx_b); #endif @@ -5335,23 +9279,36 @@ PyMODINIT_FUNC PyInit_kenlm(void) /*--- Variable export code ---*/ /*--- Function export code ---*/ /*--- Type init code ---*/ - if (PyType_Ready(&__pyx_type_5kenlm_FullScoreReturn) < 0) __PYX_ERR(0, 11, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5kenlm_Output) < 0) __PYX_ERR(0, 11, __pyx_L1_error) + __pyx_type_5kenlm_Output.tp_print = 0; + if (PyObject_SetAttrString(__pyx_m, "Output", (PyObject *)&__pyx_type_5kenlm_Output) < 0) __PYX_ERR(0, 11, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5kenlm_Output) < 0) __PYX_ERR(0, 11, __pyx_L1_error) + __pyx_ptype_5kenlm_Output = &__pyx_type_5kenlm_Output; + if (PyType_Ready(&__pyx_type_5kenlm_PrintHook) < 0) __PYX_ERR(0, 26, __pyx_L1_error) + __pyx_type_5kenlm_PrintHook.tp_print = 0; + if (PyObject_SetAttrString(__pyx_m, "PrintHook", (PyObject *)&__pyx_type_5kenlm_PrintHook) < 0) __PYX_ERR(0, 26, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5kenlm_PrintHook) < 0) __PYX_ERR(0, 26, __pyx_L1_error) + __pyx_ptype_5kenlm_PrintHook = &__pyx_type_5kenlm_PrintHook; + if (PyType_Ready(&__pyx_type_5kenlm_FullScoreReturn) < 0) __PYX_ERR(0, 170, __pyx_L1_error) __pyx_type_5kenlm_FullScoreReturn.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "FullScoreReturn", (PyObject *)&__pyx_type_5kenlm_FullScoreReturn) < 0) __PYX_ERR(0, 11, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "FullScoreReturn", (PyObject *)&__pyx_type_5kenlm_FullScoreReturn) < 0) __PYX_ERR(0, 170, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5kenlm_FullScoreReturn) < 0) __PYX_ERR(0, 170, __pyx_L1_error) __pyx_ptype_5kenlm_FullScoreReturn = &__pyx_type_5kenlm_FullScoreReturn; - if (PyType_Ready(&__pyx_type_5kenlm_State) < 0) __PYX_ERR(0, 44, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5kenlm_State) < 0) __PYX_ERR(0, 203, __pyx_L1_error) __pyx_type_5kenlm_State.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "State", (PyObject *)&__pyx_type_5kenlm_State) < 0) __PYX_ERR(0, 44, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "State", (PyObject *)&__pyx_type_5kenlm_State) < 0) __PYX_ERR(0, 203, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5kenlm_State) < 0) __PYX_ERR(0, 203, __pyx_L1_error) __pyx_ptype_5kenlm_State = &__pyx_type_5kenlm_State; - if (PyType_Ready(&__pyx_type_5kenlm_Config) < 0) __PYX_ERR(0, 80, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5kenlm_Config) < 0) __PYX_ERR(0, 239, __pyx_L1_error) __pyx_type_5kenlm_Config.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Config", (PyObject *)&__pyx_type_5kenlm_Config) < 0) __PYX_ERR(0, 80, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Config", (PyObject *)&__pyx_type_5kenlm_Config) < 0) __PYX_ERR(0, 239, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5kenlm_Config) < 0) __PYX_ERR(0, 239, __pyx_L1_error) __pyx_ptype_5kenlm_Config = &__pyx_type_5kenlm_Config; - if (PyType_Ready(&__pyx_type_5kenlm_Model) < 0) __PYX_ERR(0, 96, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5kenlm_Model) < 0) __PYX_ERR(0, 255, __pyx_L1_error) __pyx_type_5kenlm_Model.tp_print = 0; #if CYTHON_COMPILING_IN_CPYTHON { - PyObject *wrapper = PyObject_GetAttrString((PyObject *)&__pyx_type_5kenlm_Model, "__init__"); if (unlikely(!wrapper)) __PYX_ERR(0, 96, __pyx_L1_error) + PyObject *wrapper = PyObject_GetAttrString((PyObject *)&__pyx_type_5kenlm_Model, "__init__"); if (unlikely(!wrapper)) __PYX_ERR(0, 255, __pyx_L1_error) if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) { __pyx_wrapperbase_5kenlm_5Model___init__ = *((PyWrapperDescrObject *)wrapper)->d_base; __pyx_wrapperbase_5kenlm_5Model___init__.doc = __pyx_doc_5kenlm_5Model___init__; @@ -5359,9 +9316,9 @@ PyMODINIT_FUNC PyInit_kenlm(void) } } #endif - if (PyObject_SetAttrString(__pyx_m, "Model", (PyObject *)&__pyx_type_5kenlm_Model) < 0) __PYX_ERR(0, 96, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Model", (PyObject *)&__pyx_type_5kenlm_Model) < 0) __PYX_ERR(0, 255, __pyx_L1_error) __pyx_ptype_5kenlm_Model = &__pyx_type_5kenlm_Model; - if (PyType_Ready(&__pyx_type_5kenlm___pyx_scope_struct__full_scores) < 0) __PYX_ERR(0, 190, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5kenlm___pyx_scope_struct__full_scores) < 0) __PYX_ERR(0, 349, __pyx_L1_error) __pyx_type_5kenlm___pyx_scope_struct__full_scores.tp_print = 0; __pyx_ptype_5kenlm___pyx_scope_struct__full_scores = &__pyx_type_5kenlm___pyx_scope_struct__full_scores; /*--- Type import code ---*/ @@ -5382,120 +9339,167 @@ PyMODINIT_FUNC PyInit_kenlm(void) if (PyDict_SetItem(__pyx_d, __pyx_n_s_os, __pyx_t_1) < 0) __PYX_ERR(0, 1, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "kenlm.pyx":73 + /* "kenlm.pyx":58 + * renumber=False, + * collapse_values=False, + * pruning=[], # <<<<<<<<<<<<<< + * limit_vocab_file='', + * discount_fallback=[0.5, 1, 1.5]): + */ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 58, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_k__6 = __pyx_t_1; + __Pyx_GIVEREF(__pyx_t_1); + __pyx_t_1 = 0; + + /* "kenlm.pyx":60 + * pruning=[], + * limit_vocab_file='', + * discount_fallback=[0.5, 1, 1.5]): # <<<<<<<<<<<<<< + * + * cdef _kenlm.PipelineConfig pipeline + */ + __pyx_t_1 = PyList_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 60, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_float_0_5); + __Pyx_GIVEREF(__pyx_float_0_5); + PyList_SET_ITEM(__pyx_t_1, 0, __pyx_float_0_5); + __Pyx_INCREF(__pyx_int_1); + __Pyx_GIVEREF(__pyx_int_1); + PyList_SET_ITEM(__pyx_t_1, 1, __pyx_int_1); + __Pyx_INCREF(__pyx_float_1_5); + __Pyx_GIVEREF(__pyx_float_1_5); + PyList_SET_ITEM(__pyx_t_1, 2, __pyx_float_1_5); + __pyx_k__8 = __pyx_t_1; + __Pyx_GIVEREF(__pyx_t_1); + __pyx_t_1 = 0; + + /* "kenlm.pyx":42 + * + * + * def lmplz( # <<<<<<<<<<<<<< + * path_text_file, path_arpa_file, + * order=3, + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5kenlm_1lmplz, NULL, __pyx_n_s_kenlm); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 42, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_lmplz, __pyx_t_1) < 0) __PYX_ERR(0, 42, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "kenlm.pyx":232 * return _kenlm.hash_value(self._c_state) * * class LoadMethod: # <<<<<<<<<<<<<< * LAZY = _kenlm.LAZY * POPULATE_OR_LAZY = _kenlm.POPULATE_OR_LAZY */ - __pyx_t_1 = __Pyx_Py3MetaclassPrepare((PyObject *) NULL, __pyx_empty_tuple, __pyx_n_s_LoadMethod, __pyx_n_s_LoadMethod, (PyObject *) NULL, __pyx_n_s_kenlm, (PyObject *) NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 73, __pyx_L1_error) + __pyx_t_1 = __Pyx_Py3MetaclassPrepare((PyObject *) NULL, __pyx_empty_tuple, __pyx_n_s_LoadMethod, __pyx_n_s_LoadMethod, (PyObject *) NULL, __pyx_n_s_kenlm, (PyObject *) NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 232, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - /* "kenlm.pyx":74 + /* "kenlm.pyx":233 * * class LoadMethod: * LAZY = _kenlm.LAZY # <<<<<<<<<<<<<< * POPULATE_OR_LAZY = _kenlm.POPULATE_OR_LAZY * POPULATE_OR_READ = _kenlm.POPULATE_OR_READ */ - __pyx_t_2 = __Pyx_PyInt_From_enum__util_3a__3a_LoadMethod(util::LAZY); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 74, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyInt_From_enum__util_3a__3a_LoadMethod(util::LAZY); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 233, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyObject_SetItem(__pyx_t_1, __pyx_n_s_LAZY, __pyx_t_2) < 0) __PYX_ERR(0, 74, __pyx_L1_error) + if (PyObject_SetItem(__pyx_t_1, __pyx_n_s_LAZY, __pyx_t_2) < 0) __PYX_ERR(0, 233, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "kenlm.pyx":75 + /* "kenlm.pyx":234 * class LoadMethod: * LAZY = _kenlm.LAZY * POPULATE_OR_LAZY = _kenlm.POPULATE_OR_LAZY # <<<<<<<<<<<<<< * POPULATE_OR_READ = _kenlm.POPULATE_OR_READ * READ = _kenlm.READ */ - __pyx_t_2 = __Pyx_PyInt_From_enum__util_3a__3a_LoadMethod(util::POPULATE_OR_LAZY); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 75, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyInt_From_enum__util_3a__3a_LoadMethod(util::POPULATE_OR_LAZY); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 234, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyObject_SetItem(__pyx_t_1, __pyx_n_s_POPULATE_OR_LAZY, __pyx_t_2) < 0) __PYX_ERR(0, 75, __pyx_L1_error) + if (PyObject_SetItem(__pyx_t_1, __pyx_n_s_POPULATE_OR_LAZY, __pyx_t_2) < 0) __PYX_ERR(0, 234, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "kenlm.pyx":76 + /* "kenlm.pyx":235 * LAZY = _kenlm.LAZY * POPULATE_OR_LAZY = _kenlm.POPULATE_OR_LAZY * POPULATE_OR_READ = _kenlm.POPULATE_OR_READ # <<<<<<<<<<<<<< * READ = _kenlm.READ * PARALLEL_READ = _kenlm.PARALLEL_READ */ - __pyx_t_2 = __Pyx_PyInt_From_enum__util_3a__3a_LoadMethod(util::POPULATE_OR_READ); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 76, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyInt_From_enum__util_3a__3a_LoadMethod(util::POPULATE_OR_READ); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 235, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyObject_SetItem(__pyx_t_1, __pyx_n_s_POPULATE_OR_READ, __pyx_t_2) < 0) __PYX_ERR(0, 76, __pyx_L1_error) + if (PyObject_SetItem(__pyx_t_1, __pyx_n_s_POPULATE_OR_READ, __pyx_t_2) < 0) __PYX_ERR(0, 235, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "kenlm.pyx":77 + /* "kenlm.pyx":236 * POPULATE_OR_LAZY = _kenlm.POPULATE_OR_LAZY * POPULATE_OR_READ = _kenlm.POPULATE_OR_READ * READ = _kenlm.READ # <<<<<<<<<<<<<< * PARALLEL_READ = _kenlm.PARALLEL_READ * */ - __pyx_t_2 = __Pyx_PyInt_From_enum__util_3a__3a_LoadMethod(util::READ); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 77, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyInt_From_enum__util_3a__3a_LoadMethod(util::READ); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 236, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyObject_SetItem(__pyx_t_1, __pyx_n_s_READ, __pyx_t_2) < 0) __PYX_ERR(0, 77, __pyx_L1_error) + if (PyObject_SetItem(__pyx_t_1, __pyx_n_s_READ, __pyx_t_2) < 0) __PYX_ERR(0, 236, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "kenlm.pyx":78 + /* "kenlm.pyx":237 * POPULATE_OR_READ = _kenlm.POPULATE_OR_READ * READ = _kenlm.READ * PARALLEL_READ = _kenlm.PARALLEL_READ # <<<<<<<<<<<<<< * * cdef class Config: */ - __pyx_t_2 = __Pyx_PyInt_From_enum__util_3a__3a_LoadMethod(util::PARALLEL_READ); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 78, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyInt_From_enum__util_3a__3a_LoadMethod(util::PARALLEL_READ); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 237, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyObject_SetItem(__pyx_t_1, __pyx_n_s_PARALLEL_READ, __pyx_t_2) < 0) __PYX_ERR(0, 78, __pyx_L1_error) + if (PyObject_SetItem(__pyx_t_1, __pyx_n_s_PARALLEL_READ, __pyx_t_2) < 0) __PYX_ERR(0, 237, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "kenlm.pyx":73 + /* "kenlm.pyx":232 * return _kenlm.hash_value(self._c_state) * * class LoadMethod: # <<<<<<<<<<<<<< * LAZY = _kenlm.LAZY * POPULATE_OR_LAZY = _kenlm.POPULATE_OR_LAZY */ - __pyx_t_2 = __Pyx_Py3ClassCreate(((PyObject*)&__Pyx_DefaultClassType), __pyx_n_s_LoadMethod, __pyx_empty_tuple, __pyx_t_1, NULL, 0, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 73, __pyx_L1_error) + __pyx_t_2 = __Pyx_Py3ClassCreate(((PyObject*)&__Pyx_DefaultClassType), __pyx_n_s_LoadMethod, __pyx_empty_tuple, __pyx_t_1, NULL, 0, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 232, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_LoadMethod, __pyx_t_2) < 0) __PYX_ERR(0, 73, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_LoadMethod, __pyx_t_2) < 0) __PYX_ERR(0, 232, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "kenlm.pyx":105 + /* "kenlm.pyx":264 * cdef _kenlm.const_Vocabulary* vocab * * def __init__(self, path, Config config = Config()): # <<<<<<<<<<<<<< * """ * Load the language model. */ - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5kenlm_Config), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 105, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5kenlm_Config), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 264, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_k__2 = ((struct __pyx_obj_5kenlm_Config *)__pyx_t_1); + __pyx_k__21 = ((struct __pyx_obj_5kenlm_Config *)__pyx_t_1); __Pyx_GIVEREF(__pyx_t_1); __pyx_t_1 = 0; - /* "kenlm.pyx":260 + /* "kenlm.pyx":419 * return (Model, (self.path,)) * * class LanguageModel(Model): # <<<<<<<<<<<<<< * """Backwards compatability stub. Use Model.""" */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 260, __pyx_L1_error) + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 419, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(((PyObject *)__pyx_ptype_5kenlm_Model)); __Pyx_GIVEREF(((PyObject *)__pyx_ptype_5kenlm_Model)); PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_ptype_5kenlm_Model)); - __pyx_t_2 = __Pyx_CalculateMetaclass(NULL, __pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 260, __pyx_L1_error) + __pyx_t_2 = __Pyx_CalculateMetaclass(NULL, __pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 419, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_Py3MetaclassPrepare(__pyx_t_2, __pyx_t_1, __pyx_n_s_LanguageModel, __pyx_n_s_LanguageModel, (PyObject *) NULL, __pyx_n_s_kenlm, __pyx_kp_s_Backwards_compatability_stub_Use); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 260, __pyx_L1_error) + __pyx_t_3 = __Pyx_Py3MetaclassPrepare(__pyx_t_2, __pyx_t_1, __pyx_n_s_LanguageModel, __pyx_n_s_LanguageModel, (PyObject *) NULL, __pyx_n_s_kenlm, __pyx_kp_s_Backwards_compatability_stub_Use); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 419, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_Py3ClassCreate(__pyx_t_2, __pyx_n_s_LanguageModel, __pyx_t_1, __pyx_t_3, NULL, 0, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 260, __pyx_L1_error) + __pyx_t_4 = __Pyx_Py3ClassCreate(__pyx_t_2, __pyx_n_s_LanguageModel, __pyx_t_1, __pyx_t_3, NULL, 0, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 419, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_LanguageModel, __pyx_t_4) < 0) __PYX_ERR(0, 260, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_LanguageModel, __pyx_t_4) < 0) __PYX_ERR(0, 419, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -5506,11 +9510,19 @@ PyMODINIT_FUNC PyInit_kenlm(void) * cimport _kenlm * */ - __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) __PYX_ERR(0, 1, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + /* "string.to_py":55 + * + * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyByteArray_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) + * + */ + /*--- Wrapped vars code ---*/ goto __pyx_L0; @@ -5521,7 +9533,7 @@ PyMODINIT_FUNC PyInit_kenlm(void) __Pyx_XDECREF(__pyx_t_4); if (__pyx_m) { if (__pyx_d) { - __Pyx_AddTraceback("init kenlm", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("init kenlm", 0, __pyx_lineno, __pyx_filename); } Py_DECREF(__pyx_m); __pyx_m = 0; } else if (!PyErr_Occurred()) { @@ -5529,381 +9541,787 @@ PyMODINIT_FUNC PyInit_kenlm(void) } __pyx_L0:; __Pyx_RefNannyFinishContext(); - #if PY_MAJOR_VERSION < 3 - return; - #else + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #elif PY_MAJOR_VERSION >= 3 return __pyx_m; + #else + return; #endif } - -/* --- Runtime support code --- */ -/* Refnanny */ -#if CYTHON_REFNANNY -static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { - PyObject *m = NULL, *p = NULL; - void *r = NULL; - m = PyImport_ImportModule((char *)modname); - if (!m) goto end; - p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); - if (!p) goto end; - r = PyLong_AsVoidPtr(p); -end: - Py_XDECREF(p); - Py_XDECREF(m); - return (__Pyx_RefNannyAPIStruct *)r; + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule((char *)modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); + if (unlikely(!result)) { + PyErr_Format(PyExc_NameError, +#if PY_MAJOR_VERSION >= 3 + "name '%U' is not defined", name); +#else + "name '%.200s' is not defined", PyString_AS_STRING(name)); +#endif + } + return result; +} + +/* PyObjectCall */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = func->ob_type->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyErrFetchRestore */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +} +#endif + +/* RaiseException */ +#if PY_MAJOR_VERSION < 3 +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, + CYTHON_UNUSED PyObject *cause) { + __Pyx_PyThreadState_declare + Py_XINCREF(type); + if (!value || value == Py_None) + value = NULL; + else + Py_INCREF(value); + if (!tb || tb == Py_None) + tb = NULL; + else { + Py_INCREF(tb); + if (!PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto raise_error; + } + } + if (PyType_Check(type)) { +#if CYTHON_COMPILING_IN_PYPY + if (!value) { + Py_INCREF(Py_None); + value = Py_None; + } +#endif + PyErr_NormalizeException(&type, &value, &tb); + } else { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto raise_error; + } + value = type; + type = (PyObject*) Py_TYPE(type); + Py_INCREF(type); + if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto raise_error; + } + } + __Pyx_PyThreadState_assign + __Pyx_ErrRestore(type, value, tb); + return; +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(tb); + return; +} +#else +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if CYTHON_COMPILING_IN_PYPY + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#else + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#endif + } +bad: + Py_XDECREF(owned_instance); + return; } #endif -/* GetBuiltinName */ -static PyObject *__Pyx_GetBuiltinName(PyObject *name) { - PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); - if (unlikely(!result)) { - PyErr_Format(PyExc_NameError, -#if PY_MAJOR_VERSION >= 3 - "name '%U' is not defined", name); -#else - "name '%.200s' is not defined", PyString_AS_STRING(name)); -#endif +/* RaiseArgTupleInvalid */ +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; } - return result; + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); } -/* PyObjectCall */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { - PyObject *result; - ternaryfunc call = func->ob_type->tp_call; - if (unlikely(!call)) - return PyObject_Call(func, arg, kw); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = (*call)(func, arg, kw); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); +/* RaiseDoubleKeywords */ +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION >= 3 + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); + #else + "%s() got multiple values for keyword argument '%s'", func_name, + PyString_AsString(kw_name)); + #endif +} + +/* ParseKeywords */ +static int __Pyx_ParseOptionalKeywords( + PyObject *kwds, + PyObject **argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject *key = 0, *value = 0; + Py_ssize_t pos = 0; + PyObject*** name; + PyObject*** first_kw_arg = argnames + num_pos_args; + while (PyDict_Next(kwds, &pos, &key, &value)) { + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + values[name-argnames] = value; + continue; + } + name = first_kw_arg; + #if PY_MAJOR_VERSION < 3 + if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { + while (*name) { + if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) + && _PyString_Eq(**name, key)) { + values[name-argnames] = value; + break; + } + name++; + } + if (*name) continue; + else { + PyObject*** argname = argnames; + while (argname != first_kw_arg) { + if ((**argname == key) || ( + (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) + && _PyString_Eq(**argname, key))) { + goto arg_passed_twice; + } + argname++; + } + } + } else + #endif + if (likely(PyUnicode_Check(key))) { + while (*name) { + int cmp = (**name == key) ? 0 : + #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 + (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : + #endif + PyUnicode_Compare(**name, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + values[name-argnames] = value; + break; + } + name++; + } + if (*name) continue; + else { + PyObject*** argname = argnames; + while (argname != first_kw_arg) { + int cmp = (**argname == key) ? 0 : + #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 + (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : + #endif + PyUnicode_Compare(**argname, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + argname++; + } + } + } else + goto invalid_keyword_type; + if (kwds2) { + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else { + goto invalid_keyword; + } } - return result; + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION < 3 + "%.200s() got an unexpected keyword argument '%.200s'", + function_name, PyString_AsString(key)); + #else + "%s() got an unexpected keyword argument '%U'", + function_name, key); + #endif +bad: + return -1; } -#endif -/* PyErrFetchRestore */ -#if CYTHON_FAST_THREAD_STATE -static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - tmp_type = tstate->curexc_type; - tmp_value = tstate->curexc_value; - tmp_tb = tstate->curexc_traceback; - tstate->curexc_type = type; - tstate->curexc_value = value; - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -} -static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { - *type = tstate->curexc_type; - *value = tstate->curexc_value; - *tb = tstate->curexc_traceback; - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; +/* GetItemInt */ +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { + PyObject *r; + if (!j) return NULL; + r = PyObject_GetItem(o, j); + Py_DECREF(j); + return r; } -#endif - -/* RaiseException */ -#if PY_MAJOR_VERSION < 3 -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, - CYTHON_UNUSED PyObject *cause) { - __Pyx_PyThreadState_declare - Py_XINCREF(type); - if (!value || value == Py_None) - value = NULL; - else - Py_INCREF(value); - if (!tb || tb == Py_None) - tb = NULL; - else { - Py_INCREF(tb); - if (!PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto raise_error; - } +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyList_GET_SIZE(o); } - if (PyType_Check(type)) { -#if CYTHON_COMPILING_IN_PYPY - if (!value) { - Py_INCREF(Py_None); - value = Py_None; - } -#endif - PyErr_NormalizeException(&type, &value, &tb); - } else { - if (value) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto raise_error; - } - value = type; - type = (PyObject*) Py_TYPE(type); - Py_INCREF(type); - if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto raise_error; - } + if ((!boundscheck) || likely((0 <= wrapped_i) & (wrapped_i < PyList_GET_SIZE(o)))) { + PyObject *r = PyList_GET_ITEM(o, wrapped_i); + Py_INCREF(r); + return r; } - __Pyx_PyThreadState_assign - __Pyx_ErrRestore(type, value, tb); - return; -raise_error: - Py_XDECREF(value); - Py_XDECREF(type); - Py_XDECREF(tb); - return; + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +#else + return PySequence_GetItem(o, i); +#endif } +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyTuple_GET_SIZE(o); + } + if ((!boundscheck) || likely((0 <= wrapped_i) & (wrapped_i < PyTuple_GET_SIZE(o)))) { + PyObject *r = PyTuple_GET_ITEM(o, wrapped_i); + Py_INCREF(r); + return r; + } + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); #else -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { - PyObject* owned_instance = NULL; - if (tb == Py_None) { - tb = 0; - } else if (tb && !PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto bad; + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS && CYTHON_USE_TYPE_SLOTS + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); + if ((!boundscheck) || (likely((n >= 0) & (n < PyList_GET_SIZE(o))))) { + PyObject *r = PyList_GET_ITEM(o, n); + Py_INCREF(r); + return r; + } } - if (value == Py_None) - value = 0; - if (PyExceptionInstance_Check(type)) { - if (value) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto bad; + else if (PyTuple_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); + if ((!boundscheck) || likely((n >= 0) & (n < PyTuple_GET_SIZE(o)))) { + PyObject *r = PyTuple_GET_ITEM(o, n); + Py_INCREF(r); + return r; } - value = type; - type = (PyObject*) Py_TYPE(value); - } else if (PyExceptionClass_Check(type)) { - PyObject *instance_class = NULL; - if (value && PyExceptionInstance_Check(value)) { - instance_class = (PyObject*) Py_TYPE(value); - if (instance_class != type) { - int is_subclass = PyObject_IsSubclass(instance_class, type); - if (!is_subclass) { - instance_class = NULL; - } else if (unlikely(is_subclass == -1)) { - goto bad; + } else { + PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; + if (likely(m && m->sq_item)) { + if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { + Py_ssize_t l = m->sq_length(o); + if (likely(l >= 0)) { + i += l; } else { - type = instance_class; + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + PyErr_Clear(); } } + return m->sq_item(o, i); } - if (!instance_class) { - PyObject *args; - if (!value) - args = PyTuple_New(0); - else if (PyTuple_Check(value)) { - Py_INCREF(value); - args = value; - } else - args = PyTuple_Pack(1, value); - if (!args) - goto bad; - owned_instance = PyObject_Call(type, args, NULL); - Py_DECREF(args); - if (!owned_instance) - goto bad; - value = owned_instance; - if (!PyExceptionInstance_Check(value)) { - PyErr_Format(PyExc_TypeError, - "calling %R should have returned an instance of " - "BaseException, not %R", - type, Py_TYPE(value)); - goto bad; + } +#else + if (is_list || PySequence_Check(o)) { + return PySequence_GetItem(o, i); + } +#endif + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +} + +/* PyIntBinop */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject* __Pyx_PyInt_AddObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED int inplace) { + #if PY_MAJOR_VERSION < 3 + if (likely(PyInt_CheckExact(op1))) { + const long b = intval; + long x; + long a = PyInt_AS_LONG(op1); + x = (long)((unsigned long)a + b); + if (likely((x^a) >= 0 || (x^b) >= 0)) + return PyInt_FromLong(x); + return PyLong_Type.tp_as_number->nb_add(op1, op2); + } + #endif + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(PyLong_CheckExact(op1))) { + const long b = intval; + long a, x; +#ifdef HAVE_LONG_LONG + const PY_LONG_LONG llb = intval; + PY_LONG_LONG lla, llx; +#endif + const digit* digits = ((PyLongObject*)op1)->ob_digit; + const Py_ssize_t size = Py_SIZE(op1); + if (likely(__Pyx_sst_abs(size) <= 1)) { + a = likely(size) ? digits[0] : 0; + if (size == -1) a = -a; + } else { + switch (size) { + case -2: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + a = -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; +#ifdef HAVE_LONG_LONG + } else if (8 * sizeof(PY_LONG_LONG) - 1 > 2 * PyLong_SHIFT) { + lla = -(PY_LONG_LONG) (((((unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); + goto long_long; +#endif + } + case 2: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + a = (long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; +#ifdef HAVE_LONG_LONG + } else if (8 * sizeof(PY_LONG_LONG) - 1 > 2 * PyLong_SHIFT) { + lla = (PY_LONG_LONG) (((((unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); + goto long_long; +#endif + } + case -3: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + a = -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; +#ifdef HAVE_LONG_LONG + } else if (8 * sizeof(PY_LONG_LONG) - 1 > 3 * PyLong_SHIFT) { + lla = -(PY_LONG_LONG) (((((((unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); + goto long_long; +#endif + } + case 3: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + a = (long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; +#ifdef HAVE_LONG_LONG + } else if (8 * sizeof(PY_LONG_LONG) - 1 > 3 * PyLong_SHIFT) { + lla = (PY_LONG_LONG) (((((((unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); + goto long_long; +#endif + } + case -4: + if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + a = -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; +#ifdef HAVE_LONG_LONG + } else if (8 * sizeof(PY_LONG_LONG) - 1 > 4 * PyLong_SHIFT) { + lla = -(PY_LONG_LONG) (((((((((unsigned PY_LONG_LONG)digits[3]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); + goto long_long; +#endif + } + case 4: + if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + a = (long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; +#ifdef HAVE_LONG_LONG + } else if (8 * sizeof(PY_LONG_LONG) - 1 > 4 * PyLong_SHIFT) { + lla = (PY_LONG_LONG) (((((((((unsigned PY_LONG_LONG)digits[3]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0])); + goto long_long; +#endif + } + default: return PyLong_Type.tp_as_number->nb_add(op1, op2); } } - } else { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto bad; + x = a + b; + return PyLong_FromLong(x); +#ifdef HAVE_LONG_LONG + long_long: + llx = lla + llb; + return PyLong_FromLongLong(llx); +#endif + + + } + #endif + if (PyFloat_CheckExact(op1)) { + const long b = intval; + double a = PyFloat_AS_DOUBLE(op1); + double result; + PyFPE_START_PROTECT("add", return NULL) + result = ((double)a) + (double)b; + PyFPE_END_PROTECT(result) + return PyFloat_FromDouble(result); + } + return (inplace ? PyNumber_InPlaceAdd : PyNumber_Add)(op1, op2); +} +#endif + +/* PyFunctionFastCall */ +#if CYTHON_FAST_PYCALL +#include "frameobject.h" +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = f->f_localsplus; + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; +#if PY_MAJOR_VERSION >= 3 + PyObject *kwdefs; +#endif + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { + return NULL; } -#if PY_VERSION_HEX >= 0x03030000 - if (cause) { -#else - if (cause && cause != Py_None) { + if ( +#if PY_MAJOR_VERSION >= 3 + co->co_kwonlyargcount == 0 && #endif - PyObject *fixed_cause; - if (cause == Py_None) { - fixed_cause = NULL; - } else if (PyExceptionClass_Check(cause)) { - fixed_cause = PyObject_CallObject(cause, NULL); - if (fixed_cause == NULL) - goto bad; - } else if (PyExceptionInstance_Check(cause)) { - fixed_cause = cause; - Py_INCREF(fixed_cause); - } else { - PyErr_SetString(PyExc_TypeError, - "exception causes must derive from " - "BaseException"); - goto bad; + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; } - PyException_SetCause(value, fixed_cause); } - PyErr_SetObject(type, value); - if (tb) { -#if CYTHON_COMPILING_IN_PYPY - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); - Py_INCREF(tb); - PyErr_Restore(tmp_type, tmp_value, tb); - Py_XDECREF(tmp_tb); -#else - PyThreadState *tstate = PyThreadState_GET(); - PyObject* tmp_tb = tstate->curexc_traceback; - if (tb != tmp_tb) { - Py_INCREF(tb); - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_tb); + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); +#if PY_MAJOR_VERSION >= 3 + kwdefs = PyFunction_GET_KW_DEFAULTS(func); #endif + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); } -bad: - Py_XDECREF(owned_instance); - return; + else { + d = NULL; + nd = 0; + } +#if PY_MAJOR_VERSION >= 3 + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); +#else + result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, closure); +#endif + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; } #endif +#endif -/* RaiseArgTupleInvalid */ - static void __Pyx_RaiseArgtupleInvalid( - const char* func_name, - int exact, - Py_ssize_t num_min, - Py_ssize_t num_max, - Py_ssize_t num_found) -{ - Py_ssize_t num_expected; - const char *more_or_less; - if (num_found < num_min) { - num_expected = num_min; - more_or_less = "at least"; +/* PyCFunctionFastCall */ +#if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { + PyCFunctionObject *func = (PyCFunctionObject*)func_obj; + PyCFunction meth = PyCFunction_GET_FUNCTION(func); + PyObject *self = PyCFunction_GET_SELF(func); + int flags = PyCFunction_GET_FLAGS(func); + assert(PyCFunction_Check(func)); + assert(METH_FASTCALL == (flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS))); + assert(nargs >= 0); + assert(nargs == 0 || args != NULL); + /* _PyCFunction_FastCallDict() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!PyErr_Occurred()); + if ((PY_VERSION_HEX < 0x030700A0) || unlikely(flags & METH_KEYWORDS)) { + return (*((__Pyx_PyCFunctionFastWithKeywords)meth)) (self, args, nargs, NULL); } else { - num_expected = num_max; - more_or_less = "at most"; - } - if (exact) { - more_or_less = "exactly"; + return (*((__Pyx_PyCFunctionFast)meth)) (self, args, nargs); } - PyErr_Format(PyExc_TypeError, - "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", - func_name, more_or_less, num_expected, - (num_expected == 1) ? "" : "s", num_found); } +#endif -/* RaiseDoubleKeywords */ - static void __Pyx_RaiseDoubleKeywordsError( - const char* func_name, - PyObject* kw_name) +/* ArgTypeTest */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) { - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION >= 3 - "%s() got multiple values for keyword argument '%U'", func_name, kw_name); - #else - "%s() got multiple values for keyword argument '%s'", func_name, - PyString_AsString(kw_name)); + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + else if (exact) { + #if PY_MAJOR_VERSION == 2 + if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; #endif + } + else { + if (likely(__Pyx_TypeCheck(obj, type))) return 1; + } + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", + name, type->tp_name, Py_TYPE(obj)->tp_name); + return 0; } -/* ParseKeywords */ - static int __Pyx_ParseOptionalKeywords( - PyObject *kwds, - PyObject **argnames[], - PyObject *kwds2, - PyObject *values[], - Py_ssize_t num_pos_args, - const char* function_name) +/* KeywordStringCheck */ +static int __Pyx_CheckKeywordStrings( + PyObject *kwdict, + const char* function_name, + int kw_allowed) { - PyObject *key = 0, *value = 0; + PyObject* key = 0; Py_ssize_t pos = 0; - PyObject*** name; - PyObject*** first_kw_arg = argnames + num_pos_args; - while (PyDict_Next(kwds, &pos, &key, &value)) { - name = first_kw_arg; - while (*name && (**name != key)) name++; - if (*name) { - values[name-argnames] = value; - continue; - } - name = first_kw_arg; +#if CYTHON_COMPILING_IN_PYPY + if (!kw_allowed && PyDict_Next(kwdict, &pos, &key, 0)) + goto invalid_keyword; + return 1; +#else + while (PyDict_Next(kwdict, &pos, &key, 0)) { #if PY_MAJOR_VERSION < 3 - if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { - while (*name) { - if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) - && _PyString_Eq(**name, key)) { - values[name-argnames] = value; - break; - } - name++; - } - if (*name) continue; - else { - PyObject*** argname = argnames; - while (argname != first_kw_arg) { - if ((**argname == key) || ( - (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) - && _PyString_Eq(**argname, key))) { - goto arg_passed_twice; - } - argname++; - } - } - } else + if (unlikely(!PyString_Check(key))) #endif - if (likely(PyUnicode_Check(key))) { - while (*name) { - int cmp = (**name == key) ? 0 : - #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 - (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : - #endif - PyUnicode_Compare(**name, key); - if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; - if (cmp == 0) { - values[name-argnames] = value; - break; - } - name++; - } - if (*name) continue; - else { - PyObject*** argname = argnames; - while (argname != first_kw_arg) { - int cmp = (**argname == key) ? 0 : - #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 - (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : - #endif - PyUnicode_Compare(**argname, key); - if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; - if (cmp == 0) goto arg_passed_twice; - argname++; - } - } - } else - goto invalid_keyword_type; - if (kwds2) { - if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; - } else { - goto invalid_keyword; - } + if (unlikely(!PyUnicode_Check(key))) + goto invalid_keyword_type; } - return 0; -arg_passed_twice: - __Pyx_RaiseDoubleKeywordsError(function_name, key); - goto bad; + if ((!kw_allowed) && unlikely(key)) + goto invalid_keyword; + return 1; invalid_keyword_type: PyErr_Format(PyExc_TypeError, "%.200s() keywords must be strings", function_name); - goto bad; + return 0; +#endif invalid_keyword: PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION < 3 @@ -5913,914 +10331,1449 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject "%s() got an unexpected keyword argument '%U'", function_name, key); #endif -bad: - return -1; + return 0; } -/* PyFunctionFastCall */ - #if CYTHON_FAST_PYCALL -#include "frameobject.h" -static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, - PyObject *globals) { - PyFrameObject *f; - PyThreadState *tstate = PyThreadState_GET(); - PyObject **fastlocals; - Py_ssize_t i; +/* GetModuleGlobalName */ +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { PyObject *result; - assert(globals != NULL); - /* XXX Perhaps we should create a specialized - PyFrame_New() that doesn't take locals, but does - take builtins without sanity checking them. - */ - assert(tstate != NULL); - f = PyFrame_New(tstate, co, globals, NULL); - if (f == NULL) { - return NULL; +#if !CYTHON_AVOID_BORROWED_REFS + result = PyDict_GetItem(__pyx_d, name); + if (likely(result)) { + Py_INCREF(result); + } else { +#else + result = PyObject_GetItem(__pyx_d, name); + if (!result) { + PyErr_Clear(); +#endif + result = __Pyx_GetBuiltinName(name); } - fastlocals = f->f_localsplus; - for (i = 0; i < na; i++) { - Py_INCREF(*args); - fastlocals[i] = *args++; + return result; +} + +/* PyObjectCallMethO */ + #if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = PyCFunction_GET_FUNCTION(func); + self = PyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); } - result = PyEval_EvalFrameEx(f,0); - ++tstate->recursion_depth; - Py_DECREF(f); - --tstate->recursion_depth; return result; } -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { - PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); - PyObject *globals = PyFunction_GET_GLOBALS(func); - PyObject *argdefs = PyFunction_GET_DEFAULTS(func); - PyObject *closure; -#if PY_MAJOR_VERSION >= 3 - PyObject *kwdefs; #endif - PyObject *kwtuple, **k; - PyObject **d; - Py_ssize_t nd; - Py_ssize_t nk; + +/* PyObjectCallOneArg */ + #if CYTHON_COMPILING_IN_CPYTHON +static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { PyObject *result; - assert(kwargs == NULL || PyDict_Check(kwargs)); - nk = kwargs ? PyDict_Size(kwargs) : 0; - if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { - return NULL; + PyObject *args = PyTuple_New(1); + if (unlikely(!args)) return NULL; + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 0, arg); + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, &arg, 1); } - if ( -#if PY_MAJOR_VERSION >= 3 - co->co_kwonlyargcount == 0 && #endif - likely(kwargs == NULL || nk == 0) && - co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { - if (argdefs == NULL && co->co_argcount == nargs) { - result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); - goto done; - } - else if (nargs == 0 && argdefs != NULL - && co->co_argcount == Py_SIZE(argdefs)) { - /* function called with no arguments, but all parameters have - a default value: use default values as arguments .*/ - args = &PyTuple_GET_ITEM(argdefs, 0); - result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); - goto done; + if (likely(PyCFunction_Check(func))) { + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, arg); +#if CYTHON_FAST_PYCCALL + } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { + return __Pyx_PyCFunction_FastCall(func, &arg, 1); +#endif } } - if (kwargs != NULL) { - Py_ssize_t pos, i; - kwtuple = PyTuple_New(2 * nk); - if (kwtuple == NULL) { - result = NULL; - goto done; - } - k = &PyTuple_GET_ITEM(kwtuple, 0); - pos = i = 0; - while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { - Py_INCREF(k[i]); - Py_INCREF(k[i+1]); - i += 2; - } - nk = i / 2; + return __Pyx__PyObject_CallOneArg(func, arg); +} +#else +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_Pack(1, arg); + if (unlikely(!args)) return NULL; + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +#endif + +/* SaveResetException */ + #if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if PY_VERSION_HEX >= 0x030700A2 + *type = tstate->exc_state.exc_type; + *value = tstate->exc_state.exc_value; + *tb = tstate->exc_state.exc_traceback; + #else + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + #endif + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if PY_VERSION_HEX >= 0x030700A2 + tmp_type = tstate->exc_state.exc_type; + tmp_value = tstate->exc_state.exc_value; + tmp_tb = tstate->exc_state.exc_traceback; + tstate->exc_state.exc_type = type; + tstate->exc_state.exc_value = value; + tstate->exc_state.exc_traceback = tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} +#endif + +/* PyErrExceptionMatches */ + #if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); +#if PY_MAJOR_VERSION >= 3 + for (i=0; i= 3 - kwdefs = PyFunction_GET_KW_DEFAULTS(func); + return 0; +} +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err) { + PyObject *exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; + if (unlikely(PyTuple_Check(err))) + return __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + return __Pyx_PyErr_GivenExceptionMatches(exc_type, err); +} #endif - if (argdefs != NULL) { - d = &PyTuple_GET_ITEM(argdefs, 0); - nd = Py_SIZE(argdefs); + +/* GetException */ + #if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { +#endif + PyObject *local_type, *local_value, *local_tb; +#if CYTHON_FAST_THREAD_STATE + PyObject *tmp_type, *tmp_value, *tmp_tb; + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#else + PyErr_Fetch(&local_type, &local_value, &local_tb); +#endif + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_FAST_THREAD_STATE + if (unlikely(tstate->curexc_type)) +#else + if (unlikely(PyErr_Occurred())) +#endif + goto bad; + #if PY_MAJOR_VERSION >= 3 + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; } - else { - d = NULL; - nd = 0; + #endif + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_FAST_THREAD_STATE + #if PY_VERSION_HEX >= 0x030700A2 + tmp_type = tstate->exc_state.exc_type; + tmp_value = tstate->exc_state.exc_value; + tmp_tb = tstate->exc_state.exc_traceback; + tstate->exc_state.exc_type = local_type; + tstate->exc_state.exc_value = local_value; + tstate->exc_state.exc_traceback = local_tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); +#endif + return 0; +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +} + +/* PyObjectCallNoArg */ + #if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, NULL, 0); } -#if PY_MAJOR_VERSION >= 3 - result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, kwdefs, closure); +#endif +#ifdef __Pyx_CyFunction_USED + if (likely(PyCFunction_Check(func) || __Pyx_TypeCheck(func, __pyx_CyFunctionType))) { #else - result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, closure); + if (likely(PyCFunction_Check(func))) { #endif - Py_XDECREF(kwtuple); -done: - Py_LeaveRecursiveCall(); - return result; + if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { + return __Pyx_PyObject_CallMethO(func, NULL); + } + } + return __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL); } -#endif // CPython < 3.6 -#endif // CYTHON_FAST_PYCALL +#endif -/* PyCFunctionFastCall */ - #if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { - PyCFunctionObject *func = (PyCFunctionObject*)func_obj; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - assert(PyCFunction_Check(func)); - assert(METH_FASTCALL == (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))); - assert(nargs >= 0); - assert(nargs == 0 || args != NULL); - /* _PyCFunction_FastCallDict() must not be called with an exception set, - because it may clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!PyErr_Occurred()); - return (*((__Pyx_PyCFunctionFast)meth)) (self, args, nargs, NULL); +/* SetupReduce */ + static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) { + int ret; + PyObject *name_attr; + name_attr = __Pyx_PyObject_GetAttrStr(meth, __pyx_n_s_name); + if (likely(name_attr)) { + ret = PyObject_RichCompareBool(name_attr, name, Py_EQ); + } else { + ret = -1; + } + if (unlikely(ret < 0)) { + PyErr_Clear(); + ret = 0; + } + Py_XDECREF(name_attr); + return ret; +} +static int __Pyx_setup_reduce(PyObject* type_obj) { + int ret = 0; + PyObject *object_reduce = NULL; + PyObject *object_reduce_ex = NULL; + PyObject *reduce = NULL; + PyObject *reduce_ex = NULL; + PyObject *reduce_cython = NULL; + PyObject *setstate = NULL; + PyObject *setstate_cython = NULL; +#if CYTHON_USE_PYTYPE_LOOKUP + if (_PyType_Lookup((PyTypeObject*)type_obj, __pyx_n_s_getstate)) goto GOOD; +#else + if (PyObject_HasAttr(type_obj, __pyx_n_s_getstate)) goto GOOD; +#endif +#if CYTHON_USE_PYTYPE_LOOKUP + object_reduce_ex = _PyType_Lookup(&PyBaseObject_Type, __pyx_n_s_reduce_ex); if (!object_reduce_ex) goto BAD; +#else + object_reduce_ex = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_n_s_reduce_ex); if (!object_reduce_ex) goto BAD; +#endif + reduce_ex = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_reduce_ex); if (unlikely(!reduce_ex)) goto BAD; + if (reduce_ex == object_reduce_ex) { +#if CYTHON_USE_PYTYPE_LOOKUP + object_reduce = _PyType_Lookup(&PyBaseObject_Type, __pyx_n_s_reduce); if (!object_reduce) goto BAD; +#else + object_reduce = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_n_s_reduce); if (!object_reduce) goto BAD; +#endif + reduce = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_reduce); if (unlikely(!reduce)) goto BAD; + if (reduce == object_reduce || __Pyx_setup_reduce_is_named(reduce, __pyx_n_s_reduce_cython)) { + reduce_cython = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_reduce_cython); if (unlikely(!reduce_cython)) goto BAD; + ret = PyDict_SetItem(((PyTypeObject*)type_obj)->tp_dict, __pyx_n_s_reduce, reduce_cython); if (unlikely(ret < 0)) goto BAD; + ret = PyDict_DelItem(((PyTypeObject*)type_obj)->tp_dict, __pyx_n_s_reduce_cython); if (unlikely(ret < 0)) goto BAD; + setstate = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_setstate); + if (!setstate) PyErr_Clear(); + if (!setstate || __Pyx_setup_reduce_is_named(setstate, __pyx_n_s_setstate_cython)) { + setstate_cython = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_setstate_cython); if (unlikely(!setstate_cython)) goto BAD; + ret = PyDict_SetItem(((PyTypeObject*)type_obj)->tp_dict, __pyx_n_s_setstate, setstate_cython); if (unlikely(ret < 0)) goto BAD; + ret = PyDict_DelItem(((PyTypeObject*)type_obj)->tp_dict, __pyx_n_s_setstate_cython); if (unlikely(ret < 0)) goto BAD; + } + PyType_Modified((PyTypeObject*)type_obj); + } + } + goto GOOD; +BAD: + if (!PyErr_Occurred()) + PyErr_Format(PyExc_RuntimeError, "Unable to initialize pickling for %s", ((PyTypeObject*)type_obj)->tp_name); + ret = -1; +GOOD: +#if !CYTHON_USE_PYTYPE_LOOKUP + Py_XDECREF(object_reduce); + Py_XDECREF(object_reduce_ex); +#endif + Py_XDECREF(reduce); + Py_XDECREF(reduce_ex); + Py_XDECREF(reduce_cython); + Py_XDECREF(setstate); + Py_XDECREF(setstate_cython); + return ret; } -#endif // CYTHON_FAST_PYCCALL -/* ArgTypeTest */ - static void __Pyx_RaiseArgumentTypeInvalid(const char* name, PyObject *obj, PyTypeObject *type) { - PyErr_Format(PyExc_TypeError, - "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", - name, type->tp_name, Py_TYPE(obj)->tp_name); -} -static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, - const char *name, int exact) -{ - if (unlikely(!type)) { - PyErr_SetString(PyExc_SystemError, "Missing type object"); - return 0; - } - if (none_allowed && obj == Py_None) return 1; - else if (exact) { - if (likely(Py_TYPE(obj) == type)) return 1; - #if PY_MAJOR_VERSION == 2 - else if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; - #endif - } +/* Import */ + static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { + PyObject *empty_list = 0; + PyObject *module = 0; + PyObject *global_dict = 0; + PyObject *empty_dict = 0; + PyObject *list; + #if PY_MAJOR_VERSION < 3 + PyObject *py_import; + py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s_import); + if (!py_import) + goto bad; + #endif + if (from_list) + list = from_list; else { - if (likely(PyObject_TypeCheck(obj, type))) return 1; + empty_list = PyList_New(0); + if (!empty_list) + goto bad; + list = empty_list; } - __Pyx_RaiseArgumentTypeInvalid(name, obj, type); - return 0; -} - -/* KeywordStringCheck */ - static CYTHON_INLINE int __Pyx_CheckKeywordStrings( - PyObject *kwdict, - const char* function_name, - int kw_allowed) -{ - PyObject* key = 0; - Py_ssize_t pos = 0; -#if CYTHON_COMPILING_IN_PYPY - if (!kw_allowed && PyDict_Next(kwdict, &pos, &key, 0)) - goto invalid_keyword; - return 1; -#else - while (PyDict_Next(kwdict, &pos, &key, 0)) { - #if PY_MAJOR_VERSION < 3 - if (unlikely(!PyString_CheckExact(key)) && unlikely(!PyString_Check(key))) + global_dict = PyModule_GetDict(__pyx_m); + if (!global_dict) + goto bad; + empty_dict = PyDict_New(); + if (!empty_dict) + goto bad; + { + #if PY_MAJOR_VERSION >= 3 + if (level == -1) { + if (strchr(__Pyx_MODULE_NAME, '.')) { + module = PyImport_ImportModuleLevelObject( + name, global_dict, empty_dict, list, 1); + if (!module) { + if (!PyErr_ExceptionMatches(PyExc_ImportError)) + goto bad; + PyErr_Clear(); + } + } + level = 0; + } #endif - if (unlikely(!PyUnicode_Check(key))) - goto invalid_keyword_type; + if (!module) { + #if PY_MAJOR_VERSION < 3 + PyObject *py_level = PyInt_FromLong(level); + if (!py_level) + goto bad; + module = PyObject_CallFunctionObjArgs(py_import, + name, global_dict, empty_dict, list, py_level, NULL); + Py_DECREF(py_level); + #else + module = PyImport_ImportModuleLevelObject( + name, global_dict, empty_dict, list, level); + #endif + } } - if ((!kw_allowed) && unlikely(key)) - goto invalid_keyword; - return 1; -invalid_keyword_type: - PyErr_Format(PyExc_TypeError, - "%.200s() keywords must be strings", function_name); - return 0; -#endif -invalid_keyword: - PyErr_Format(PyExc_TypeError, +bad: #if PY_MAJOR_VERSION < 3 - "%.200s() got an unexpected keyword argument '%.200s'", - function_name, PyString_AsString(key)); - #else - "%s() got an unexpected keyword argument '%U'", - function_name, key); + Py_XDECREF(py_import); #endif - return 0; + Py_XDECREF(empty_list); + Py_XDECREF(empty_dict); + return module; } -/* GetModuleGlobalName */ - static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { - PyObject *result; -#if !CYTHON_AVOID_BORROWED_REFS - result = PyDict_GetItem(__pyx_d, name); - if (likely(result)) { - Py_INCREF(result); - } else { +/* CalculateMetaclass */ + static PyObject *__Pyx_CalculateMetaclass(PyTypeObject *metaclass, PyObject *bases) { + Py_ssize_t i, nbases = PyTuple_GET_SIZE(bases); + for (i=0; i < nbases; i++) { + PyTypeObject *tmptype; + PyObject *tmp = PyTuple_GET_ITEM(bases, i); + tmptype = Py_TYPE(tmp); +#if PY_MAJOR_VERSION < 3 + if (tmptype == &PyClass_Type) + continue; +#endif + if (!metaclass) { + metaclass = tmptype; + continue; + } + if (PyType_IsSubtype(metaclass, tmptype)) + continue; + if (PyType_IsSubtype(tmptype, metaclass)) { + metaclass = tmptype; + continue; + } + PyErr_SetString(PyExc_TypeError, + "metaclass conflict: " + "the metaclass of a derived class " + "must be a (non-strict) subclass " + "of the metaclasses of all its bases"); + return NULL; + } + if (!metaclass) { +#if PY_MAJOR_VERSION < 3 + metaclass = &PyClass_Type; #else - result = PyObject_GetItem(__pyx_d, name); - if (!result) { - PyErr_Clear(); + metaclass = &PyType_Type; #endif - result = __Pyx_GetBuiltinName(name); } + Py_INCREF((PyObject*) metaclass); + return (PyObject*) metaclass; +} + +/* Py3ClassCreate */ + static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, + PyObject *qualname, PyObject *mkw, PyObject *modname, PyObject *doc) { + PyObject *ns; + if (metaclass) { + PyObject *prep = __Pyx_PyObject_GetAttrStr(metaclass, __pyx_n_s_prepare); + if (prep) { + PyObject *pargs = PyTuple_Pack(2, name, bases); + if (unlikely(!pargs)) { + Py_DECREF(prep); + return NULL; + } + ns = PyObject_Call(prep, pargs, mkw); + Py_DECREF(prep); + Py_DECREF(pargs); + } else { + if (unlikely(!PyErr_ExceptionMatches(PyExc_AttributeError))) + return NULL; + PyErr_Clear(); + ns = PyDict_New(); + } + } else { + ns = PyDict_New(); + } + if (unlikely(!ns)) + return NULL; + if (unlikely(PyObject_SetItem(ns, __pyx_n_s_module, modname) < 0)) goto bad; + if (unlikely(PyObject_SetItem(ns, __pyx_n_s_qualname, qualname) < 0)) goto bad; + if (unlikely(doc && PyObject_SetItem(ns, __pyx_n_s_doc, doc) < 0)) goto bad; + return ns; +bad: + Py_DECREF(ns); + return NULL; +} +static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, + PyObject *dict, PyObject *mkw, + int calculate_metaclass, int allow_py2_metaclass) { + PyObject *result, *margs; + PyObject *owned_metaclass = NULL; + if (allow_py2_metaclass) { + owned_metaclass = PyObject_GetItem(dict, __pyx_n_s_metaclass); + if (owned_metaclass) { + metaclass = owned_metaclass; + } else if (likely(PyErr_ExceptionMatches(PyExc_KeyError))) { + PyErr_Clear(); + } else { + return NULL; + } + } + if (calculate_metaclass && (!metaclass || PyType_Check(metaclass))) { + metaclass = __Pyx_CalculateMetaclass((PyTypeObject*) metaclass, bases); + Py_XDECREF(owned_metaclass); + if (unlikely(!metaclass)) + return NULL; + owned_metaclass = metaclass; + } + margs = PyTuple_Pack(3, name, bases, dict); + if (unlikely(!margs)) { + result = NULL; + } else { + result = PyObject_Call(metaclass, margs, mkw); + Py_DECREF(margs); + } + Py_XDECREF(owned_metaclass); return result; } -/* PyObjectCallMethO */ - #if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { - PyObject *self, *result; - PyCFunction cfunc; - cfunc = PyCFunction_GET_FUNCTION(func); - self = PyCFunction_GET_SELF(func); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = cfunc(self, arg); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); +/* CLineInTraceback */ + #ifndef CYTHON_CLINE_IN_TRACEBACK +static int __Pyx_CLineForTraceback(CYTHON_UNUSED PyThreadState *tstate, int c_line) { + PyObject *use_cline; + PyObject *ptype, *pvalue, *ptraceback; +#if CYTHON_COMPILING_IN_CPYTHON + PyObject **cython_runtime_dict; +#endif + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); +#if CYTHON_COMPILING_IN_CPYTHON + cython_runtime_dict = _PyObject_GetDictPtr(__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + use_cline = PyDict_GetItem(*cython_runtime_dict, __pyx_n_s_cline_in_traceback); + } else +#endif + { + PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStr(__pyx_cython_runtime, __pyx_n_s_cline_in_traceback); + if (use_cline_obj) { + use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True; + Py_DECREF(use_cline_obj); + } else { + PyErr_Clear(); + use_cline = NULL; + } } - return result; + if (!use_cline) { + c_line = 0; + PyObject_SetAttr(__pyx_cython_runtime, __pyx_n_s_cline_in_traceback, Py_False); + } + else if (PyObject_Not(use_cline) != 0) { + c_line = 0; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; } #endif -/* PyObjectCallOneArg */ - #if CYTHON_COMPILING_IN_CPYTHON -static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_New(1); - if (unlikely(!args)) return NULL; - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 0, arg); - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, &arg, 1); +/* CodeObjectCache */ + static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; } -#endif -#ifdef __Pyx_CyFunction_USED - if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { -#else - if (likely(PyCFunction_Check(func))) { -#endif - if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { - return __Pyx_PyObject_CallMethO(func, arg); -#if CYTHON_FAST_PYCCALL - } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { - return __Pyx_PyCFunction_FastCall(func, &arg, 1); -#endif + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; } } - return __Pyx__PyObject_CallOneArg(func, arg); -} -#else -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_Pack(1, arg); - if (unlikely(!args)) return NULL; - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } } -#endif - -/* SaveResetException */ - #if CYTHON_FAST_THREAD_STATE -static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { - *type = tstate->exc_type; - *value = tstate->exc_value; - *tb = tstate->exc_traceback; - Py_XINCREF(*type); - Py_XINCREF(*value); - Py_XINCREF(*tb); +static PyCodeObject *__pyx_find_code_object(int code_line) { + PyCodeObject* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { + return NULL; + } + code_object = __pyx_code_cache.entries[pos].code_object; + Py_INCREF(code_object); + return code_object; } -static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = type; - tstate->exc_value = value; - tstate->exc_traceback = tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = 64; + __pyx_code_cache.count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { + PyCodeObject* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_DECREF(tmp); + return; + } + if (__pyx_code_cache.count == __pyx_code_cache.max_count) { + int new_max = __pyx_code_cache.max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = new_max; + } + for (i=__pyx_code_cache.count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + __pyx_code_cache.count++; + Py_INCREF(code_object); } -#endif -/* PyErrExceptionMatches */ - #if CYTHON_FAST_THREAD_STATE -static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err) { - PyObject *exc_type = tstate->curexc_type; - if (exc_type == err) return 1; - if (unlikely(!exc_type)) return 0; - return PyErr_GivenExceptionMatches(exc_type, err); +/* AddTraceback */ + #include "compile.h" +#include "frameobject.h" +#include "traceback.h" +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyObject *py_srcfile = 0; + PyObject *py_funcname = 0; + #if PY_MAJOR_VERSION < 3 + py_srcfile = PyString_FromString(filename); + #else + py_srcfile = PyUnicode_FromString(filename); + #endif + if (!py_srcfile) goto bad; + if (c_line) { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #else + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #endif + } + else { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromString(funcname); + #else + py_funcname = PyUnicode_FromString(funcname); + #endif + } + if (!py_funcname) goto bad; + py_code = __Pyx_PyCode_New( + 0, + 0, + 0, + 0, + 0, + __pyx_empty_bytes, /*PyObject *code,*/ + __pyx_empty_tuple, /*PyObject *consts,*/ + __pyx_empty_tuple, /*PyObject *names,*/ + __pyx_empty_tuple, /*PyObject *varnames,*/ + __pyx_empty_tuple, /*PyObject *freevars,*/ + __pyx_empty_tuple, /*PyObject *cellvars,*/ + py_srcfile, /*PyObject *filename,*/ + py_funcname, /*PyObject *name,*/ + py_line, + __pyx_empty_bytes /*PyObject *lnotab*/ + ); + Py_DECREF(py_srcfile); + Py_DECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_srcfile); + Py_XDECREF(py_funcname); + return NULL; } -#endif - -/* GetException */ - #if CYTHON_FAST_THREAD_STATE -static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { -#else -static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { -#endif - PyObject *local_type, *local_value, *local_tb; -#if CYTHON_FAST_THREAD_STATE - PyObject *tmp_type, *tmp_value, *tmp_tb; - local_type = tstate->curexc_type; - local_value = tstate->curexc_value; - local_tb = tstate->curexc_traceback; - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; -#else - PyErr_Fetch(&local_type, &local_value, &local_tb); -#endif - PyErr_NormalizeException(&local_type, &local_value, &local_tb); -#if CYTHON_FAST_THREAD_STATE - if (unlikely(tstate->curexc_type)) -#else - if (unlikely(PyErr_Occurred())) -#endif - goto bad; - #if PY_MAJOR_VERSION >= 3 - if (local_tb) { - if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) - goto bad; +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); } - #endif - Py_XINCREF(local_tb); - Py_XINCREF(local_type); - Py_XINCREF(local_value); - *type = local_type; - *value = local_value; - *tb = local_tb; -#if CYTHON_FAST_THREAD_STATE - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = local_type; - tstate->exc_value = local_value; - tstate->exc_traceback = local_tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -#else - PyErr_SetExcInfo(local_type, local_value, local_tb); -#endif - return 0; + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); bad: - *type = 0; - *value = 0; - *tb = 0; - Py_XDECREF(local_type); - Py_XDECREF(local_value); - Py_XDECREF(local_tb); - return -1; + Py_XDECREF(py_code); + Py_XDECREF(py_frame); } -/* PyObjectCallNoArg */ - #if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, NULL, 0); +/* CIntFromPyVerify */ + #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ } + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_enum__util_3a__3a_LoadMethod(enum util::LoadMethod value) { + const enum util::LoadMethod neg_one = (enum util::LoadMethod) -1, const_zero = (enum util::LoadMethod) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(enum util::LoadMethod) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(enum util::LoadMethod) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(enum util::LoadMethod) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); #endif -#ifdef __Pyx_CyFunction_USED - if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { -#else - if (likely(PyCFunction_Check(func))) { + } + } else { + if (sizeof(enum util::LoadMethod) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(enum util::LoadMethod) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); #endif - if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { - return __Pyx_PyObject_CallMethO(func, NULL); } } - return __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL); + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(enum util::LoadMethod), + little, !is_unsigned); + } } -#endif -/* Import */ - static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { - PyObject *empty_list = 0; - PyObject *module = 0; - PyObject *global_dict = 0; - PyObject *empty_dict = 0; - PyObject *list; - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_import; - py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s_import); - if (!py_import) - goto bad; - #endif - if (from_list) - list = from_list; - else { - empty_list = PyList_New(0); - if (!empty_list) - goto bad; - list = empty_list; - } - global_dict = PyModule_GetDict(__pyx_m); - if (!global_dict) - goto bad; - empty_dict = PyDict_New(); - if (!empty_dict) - goto bad; - { - #if PY_MAJOR_VERSION >= 3 - if (level == -1) { - if (strchr(__Pyx_MODULE_NAME, '.')) { - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_level = PyInt_FromLong(1); - if (!py_level) - goto bad; - module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, NULL); - Py_DECREF(py_level); - #else - module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, 1); - #endif - if (!module) { - if (!PyErr_ExceptionMatches(PyExc_ImportError)) - goto bad; - PyErr_Clear(); - } - } - level = 0; +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_uint64_t(uint64_t value) { + const uint64_t neg_one = (uint64_t) -1, const_zero = (uint64_t) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(uint64_t) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(uint64_t) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(uint64_t) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif } - #endif - if (!module) { - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_level = PyInt_FromLong(level); - if (!py_level) - goto bad; - module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, NULL); - Py_DECREF(py_level); - #else - module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, level); - #endif + } else { + if (sizeof(uint64_t) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(uint64_t) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif } } -bad: - #if PY_VERSION_HEX < 0x03030000 - Py_XDECREF(py_import); - #endif - Py_XDECREF(empty_list); - Py_XDECREF(empty_dict); - return module; + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(uint64_t), + little, !is_unsigned); + } } -/* CalculateMetaclass */ - static PyObject *__Pyx_CalculateMetaclass(PyTypeObject *metaclass, PyObject *bases) { - Py_ssize_t i, nbases = PyTuple_GET_SIZE(bases); - for (i=0; i < nbases; i++) { - PyTypeObject *tmptype; - PyObject *tmp = PyTuple_GET_ITEM(bases, i); - tmptype = Py_TYPE(tmp); -#if PY_MAJOR_VERSION < 3 - if (tmptype == &PyClass_Type) - continue; +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { + const int neg_one = (int) -1, const_zero = (int) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(int) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); #endif - if (!metaclass) { - metaclass = tmptype; - continue; } - if (PyType_IsSubtype(metaclass, tmptype)) - continue; - if (PyType_IsSubtype(tmptype, metaclass)) { - metaclass = tmptype; - continue; + } else { + if (sizeof(int) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif } - PyErr_SetString(PyExc_TypeError, - "metaclass conflict: " - "the metaclass of a derived class " - "must be a (non-strict) subclass " - "of the metaclasses of all its bases"); - return NULL; } - if (!metaclass) { -#if PY_MAJOR_VERSION < 3 - metaclass = &PyClass_Type; -#else - metaclass = &PyType_Type; -#endif + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(int), + little, !is_unsigned); } - Py_INCREF((PyObject*) metaclass); - return (PyObject*) metaclass; } -/* Py3ClassCreate */ - static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, - PyObject *qualname, PyObject *mkw, PyObject *modname, PyObject *doc) { - PyObject *ns; - if (metaclass) { - PyObject *prep = __Pyx_PyObject_GetAttrStr(metaclass, __pyx_n_s_prepare); - if (prep) { - PyObject *pargs = PyTuple_Pack(2, name, bases); - if (unlikely(!pargs)) { - Py_DECREF(prep); - return NULL; - } - ns = PyObject_Call(prep, pargs, mkw); - Py_DECREF(prep); - Py_DECREF(pargs); - } else { - if (unlikely(!PyErr_ExceptionMatches(PyExc_AttributeError))) - return NULL; - PyErr_Clear(); - ns = PyDict_New(); +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value) { + const unsigned int neg_one = (unsigned int) -1, const_zero = (unsigned int) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(unsigned int) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(unsigned int) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(unsigned int) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif } } else { - ns = PyDict_New(); - } - if (unlikely(!ns)) - return NULL; - if (unlikely(PyObject_SetItem(ns, __pyx_n_s_module, modname) < 0)) goto bad; - if (unlikely(PyObject_SetItem(ns, __pyx_n_s_qualname, qualname) < 0)) goto bad; - if (unlikely(doc && PyObject_SetItem(ns, __pyx_n_s_doc, doc) < 0)) goto bad; - return ns; -bad: - Py_DECREF(ns); - return NULL; -} -static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, - PyObject *dict, PyObject *mkw, - int calculate_metaclass, int allow_py2_metaclass) { - PyObject *result, *margs; - PyObject *owned_metaclass = NULL; - if (allow_py2_metaclass) { - owned_metaclass = PyObject_GetItem(dict, __pyx_n_s_metaclass); - if (owned_metaclass) { - metaclass = owned_metaclass; - } else if (likely(PyErr_ExceptionMatches(PyExc_KeyError))) { - PyErr_Clear(); - } else { - return NULL; + if (sizeof(unsigned int) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(unsigned int) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif } } - if (calculate_metaclass && (!metaclass || PyType_Check(metaclass))) { - metaclass = __Pyx_CalculateMetaclass((PyTypeObject*) metaclass, bases); - Py_XDECREF(owned_metaclass); - if (unlikely(!metaclass)) - return NULL; - owned_metaclass = metaclass; + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(unsigned int), + little, !is_unsigned); } - margs = PyTuple_Pack(3, name, bases, dict); - if (unlikely(!margs)) { - result = NULL; +} + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_char(unsigned char value) { + const unsigned char neg_one = (unsigned char) -1, const_zero = (unsigned char) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(unsigned char) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(unsigned char) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(unsigned char) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } } else { - result = PyObject_Call(metaclass, margs, mkw); - Py_DECREF(margs); + if (sizeof(unsigned char) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(unsigned char) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(unsigned char), + little, !is_unsigned); } - Py_XDECREF(owned_metaclass); - return result; } -/* CodeObjectCache */ - static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { - int start = 0, mid = 0, end = count - 1; - if (end >= 0 && code_line > entries[end].code_line) { - return count; - } - while (start < end) { - mid = start + (end - start) / 2; - if (code_line < entries[mid].code_line) { - end = mid; - } else if (code_line > entries[mid].code_line) { - start = mid + 1; +/* CIntFromPy */ + static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { + const int neg_one = (int) -1, const_zero = (int) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(int) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) } else { - return mid; + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (int) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int) 0; + case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) + case 2: + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(int) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int) 0; + case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) + case -2: + if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } +#endif + if (sizeof(int) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + int val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (int) -1; } - } - if (code_line <= entries[mid].code_line) { - return mid; } else { - return mid + 1; - } -} -static PyCodeObject *__pyx_find_code_object(int code_line) { - PyCodeObject* code_object; - int pos; - if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { - return NULL; - } - pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); - if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { - return NULL; - } - code_object = __pyx_code_cache.entries[pos].code_object; - Py_INCREF(code_object); - return code_object; -} -static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { - int pos, i; - __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; - if (unlikely(!code_line)) { - return; - } - if (unlikely(!entries)) { - entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); - if (likely(entries)) { - __pyx_code_cache.entries = entries; - __pyx_code_cache.max_count = 64; - __pyx_code_cache.count = 1; - entries[0].code_line = code_line; - entries[0].code_object = code_object; - Py_INCREF(code_object); - } - return; - } - pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); - if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { - PyCodeObject* tmp = entries[pos].code_object; - entries[pos].code_object = code_object; + int val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (int) -1; + val = __Pyx_PyInt_As_int(tmp); Py_DECREF(tmp); - return; - } - if (__pyx_code_cache.count == __pyx_code_cache.max_count) { - int new_max = __pyx_code_cache.max_count + 64; - entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( - __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); - if (unlikely(!entries)) { - return; - } - __pyx_code_cache.entries = entries; - __pyx_code_cache.max_count = new_max; - } - for (i=__pyx_code_cache.count; i>pos; i--) { - entries[i] = entries[i-1]; - } - entries[pos].code_line = code_line; - entries[pos].code_object = code_object; - __pyx_code_cache.count++; - Py_INCREF(code_object); -} - -/* AddTraceback */ - #include "compile.h" -#include "frameobject.h" -#include "traceback.h" -static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( - const char *funcname, int c_line, - int py_line, const char *filename) { - PyCodeObject *py_code = 0; - PyObject *py_srcfile = 0; - PyObject *py_funcname = 0; - #if PY_MAJOR_VERSION < 3 - py_srcfile = PyString_FromString(filename); - #else - py_srcfile = PyUnicode_FromString(filename); - #endif - if (!py_srcfile) goto bad; - if (c_line) { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); - #else - py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); - #endif - } - else { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromString(funcname); - #else - py_funcname = PyUnicode_FromString(funcname); - #endif - } - if (!py_funcname) goto bad; - py_code = __Pyx_PyCode_New( - 0, - 0, - 0, - 0, - 0, - __pyx_empty_bytes, /*PyObject *code,*/ - __pyx_empty_tuple, /*PyObject *consts,*/ - __pyx_empty_tuple, /*PyObject *names,*/ - __pyx_empty_tuple, /*PyObject *varnames,*/ - __pyx_empty_tuple, /*PyObject *freevars,*/ - __pyx_empty_tuple, /*PyObject *cellvars,*/ - py_srcfile, /*PyObject *filename,*/ - py_funcname, /*PyObject *name,*/ - py_line, - __pyx_empty_bytes /*PyObject *lnotab*/ - ); - Py_DECREF(py_srcfile); - Py_DECREF(py_funcname); - return py_code; -bad: - Py_XDECREF(py_srcfile); - Py_XDECREF(py_funcname); - return NULL; -} -static void __Pyx_AddTraceback(const char *funcname, int c_line, - int py_line, const char *filename) { - PyCodeObject *py_code = 0; - PyFrameObject *py_frame = 0; - py_code = __pyx_find_code_object(c_line ? c_line : py_line); - if (!py_code) { - py_code = __Pyx_CreateCodeObjectForTraceback( - funcname, c_line, py_line, filename); - if (!py_code) goto bad; - __pyx_insert_code_object(c_line ? c_line : py_line, py_code); + return val; } - py_frame = PyFrame_New( - PyThreadState_GET(), /*PyThreadState *tstate,*/ - py_code, /*PyCodeObject *code,*/ - __pyx_d, /*PyObject *globals,*/ - 0 /*PyObject *locals*/ - ); - if (!py_frame) goto bad; - __Pyx_PyFrame_SetLineNumber(py_frame, py_line); - PyTraceBack_Here(py_frame); -bad: - Py_XDECREF(py_code); - Py_XDECREF(py_frame); +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; } -/* CIntFromPyVerify */ - #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ - __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) -#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ - __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) -#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ - {\ - func_type value = func_value;\ - if (sizeof(target_type) < sizeof(func_type)) {\ - if (unlikely(value != (func_type) (target_type) value)) {\ - func_type zero = 0;\ - if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ - return (target_type) -1;\ - if (is_unsigned && unlikely(value < zero))\ - goto raise_neg_overflow;\ - else\ - goto raise_overflow;\ - }\ - }\ - return (target_type) value;\ - } - -/* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_enum__util_3a__3a_LoadMethod(enum util::LoadMethod value) { - const enum util::LoadMethod neg_one = (enum util::LoadMethod) -1, const_zero = (enum util::LoadMethod) 0; +/* CIntFromPy */ + static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { + const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(enum util::LoadMethod) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(enum util::LoadMethod) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(size_t) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(size_t, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (size_t) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (size_t) 0; + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, digits[0]) + case 2: + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 2 * PyLong_SHIFT) { + return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 3 * PyLong_SHIFT) { + return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 4 * PyLong_SHIFT) { + return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (size_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(size_t) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof(enum util::LoadMethod) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + } else if (sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (size_t) 0; + case -1: __PYX_VERIFY_RETURN_INT(size_t, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, +digits[0]) + case -2: + if (8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + } #endif - } - } else { - if (sizeof(enum util::LoadMethod) <= sizeof(long)) { - return PyInt_FromLong((long) value); + if (sizeof(size_t) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof(enum util::LoadMethod) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); + } else if (sizeof(size_t) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) #endif + } } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(enum util::LoadMethod), - little, !is_unsigned); - } -} - -/* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { - const int neg_one = (int) -1, const_zero = (int) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(int) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(int) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + size_t val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } #endif + return (size_t) -1; } } else { - if (sizeof(int) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); -#endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(int), - little, !is_unsigned); + size_t val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (size_t) -1; + val = __Pyx_PyInt_As_size_t(tmp); + Py_DECREF(tmp); + return val; } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to size_t"); + return (size_t) -1; } -/* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value) { - const unsigned int neg_one = (unsigned int) -1, const_zero = (unsigned int) 0; +/* CIntFromPy */ + static CYTHON_INLINE lm::WordIndex __Pyx_PyInt_As_lm_3a__3a_WordIndex(PyObject *x) { + const lm::WordIndex neg_one = (lm::WordIndex) -1, const_zero = (lm::WordIndex) 0; const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(unsigned int) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(unsigned int) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(unsigned int) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); -#endif +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(lm::WordIndex) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(lm::WordIndex, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (lm::WordIndex) val; } - } else { - if (sizeof(unsigned int) <= sizeof(long)) { - return PyInt_FromLong((long) value); + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (lm::WordIndex) 0; + case 1: __PYX_VERIFY_RETURN_INT(lm::WordIndex, digit, digits[0]) + case 2: + if (8 * sizeof(lm::WordIndex) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(lm::WordIndex, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(lm::WordIndex) >= 2 * PyLong_SHIFT) { + return (lm::WordIndex) (((((lm::WordIndex)digits[1]) << PyLong_SHIFT) | (lm::WordIndex)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(lm::WordIndex) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(lm::WordIndex, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(lm::WordIndex) >= 3 * PyLong_SHIFT) { + return (lm::WordIndex) (((((((lm::WordIndex)digits[2]) << PyLong_SHIFT) | (lm::WordIndex)digits[1]) << PyLong_SHIFT) | (lm::WordIndex)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(lm::WordIndex) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(lm::WordIndex, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(lm::WordIndex) >= 4 * PyLong_SHIFT) { + return (lm::WordIndex) (((((((((lm::WordIndex)digits[3]) << PyLong_SHIFT) | (lm::WordIndex)digits[2]) << PyLong_SHIFT) | (lm::WordIndex)digits[1]) << PyLong_SHIFT) | (lm::WordIndex)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (lm::WordIndex) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(lm::WordIndex) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(lm::WordIndex, unsigned long, PyLong_AsUnsignedLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof(unsigned int) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); + } else if (sizeof(lm::WordIndex) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(lm::WordIndex, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) #endif - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(unsigned int), - little, !is_unsigned); - } -} - -/* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_char(unsigned char value) { - const unsigned char neg_one = (unsigned char) -1, const_zero = (unsigned char) 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(unsigned char) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(unsigned char) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (lm::WordIndex) 0; + case -1: __PYX_VERIFY_RETURN_INT(lm::WordIndex, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(lm::WordIndex, digit, +digits[0]) + case -2: + if (8 * sizeof(lm::WordIndex) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(lm::WordIndex, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(lm::WordIndex) - 1 > 2 * PyLong_SHIFT) { + return (lm::WordIndex) (((lm::WordIndex)-1)*(((((lm::WordIndex)digits[1]) << PyLong_SHIFT) | (lm::WordIndex)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(lm::WordIndex) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(lm::WordIndex, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(lm::WordIndex) - 1 > 2 * PyLong_SHIFT) { + return (lm::WordIndex) ((((((lm::WordIndex)digits[1]) << PyLong_SHIFT) | (lm::WordIndex)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(lm::WordIndex) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(lm::WordIndex, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(lm::WordIndex) - 1 > 3 * PyLong_SHIFT) { + return (lm::WordIndex) (((lm::WordIndex)-1)*(((((((lm::WordIndex)digits[2]) << PyLong_SHIFT) | (lm::WordIndex)digits[1]) << PyLong_SHIFT) | (lm::WordIndex)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(lm::WordIndex) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(lm::WordIndex, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(lm::WordIndex) - 1 > 3 * PyLong_SHIFT) { + return (lm::WordIndex) ((((((((lm::WordIndex)digits[2]) << PyLong_SHIFT) | (lm::WordIndex)digits[1]) << PyLong_SHIFT) | (lm::WordIndex)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(lm::WordIndex) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(lm::WordIndex, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(lm::WordIndex) - 1 > 4 * PyLong_SHIFT) { + return (lm::WordIndex) (((lm::WordIndex)-1)*(((((((((lm::WordIndex)digits[3]) << PyLong_SHIFT) | (lm::WordIndex)digits[2]) << PyLong_SHIFT) | (lm::WordIndex)digits[1]) << PyLong_SHIFT) | (lm::WordIndex)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(lm::WordIndex) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(lm::WordIndex, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(lm::WordIndex) - 1 > 4 * PyLong_SHIFT) { + return (lm::WordIndex) ((((((((((lm::WordIndex)digits[3]) << PyLong_SHIFT) | (lm::WordIndex)digits[2]) << PyLong_SHIFT) | (lm::WordIndex)digits[1]) << PyLong_SHIFT) | (lm::WordIndex)digits[0]))); + } + } + break; + } +#endif + if (sizeof(lm::WordIndex) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(lm::WordIndex, long, PyLong_AsLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof(unsigned char) <= sizeof(unsigned PY_LONG_LONG)) { - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + } else if (sizeof(lm::WordIndex) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(lm::WordIndex, PY_LONG_LONG, PyLong_AsLongLong(x)) #endif + } } - } else { - if (sizeof(unsigned char) <= sizeof(long)) { - return PyInt_FromLong((long) value); -#ifdef HAVE_LONG_LONG - } else if (sizeof(unsigned char) <= sizeof(PY_LONG_LONG)) { - return PyLong_FromLongLong((PY_LONG_LONG) value); + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + lm::WordIndex val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } #endif + return (lm::WordIndex) -1; } + } else { + lm::WordIndex val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (lm::WordIndex) -1; + val = __Pyx_PyInt_As_lm_3a__3a_WordIndex(tmp); + Py_DECREF(tmp); + return val; } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(unsigned char), - little, !is_unsigned); - } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to lm::WordIndex"); + return (lm::WordIndex) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to lm::WordIndex"); + return (lm::WordIndex) -1; } /* CIntFromPy */ - static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { - const int neg_one = (int) -1, const_zero = (int) 0; + static CYTHON_INLINE uint64_t __Pyx_PyInt_As_uint64_t(PyObject *x) { + const uint64_t neg_one = (uint64_t) -1, const_zero = (uint64_t) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { - if (sizeof(int) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) + if (sizeof(uint64_t) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(uint64_t, long, PyInt_AS_LONG(x)) } else { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { goto raise_neg_overflow; } - return (int) val; + return (uint64_t) val; } } else #endif @@ -6829,32 +11782,32 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { - case 0: return (int) 0; - case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) + case 0: return (uint64_t) 0; + case 1: __PYX_VERIFY_RETURN_INT(uint64_t, digit, digits[0]) case 2: - if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { - return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) >= 2 * PyLong_SHIFT) { + return (uint64_t) (((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); } } break; case 3: - if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { - return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) >= 3 * PyLong_SHIFT) { + return (uint64_t) (((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); } } break; case 4: - if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { - return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) >= 4 * PyLong_SHIFT) { + return (uint64_t) (((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); } } break; @@ -6868,86 +11821,86 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, { int result = PyObject_RichCompareBool(x, Py_False, Py_LT); if (unlikely(result < 0)) - return (int) -1; + return (uint64_t) -1; if (unlikely(result == 1)) goto raise_neg_overflow; } #endif - if (sizeof(int) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) + if (sizeof(uint64_t) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(uint64_t, unsigned long, PyLong_AsUnsignedLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } else if (sizeof(uint64_t) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(uint64_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) #endif } } else { #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { - case 0: return (int) 0; - case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) + case 0: return (uint64_t) 0; + case -1: __PYX_VERIFY_RETURN_INT(uint64_t, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(uint64_t, digit, +digits[0]) case -2: - if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) - 1 > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT) { + return (uint64_t) (((uint64_t)-1)*(((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); } } break; case 2: - if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { - return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT) { + return (uint64_t) ((((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); } } break; case -3: - if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT) { + return (uint64_t) (((uint64_t)-1)*(((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); } } break; case 3: - if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { - return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT) { + return (uint64_t) ((((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); } } break; case -4: - if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { - return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 4 * PyLong_SHIFT) { + return (uint64_t) (((uint64_t)-1)*(((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); } } break; case 4: - if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(uint64_t) > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) - } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { - return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(uint64_t) - 1 > 4 * PyLong_SHIFT) { + return (uint64_t) ((((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); } } break; } #endif - if (sizeof(int) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) + if (sizeof(uint64_t) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(uint64_t, long, PyLong_AsLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { - __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) + } else if (sizeof(uint64_t) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(uint64_t, PY_LONG_LONG, PyLong_AsLongLong(x)) #endif } } @@ -6956,7 +11909,7 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, PyErr_SetString(PyExc_RuntimeError, "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); #else - int val; + uint64_t val; PyObject *v = __Pyx_PyNumber_IntOrLong(x); #if PY_MAJOR_VERSION < 3 if (likely(v) && !PyLong_Check(v)) { @@ -6976,28 +11929,28 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, return val; } #endif - return (int) -1; + return (uint64_t) -1; } } else { - int val; + uint64_t val; PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); - if (!tmp) return (int) -1; - val = __Pyx_PyInt_As_int(tmp); + if (!tmp) return (uint64_t) -1; + val = __Pyx_PyInt_As_uint64_t(tmp); Py_DECREF(tmp); return val; } raise_overflow: PyErr_SetString(PyExc_OverflowError, - "value too large to convert to int"); - return (int) -1; + "value too large to convert to uint64_t"); + return (uint64_t) -1; raise_neg_overflow: PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to int"); - return (int) -1; + "can't convert negative value to uint64_t"); + return (uint64_t) -1; } /* CIntFromPy */ - static CYTHON_INLINE enum util::LoadMethod __Pyx_PyInt_As_enum__util_3a__3a_LoadMethod(PyObject *x) { + static CYTHON_INLINE enum util::LoadMethod __Pyx_PyInt_As_enum__util_3a__3a_LoadMethod(PyObject *x) { const enum util::LoadMethod neg_one = (enum util::LoadMethod) -1, const_zero = (enum util::LoadMethod) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 @@ -7186,7 +12139,7 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, } /* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { const long neg_one = (long) -1, const_zero = (long) 0; const int is_unsigned = neg_one > const_zero; if (is_unsigned) { @@ -7217,7 +12170,7 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, } /* CIntFromPy */ - static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { + static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { const long neg_one = (long) -1, const_zero = (long) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 @@ -7405,8 +12358,80 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, return (long) -1; } +/* FastTypeChecks */ + #if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = a->tp_base; + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +#if PY_MAJOR_VERSION == 2 +static int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject* exc_type2) { + PyObject *exception, *value, *tb; + int res; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&exception, &value, &tb); + res = exc_type1 ? PyObject_IsSubclass(err, exc_type1) : 0; + if (unlikely(res == -1)) { + PyErr_WriteUnraisable(err); + res = 0; + } + if (!res) { + res = PyObject_IsSubclass(err, exc_type2); + if (unlikely(res == -1)) { + PyErr_WriteUnraisable(err); + res = 0; + } + } + __Pyx_ErrRestore(exception, value, tb); + return res; +} +#else +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + int res = exc_type1 ? __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type1) : 0; + if (!res) { + res = __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } + return res; +} +#endif +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject* exc_type) { + if (likely(err == exc_type)) return 1; + if (likely(PyExceptionClass_Check(err))) { + return __Pyx_inner_PyErr_GivenExceptionMatches2(err, NULL, exc_type); + } + return PyErr_GivenExceptionMatches(err, exc_type); +} +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *exc_type1, PyObject *exc_type2) { + if (likely(err == exc_type1 || err == exc_type2)) return 1; + if (likely(PyExceptionClass_Check(err))) { + return __Pyx_inner_PyErr_GivenExceptionMatches2(err, exc_type1, exc_type2); + } + return (PyErr_GivenExceptionMatches(err, exc_type1) || PyErr_GivenExceptionMatches(err, exc_type2)); +} +#endif + /* FetchCommonType */ - static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { + static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { PyObject* fake_module; PyTypeObject* cached_type = NULL; fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); @@ -7445,15 +12470,24 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, } /* SwapException */ - #if CYTHON_FAST_THREAD_STATE + #if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { PyObject *tmp_type, *tmp_value, *tmp_tb; + #if PY_VERSION_HEX >= 0x030700A2 + tmp_type = tstate->exc_state.exc_type; + tmp_value = tstate->exc_state.exc_value; + tmp_tb = tstate->exc_state.exc_traceback; + tstate->exc_state.exc_type = *type; + tstate->exc_state.exc_value = *value; + tstate->exc_state.exc_traceback = *tb; + #else tmp_type = tstate->exc_type; tmp_value = tstate->exc_value; tmp_tb = tstate->exc_traceback; tstate->exc_type = *type; tstate->exc_value = *value; tstate->exc_traceback = *tb; + #endif *type = tmp_type; *value = tmp_value; *tb = tmp_tb; @@ -7470,10 +12504,8 @@ static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, #endif /* PyObjectCallMethod1 */ - static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { - PyObject *method, *result = NULL; - method = __Pyx_PyObject_GetAttrStr(obj, method_name); - if (unlikely(!method)) goto done; + static PyObject* __Pyx__PyObject_CallMethod1(PyObject* method, PyObject* arg) { + PyObject *result = NULL; #if CYTHON_UNPACK_METHODS if (likely(PyMethod_Check(method))) { PyObject *self = PyMethod_GET_SELF(method); @@ -7501,7 +12533,6 @@ static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, Py_INCREF(arg); PyTuple_SET_ITEM(args, 1, arg); Py_INCREF(function); - Py_DECREF(method); method = NULL; result = __Pyx_PyObject_Call(function, args, NULL); Py_DECREF(args); Py_DECREF(function); @@ -7510,24 +12541,27 @@ static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, } #endif result = __Pyx_PyObject_CallOneArg(method, arg); + goto done; +done: + return result; +} +static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { + PyObject *method, *result = NULL; + method = __Pyx_PyObject_GetAttrStr(obj, method_name); + if (unlikely(!method)) goto done; + result = __Pyx__PyObject_CallMethod1(method, arg); done: Py_XDECREF(method); return result; } /* CoroutineBase */ - #include + #include #include -static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value); -static PyObject *__Pyx_Coroutine_Close(PyObject *self); -static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args); #define __Pyx_Coroutine_Undelegate(gen) Py_CLEAR((gen)->yieldfrom) -#if 1 || PY_VERSION_HEX < 0x030300B0 -static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) { +static int __Pyx_PyGen__FetchStopIterationValue(CYTHON_UNUSED PyThreadState *__pyx_tstate, PyObject **pvalue) { PyObject *et, *ev, *tb; PyObject *value = NULL; - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign __Pyx_ErrFetch(&et, &ev, &tb); if (!et) { Py_XDECREF(tb); @@ -7562,7 +12596,7 @@ static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) { } Py_DECREF(ev); } - else if (!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration)) { + else if (!__Pyx_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration)) { value = ev; } if (likely(value)) { @@ -7571,7 +12605,7 @@ static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) { *pvalue = value; return 0; } - } else if (!PyErr_GivenExceptionMatches(et, PyExc_StopIteration)) { + } else if (!__Pyx_PyErr_GivenExceptionMatches(et, PyExc_StopIteration)) { __Pyx_ErrRestore(et, ev, tb); return -1; } @@ -7604,7 +12638,6 @@ static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) { *pvalue = value; return 0; } -#endif static CYTHON_INLINE void __Pyx_Coroutine_ExceptionClear(__pyx_CoroutineObject *self) { PyObject *exc_type = self->exc_type; @@ -7617,72 +12650,121 @@ void __Pyx_Coroutine_ExceptionClear(__pyx_CoroutineObject *self) { Py_XDECREF(exc_value); Py_XDECREF(exc_traceback); } -static CYTHON_INLINE -int __Pyx_Coroutine_CheckRunning(__pyx_CoroutineObject *gen) { - if (unlikely(gen->is_running)) { - PyErr_SetString(PyExc_ValueError, - "generator already executing"); - return 1; +#define __Pyx_Coroutine_AlreadyRunningError(gen) (__Pyx__Coroutine_AlreadyRunningError(gen), (PyObject*)NULL) +static void __Pyx__Coroutine_AlreadyRunningError(CYTHON_UNUSED __pyx_CoroutineObject *gen) { + const char *msg; + if (0) { + #ifdef __Pyx_Coroutine_USED + } else if (__Pyx_Coroutine_CheckExact((PyObject*)gen)) { + msg = "coroutine already executing"; + #endif + #ifdef __Pyx_AsyncGen_USED + } else if (__Pyx_AsyncGen_CheckExact((PyObject*)gen)) { + msg = "async generator already executing"; + #endif + } else { + msg = "generator already executing"; } - return 0; + PyErr_SetString(PyExc_ValueError, msg); } -static CYTHON_INLINE -PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value) { - PyObject *retval; +#define __Pyx_Coroutine_NotStartedError(gen) (__Pyx__Coroutine_NotStartedError(gen), (PyObject*)NULL) +static void __Pyx__Coroutine_NotStartedError(CYTHON_UNUSED PyObject *gen) { + const char *msg; + if (0) { + #ifdef __Pyx_Coroutine_USED + } else if (__Pyx_Coroutine_CheckExact(gen)) { + msg = "can't send non-None value to a just-started coroutine"; + #endif + #ifdef __Pyx_AsyncGen_USED + } else if (__Pyx_AsyncGen_CheckExact(gen)) { + msg = "can't send non-None value to a just-started async generator"; + #endif + } else { + msg = "can't send non-None value to a just-started generator"; + } + PyErr_SetString(PyExc_TypeError, msg); +} +#define __Pyx_Coroutine_AlreadyTerminatedError(gen, value, closing) (__Pyx__Coroutine_AlreadyTerminatedError(gen, value, closing), (PyObject*)NULL) +static void __Pyx__Coroutine_AlreadyTerminatedError(CYTHON_UNUSED PyObject *gen, PyObject *value, CYTHON_UNUSED int closing) { + #ifdef __Pyx_Coroutine_USED + if (!closing && __Pyx_Coroutine_CheckExact(gen)) { + PyErr_SetString(PyExc_RuntimeError, "cannot reuse already awaited coroutine"); + } else + #endif + if (value) { + #ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(gen)) + PyErr_SetNone(__Pyx_PyExc_StopAsyncIteration); + else + #endif + PyErr_SetNone(PyExc_StopIteration); + } +} +static +PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, int closing) { __Pyx_PyThreadState_declare + PyThreadState *tstate; + PyObject *retval; assert(!self->is_running); if (unlikely(self->resume_label == 0)) { if (unlikely(value && value != Py_None)) { - PyErr_SetString(PyExc_TypeError, - "can't send non-None value to a " - "just-started generator"); - return NULL; + return __Pyx_Coroutine_NotStartedError((PyObject*)self); } } if (unlikely(self->resume_label == -1)) { - PyErr_SetNone(PyExc_StopIteration); - return NULL; + return __Pyx_Coroutine_AlreadyTerminatedError((PyObject*)self, value, closing); } +#if CYTHON_FAST_THREAD_STATE __Pyx_PyThreadState_assign - if (value) { + tstate = __pyx_tstate; +#else + tstate = __Pyx_PyThreadState_Current; +#endif + if (self->exc_type) { #if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON #else if (self->exc_traceback) { PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; PyFrameObject *f = tb->tb_frame; - Py_XINCREF(__pyx_tstate->frame); + Py_XINCREF(tstate->frame); assert(f->f_back == NULL); - f->f_back = __pyx_tstate->frame; + f->f_back = tstate->frame; } #endif __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, &self->exc_traceback); } else { __Pyx_Coroutine_ExceptionClear(self); + __Pyx_ExceptionSave(&self->exc_type, &self->exc_value, &self->exc_traceback); } self->is_running = 1; - retval = self->body((PyObject *) self, value); + retval = self->body((PyObject *) self, tstate, value); self->is_running = 0; - if (retval) { - __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, - &self->exc_traceback); + return retval; +} +static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__pyx_CoroutineObject *self) { + if (likely(self->exc_traceback)) { #if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON #else - if (self->exc_traceback) { - PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; - PyFrameObject *f = tb->tb_frame; - Py_CLEAR(f->f_back); - } + PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; + PyFrameObject *f = tb->tb_frame; + Py_CLEAR(f->f_back); #endif - } else { - __Pyx_Coroutine_ExceptionClear(self); } - return retval; } static CYTHON_INLINE -PyObject *__Pyx_Coroutine_MethodReturn(PyObject *retval) { - if (unlikely(!retval && !PyErr_Occurred())) { - PyErr_SetNone(PyExc_StopIteration); +PyObject *__Pyx_Coroutine_MethodReturn(CYTHON_UNUSED PyObject* gen, PyObject *retval) { + if (unlikely(!retval)) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (!__Pyx_PyErr_Occurred()) { + PyObject *exc = PyExc_StopIteration; + #ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(gen)) + exc = __Pyx_PyExc_StopAsyncIteration; + #endif + __Pyx_PyErr_SetNone(exc); + } } return retval; } @@ -7691,8 +12773,8 @@ PyObject *__Pyx_Coroutine_FinishDelegation(__pyx_CoroutineObject *gen) { PyObject *ret; PyObject *val = NULL; __Pyx_Coroutine_Undelegate(gen); - __Pyx_PyGen_FetchStopIterationValue(&val); - ret = __Pyx_Coroutine_SendEx(gen, val); + __Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, &val); + ret = __Pyx_Coroutine_SendEx(gen, val, 0); Py_XDECREF(val); return ret; } @@ -7700,8 +12782,8 @@ static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) { PyObject *retval; __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; PyObject *yf = gen->yieldfrom; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; + if (unlikely(gen->is_running)) + return __Pyx_Coroutine_AlreadyRunningError(gen); if (yf) { PyObject *ret; gen->is_running = 1; @@ -7715,6 +12797,21 @@ static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) { ret = __Pyx_Coroutine_Send(yf, value); } else #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_PyAsyncGenASend_CheckExact(yf)) { + ret = __Pyx_async_gen_asend_send(yf, value); + } else + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03030000 && (defined(__linux__) || PY_VERSION_HEX >= 0x030600B3) + if (PyGen_CheckExact(yf)) { + ret = _PyGen_Send((PyGenObject*)yf, value == Py_None ? NULL : value); + } else + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03050000 && defined(PyCoro_CheckExact) && (defined(__linux__) || PY_VERSION_HEX >= 0x030600B3) + if (PyCoro_CheckExact(yf)) { + ret = _PyGen_Send((PyGenObject*)yf, value == Py_None ? NULL : value); + } else + #endif { if (value == Py_None) ret = Py_TYPE(yf)->tp_iternext(yf); @@ -7727,9 +12824,9 @@ static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) { } retval = __Pyx_Coroutine_FinishDelegation(gen); } else { - retval = __Pyx_Coroutine_SendEx(gen, value); + retval = __Pyx_Coroutine_SendEx(gen, value, 0); } - return __Pyx_Coroutine_MethodReturn(retval); + return __Pyx_Coroutine_MethodReturn(self, retval); } static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) { PyObject *retval = NULL; @@ -7747,6 +12844,19 @@ static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) { if (!retval) return -1; } else + if (__Pyx_CoroutineAwait_CheckExact(yf)) { + retval = __Pyx_CoroutineAwait_Close((__pyx_CoroutineAwaitObject*)yf); + if (!retval) + return -1; + } else + #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_PyAsyncGenASend_CheckExact(yf)) { + retval = __Pyx_async_gen_asend_close(yf, NULL); + } else + if (__pyx_PyAsyncGenAThrow_CheckExact(yf)) { + retval = __Pyx_async_gen_athrow_close(yf, NULL); + } else #endif { PyObject *meth; @@ -7771,8 +12881,8 @@ static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) { static PyObject *__Pyx_Generator_Next(PyObject *self) { __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; PyObject *yf = gen->yieldfrom; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; + if (unlikely(gen->is_running)) + return __Pyx_Coroutine_AlreadyRunningError(gen); if (yf) { PyObject *ret; gen->is_running = 1; @@ -7780,6 +12890,11 @@ static PyObject *__Pyx_Generator_Next(PyObject *self) { if (__Pyx_Generator_CheckExact(yf)) { ret = __Pyx_Generator_Next(yf); } else + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03030000 && (defined(__linux__) || PY_VERSION_HEX >= 0x030600B3) + if (PyGen_CheckExact(yf)) { + ret = _PyGen_Send((PyGenObject*)yf, NULL); + } else #endif ret = Py_TYPE(yf)->tp_iternext(yf); gen->is_running = 0; @@ -7788,15 +12903,15 @@ static PyObject *__Pyx_Generator_Next(PyObject *self) { } return __Pyx_Coroutine_FinishDelegation(gen); } - return __Pyx_Coroutine_SendEx(gen, Py_None); + return __Pyx_Coroutine_SendEx(gen, Py_None, 0); } static PyObject *__Pyx_Coroutine_Close(PyObject *self) { __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; PyObject *retval, *raised_exception; PyObject *yf = gen->yieldfrom; int err = 0; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; + if (unlikely(gen->is_running)) + return __Pyx_Coroutine_AlreadyRunningError(gen); if (yf) { Py_INCREF(yf); err = __Pyx_Coroutine_CloseIter(gen, yf); @@ -7805,59 +12920,69 @@ static PyObject *__Pyx_Coroutine_Close(PyObject *self) { } if (err == 0) PyErr_SetNone(PyExc_GeneratorExit); - retval = __Pyx_Coroutine_SendEx(gen, NULL); - if (retval) { + retval = __Pyx_Coroutine_SendEx(gen, NULL, 1); + if (unlikely(retval)) { + const char *msg; Py_DECREF(retval); - PyErr_SetString(PyExc_RuntimeError, - "generator ignored GeneratorExit"); + if ((0)) { + #ifdef __Pyx_Coroutine_USED + } else if (__Pyx_Coroutine_CheckExact(self)) { + msg = "coroutine ignored GeneratorExit"; + #endif + #ifdef __Pyx_AsyncGen_USED + } else if (__Pyx_AsyncGen_CheckExact(self)) { +#if PY_VERSION_HEX < 0x03060000 + msg = "async generator ignored GeneratorExit - might require Python 3.6+ finalisation (PEP 525)"; +#else + msg = "async generator ignored GeneratorExit"; +#endif + #endif + } else { + msg = "generator ignored GeneratorExit"; + } + PyErr_SetString(PyExc_RuntimeError, msg); return NULL; } raised_exception = PyErr_Occurred(); - if (!raised_exception - || raised_exception == PyExc_StopIteration - || raised_exception == PyExc_GeneratorExit - || PyErr_GivenExceptionMatches(raised_exception, PyExc_GeneratorExit) - || PyErr_GivenExceptionMatches(raised_exception, PyExc_StopIteration)) - { + if (likely(!raised_exception || __Pyx_PyErr_GivenExceptionMatches2(raised_exception, PyExc_GeneratorExit, PyExc_StopIteration))) { if (raised_exception) PyErr_Clear(); Py_INCREF(Py_None); return Py_None; } return NULL; } -static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { +static PyObject *__Pyx__Coroutine_Throw(PyObject *self, PyObject *typ, PyObject *val, PyObject *tb, + PyObject *args, int close_on_genexit) { __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - PyObject *typ; - PyObject *tb = NULL; - PyObject *val = NULL; PyObject *yf = gen->yieldfrom; - if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb)) - return NULL; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; + if (unlikely(gen->is_running)) + return __Pyx_Coroutine_AlreadyRunningError(gen); if (yf) { PyObject *ret; Py_INCREF(yf); - if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) { + if (__Pyx_PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit) && close_on_genexit) { int err = __Pyx_Coroutine_CloseIter(gen, yf); Py_DECREF(yf); __Pyx_Coroutine_Undelegate(gen); if (err < 0) - return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL)); + return __Pyx_Coroutine_MethodReturn(self, __Pyx_Coroutine_SendEx(gen, NULL, 0)); goto throw_here; } gen->is_running = 1; + if (0 #ifdef __Pyx_Generator_USED - if (__Pyx_Generator_CheckExact(yf)) { - ret = __Pyx_Coroutine_Throw(yf, args); - } else + || __Pyx_Generator_CheckExact(yf) #endif #ifdef __Pyx_Coroutine_USED - if (__Pyx_Coroutine_CheckExact(yf)) { - ret = __Pyx_Coroutine_Throw(yf, args); - } else + || __Pyx_Coroutine_CheckExact(yf) #endif - { + ) { + ret = __Pyx__Coroutine_Throw(yf, typ, val, tb, args, close_on_genexit); + #ifdef __Pyx_Coroutine_USED + } else if (__Pyx_CoroutineAwait_CheckExact(yf)) { + ret = __Pyx__Coroutine_Throw(((__pyx_CoroutineAwaitObject*)yf)->coroutine, typ, val, tb, args, close_on_genexit); + #endif + } else { PyObject *meth = __Pyx_PyObject_GetAttrStr(yf, __pyx_n_s_throw); if (unlikely(!meth)) { Py_DECREF(yf); @@ -7870,7 +12995,11 @@ static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { gen->is_running = 0; goto throw_here; } - ret = PyObject_CallObject(meth, args); + if (likely(args)) { + ret = PyObject_CallObject(meth, args); + } else { + ret = PyObject_CallFunctionObjArgs(meth, typ, val, tb, NULL); + } Py_DECREF(meth); } gen->is_running = 0; @@ -7878,14 +13007,21 @@ static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { if (!ret) { ret = __Pyx_Coroutine_FinishDelegation(gen); } - return __Pyx_Coroutine_MethodReturn(ret); + return __Pyx_Coroutine_MethodReturn(self, ret); } throw_here: __Pyx_Raise(typ, val, tb, NULL); - return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL)); + return __Pyx_Coroutine_MethodReturn(self, __Pyx_Coroutine_SendEx(gen, NULL, 0)); } -static int __Pyx_Coroutine_traverse(PyObject *self, visitproc visit, void *arg) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; +static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { + PyObject *typ; + PyObject *val = NULL; + PyObject *tb = NULL; + if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb)) + return NULL; + return __Pyx__Coroutine_Throw(self, typ, val, tb, args, 1); +} +static int __Pyx_Coroutine_traverse(__pyx_CoroutineObject *gen, visitproc visit, void *arg) { Py_VISIT(gen->closure); Py_VISIT(gen->classobj); Py_VISIT(gen->yieldfrom); @@ -7902,8 +13038,14 @@ static int __Pyx_Coroutine_clear(PyObject *self) { Py_CLEAR(gen->exc_type); Py_CLEAR(gen->exc_value); Py_CLEAR(gen->exc_traceback); +#ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(self)) { + Py_CLEAR(((__pyx_PyAsyncGenObject*)gen)->ag_finalizer); + } +#endif Py_CLEAR(gen->gi_name); Py_CLEAR(gen->gi_qualname); + Py_CLEAR(gen->gi_modulename); return 0; } static void __Pyx_Coroutine_dealloc(PyObject *self) { @@ -7911,9 +13053,9 @@ static void __Pyx_Coroutine_dealloc(PyObject *self) { PyObject_GC_UnTrack(gen); if (gen->gi_weakreflist != NULL) PyObject_ClearWeakRefs(self); - if (gen->resume_label > 0) { + if (gen->resume_label >= 0) { PyObject_GC_Track(self); -#if PY_VERSION_HEX >= 0x030400a1 +#if PY_VERSION_HEX >= 0x030400a1 && CYTHON_USE_TP_FINALIZE if (PyObject_CallFinalizerFromDealloc(self)) #else Py_TYPE(gen)->tp_del(self); @@ -7924,29 +13066,93 @@ static void __Pyx_Coroutine_dealloc(PyObject *self) { } PyObject_GC_UnTrack(self); } +#ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(self)) { + /* We have to handle this case for asynchronous generators + right here, because this code has to be between UNTRACK + and GC_Del. */ + Py_CLEAR(((__pyx_PyAsyncGenObject*)self)->ag_finalizer); + } +#endif __Pyx_Coroutine_clear(self); PyObject_GC_Del(gen); } static void __Pyx_Coroutine_del(PyObject *self) { - PyObject *res; PyObject *error_type, *error_value, *error_traceback; __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; __Pyx_PyThreadState_declare - if (gen->resume_label <= 0) - return ; -#if PY_VERSION_HEX < 0x030400a1 + if (gen->resume_label < 0) { + return; + } +#if !CYTHON_USE_TP_FINALIZE assert(self->ob_refcnt == 0); self->ob_refcnt = 1; #endif __Pyx_PyThreadState_assign __Pyx_ErrFetch(&error_type, &error_value, &error_traceback); - res = __Pyx_Coroutine_Close(self); - if (res == NULL) - PyErr_WriteUnraisable(self); - else - Py_DECREF(res); +#ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(self)) { + __pyx_PyAsyncGenObject *agen = (__pyx_PyAsyncGenObject*)self; + PyObject *finalizer = agen->ag_finalizer; + if (finalizer && !agen->ag_closed) { + PyObject *res = __Pyx_PyObject_CallOneArg(finalizer, self); + if (unlikely(!res)) { + PyErr_WriteUnraisable(self); + } else { + Py_DECREF(res); + } + __Pyx_ErrRestore(error_type, error_value, error_traceback); + return; + } + } +#endif + if (unlikely(gen->resume_label == 0 && !error_value)) { +#ifdef __Pyx_Coroutine_USED +#ifdef __Pyx_Generator_USED + if (!__Pyx_Generator_CheckExact(self)) +#endif + { + PyObject_GC_UnTrack(self); +#if PY_MAJOR_VERSION >= 3 || defined(PyErr_WarnFormat) + if (unlikely(PyErr_WarnFormat(PyExc_RuntimeWarning, 1, "coroutine '%.50S' was never awaited", gen->gi_qualname) < 0)) + PyErr_WriteUnraisable(self); +#else + {PyObject *msg; + char *cmsg; + #if CYTHON_COMPILING_IN_PYPY + msg = NULL; + cmsg = (char*) "coroutine was never awaited"; + #else + char *cname; + PyObject *qualname; + qualname = gen->gi_qualname; + cname = PyString_AS_STRING(qualname); + msg = PyString_FromFormat("coroutine '%.50s' was never awaited", cname); + if (unlikely(!msg)) { + PyErr_Clear(); + cmsg = (char*) "coroutine was never awaited"; + } else { + cmsg = PyString_AS_STRING(msg); + } + #endif + if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, cmsg, 1) < 0)) + PyErr_WriteUnraisable(self); + Py_XDECREF(msg);} +#endif + PyObject_GC_Track(self); + } +#endif + } else { + PyObject *res = __Pyx_Coroutine_Close(self); + if (unlikely(!res)) { + if (PyErr_Occurred()) + PyErr_WriteUnraisable(self); + } else { + Py_DECREF(res); + } + } __Pyx_ErrRestore(error_type, error_value, error_traceback); -#if PY_VERSION_HEX < 0x030400a1 +#if !CYTHON_USE_TP_FINALIZE assert(self->ob_refcnt > 0); if (--self->ob_refcnt == 0) { return; @@ -8025,8 +13231,13 @@ static __pyx_CoroutineObject *__Pyx__Coroutine_New( PyTypeObject* type, __pyx_coroutine_body_t body, PyObject *closure, PyObject *name, PyObject *qualname, PyObject *module_name) { __pyx_CoroutineObject *gen = PyObject_GC_New(__pyx_CoroutineObject, type); - if (gen == NULL) + if (unlikely(!gen)) return NULL; + return __Pyx__Coroutine_NewInit(gen, body, closure, name, qualname, module_name); +} +static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit( + __pyx_CoroutineObject *gen, __pyx_coroutine_body_t body, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name) { gen->body = body; gen->closure = closure; Py_XINCREF(closure); @@ -8049,7 +13260,7 @@ static __pyx_CoroutineObject *__Pyx__Coroutine_New( } /* PatchModuleWithCoroutine */ - static PyObject* __Pyx_Coroutine_patch_module(PyObject* module, const char* py_code) { + static PyObject* __Pyx_Coroutine_patch_module(PyObject* module, const char* py_code) { #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) int result; PyObject *globals, *result_obj; @@ -8089,7 +13300,10 @@ static __pyx_CoroutineObject *__Pyx__Coroutine_New( } /* PatchGeneratorABC */ - #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) + #ifndef CYTHON_REGISTER_ABCS +#define CYTHON_REGISTER_ABCS 1 +#endif +#if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) static PyObject* __Pyx_patch_abc_module(PyObject *module); static PyObject* __Pyx_patch_abc_module(PyObject *module) { module = __Pyx_Coroutine_patch_module( @@ -8109,13 +13323,13 @@ static PyObject* __Pyx_patch_abc_module(PyObject *module) { static int __Pyx_patch_abc(void) { #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) static int abc_patched = 0; - if (!abc_patched) { + if (CYTHON_REGISTER_ABCS && !abc_patched) { PyObject *module; - module = PyImport_ImportModule((PY_VERSION_HEX >= 0x03030000) ? "collections.abc" : "collections"); + module = PyImport_ImportModule((PY_MAJOR_VERSION >= 3) ? "collections.abc" : "collections"); if (!module) { PyErr_WriteUnraisable(NULL); if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, - ((PY_VERSION_HEX >= 0x03030000) ? + ((PY_MAJOR_VERSION >= 3) ? "Cython module failed to register with collections.abc module" : "Cython module failed to register with collections module"), 1) < 0)) { return -1; @@ -8137,13 +13351,13 @@ static int __Pyx_patch_abc(void) { } } #else - if (0) __Pyx_Coroutine_patch_module(NULL, NULL); + if ((0)) __Pyx_Coroutine_patch_module(NULL, NULL); #endif return 0; } /* Generator */ - static PyMethodDef __pyx_Generator_methods[] = { + static PyMethodDef __pyx_Generator_methods[] = { {"send", (PyCFunction) __Pyx_Coroutine_Send, METH_O, (char*) PyDoc_STR("send(arg) -> send 'arg' into generator,\nreturn next yielded value or raise StopIteration.")}, {"throw", (PyCFunction) __Pyx_Coroutine_Throw, METH_VARARGS, @@ -8211,14 +13425,16 @@ static PyTypeObject __pyx_GeneratorType_type = { 0, 0, 0, -#if PY_VERSION_HEX >= 0x030400a1 +#if CYTHON_USE_TP_FINALIZE 0, #else __Pyx_Coroutine_del, #endif 0, -#if PY_VERSION_HEX >= 0x030400a1 +#if CYTHON_USE_TP_FINALIZE __Pyx_Coroutine_del, +#elif PY_VERSION_HEX >= 0x030400a1 + 0, #endif }; static int __pyx_Generator_init(void) { @@ -8232,7 +13448,7 @@ static int __pyx_Generator_init(void) { } /* CheckBinaryVersion */ - static int __Pyx_check_binary_version(void) { + static int __Pyx_check_binary_version(void) { char ctversion[4], rtversion[4]; PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); @@ -8248,7 +13464,7 @@ static int __pyx_Generator_init(void) { } /* InitStrings */ - static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { + static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { while (t->p) { #if PY_MAJOR_VERSION < 3 if (t->is_unicode) { @@ -8273,6 +13489,8 @@ static int __pyx_Generator_init(void) { #endif if (!*t->p) return -1; + if (PyObject_Hash(*t->p) == -1) + PyErr_Clear(); ++t; } return 0; @@ -8281,50 +13499,57 @@ static int __pyx_Generator_init(void) { static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); } -static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { Py_ssize_t ignore; return __Pyx_PyObject_AsStringAndSize(o, &ignore); } -static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { -#if CYTHON_COMPILING_IN_CPYTHON && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) - if ( -#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - __Pyx_sys_getdefaultencoding_not_ascii && -#endif - PyUnicode_Check(o)) { -#if PY_VERSION_HEX < 0x03030000 - char* defenc_c; - PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); - if (!defenc) return NULL; - defenc_c = PyBytes_AS_STRING(defenc); +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT +#if !CYTHON_PEP393_ENABLED +static const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + char* defenc_c; + PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); + if (!defenc) return NULL; + defenc_c = PyBytes_AS_STRING(defenc); #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - { - char* end = defenc_c + PyBytes_GET_SIZE(defenc); - char* c; - for (c = defenc_c; c < end; c++) { - if ((unsigned char) (*c) >= 128) { - PyUnicode_AsASCIIString(o); - return NULL; - } + { + char* end = defenc_c + PyBytes_GET_SIZE(defenc); + char* c; + for (c = defenc_c; c < end; c++) { + if ((unsigned char) (*c) >= 128) { + PyUnicode_AsASCIIString(o); + return NULL; } } + } #endif - *length = PyBytes_GET_SIZE(defenc); - return defenc_c; + *length = PyBytes_GET_SIZE(defenc); + return defenc_c; +} #else - if (__Pyx_PyUnicode_READY(o) == -1) return NULL; +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - if (PyUnicode_IS_ASCII(o)) { - *length = PyUnicode_GET_LENGTH(o); - return PyUnicode_AsUTF8(o); - } else { - PyUnicode_AsASCIIString(o); - return NULL; - } + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } #else - return PyUnicode_AsUTF8AndSize(o, length); + return PyUnicode_AsUTF8AndSize(o, length); +#endif +} +#endif #endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT + if ( +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + __Pyx_sys_getdefaultencoding_not_ascii && #endif + PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); } else #endif #if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) @@ -8348,6 +13573,26 @@ static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { if (is_true | (x == Py_False) | (x == Py_None)) return is_true; else return PyObject_IsTrue(x); } +static PyObject* __Pyx_PyNumber_IntOrLongWrongResultType(PyObject* result, const char* type_name) { +#if PY_MAJOR_VERSION >= 3 + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type %.200s). " + "The ability to return an instance of a strict subclass of int " + "is deprecated, and may be removed in a future version of Python.", + Py_TYPE(result)->tp_name)) { + Py_DECREF(result); + return NULL; + } + return result; + } +#endif + PyErr_Format(PyExc_TypeError, + "__%.4s__ returned non-%.4s (type %.200s)", + type_name, type_name, Py_TYPE(result)->tp_name); + Py_DECREF(result); + return NULL; +} static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { #if CYTHON_USE_TYPE_SLOTS PyNumberMethods *m; @@ -8355,9 +13600,9 @@ static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { const char *name = NULL; PyObject *res = NULL; #if PY_MAJOR_VERSION < 3 - if (PyInt_Check(x) || PyLong_Check(x)) + if (likely(PyInt_Check(x) || PyLong_Check(x))) #else - if (PyLong_Check(x)) + if (likely(PyLong_Check(x))) #endif return __Pyx_NewRef(x); #if CYTHON_USE_TYPE_SLOTS @@ -8365,32 +13610,30 @@ static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { #if PY_MAJOR_VERSION < 3 if (m && m->nb_int) { name = "int"; - res = PyNumber_Int(x); + res = m->nb_int(x); } else if (m && m->nb_long) { name = "long"; - res = PyNumber_Long(x); + res = m->nb_long(x); } #else - if (m && m->nb_int) { + if (likely(m && m->nb_int)) { name = "int"; - res = PyNumber_Long(x); + res = m->nb_int(x); } #endif #else - res = PyNumber_Int(x); + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Int(x); + } #endif - if (res) { + if (likely(res)) { #if PY_MAJOR_VERSION < 3 - if (!PyInt_Check(res) && !PyLong_Check(res)) { + if (unlikely(!PyInt_Check(res) && !PyLong_Check(res))) { #else - if (!PyLong_Check(res)) { + if (unlikely(!PyLong_CheckExact(res))) { #endif - PyErr_Format(PyExc_TypeError, - "__%.4s__ returned non-%.4s (type %.200s)", - name, name, Py_TYPE(res)->tp_name); - Py_DECREF(res); - return NULL; + return __Pyx_PyNumber_IntOrLongWrongResultType(res, name); } } else if (!PyErr_Occurred()) { diff --git a/python/kenlm.pyx b/python/kenlm.pyx index e1c2453a1..085fa4fb1 100644 --- a/python/kenlm.pyx +++ b/python/kenlm.pyx @@ -8,6 +8,165 @@ cdef bytes as_str(data): return data.encode('utf8') raise TypeError('Cannot convert %s to string' % type(data)) +cdef class Output: + """ + Wrapper around lm::builder::Output. + """ + cdef _kenlm.Output* _c_output + + def __cinit__(self, file_base, keep_buffer, output_q): + self._c_output = new _kenlm.Output(file_base, keep_buffer, output_q) + + def Add(self, write_fd, verbose_header): + self._c_output.Add(new _kenlm.PrintHook(write_fd, verbose_header)) + + def __dealloc__(self): + del self._c_output + +cdef class PrintHook: + """ + Wrapper around lm::builder::Printhook + """ + cdef _kenlm.PrintHook* _c_printhook + + def __cinit__(self, write_fd, verbose_header): + self._c_printhook = new _kenlm.PrintHook(write_fd, verbose_header) + + def __dealloc__(self): + del self._c_printhook + +cdef Pipeline(_kenlm.PipelineConfig pipeline, __in, Output output): + _kenlm.Pipeline(pipeline, __in, output._c_output[0]) + + +def lmplz( + path_text_file, path_arpa_file, + order=3, + interpolate_unigrams=True, + skip_symbols=False, + temp_prefix=None, + memory="1G", + minimum_block="8K", + sort_block="64M", + block_count=2, + vocab_estimate=1000000, + vocab_pad=0, + verbose_header=False, + intermediate=None, + renumber=False, + collapse_values=False, + pruning=[], + limit_vocab_file='', + discount_fallback=[0.5, 1, 1.5]): + + cdef _kenlm.PipelineConfig pipeline + pipeline.order = order + pipeline.initial_probs.interpolate_unigrams = interpolate_unigrams + + if temp_prefix is None: + pipeline.sort.temp_prefix = _kenlm.DefaultTempDirectory() + else: + pipeline.sort.temp_prefix = temp_prefix + + if memory is None: + pipeline.sort.total_memory = _kenlm.GuessPhysicalMemory() + else: + pipeline.sort.total_memory = _kenlm.ParseSize(memory) + + pipeline.minimum_block = _kenlm.ParseSize(minimum_block) + pipeline.sort.buffer_size = _kenlm.ParseSize(sort_block) + pipeline.block_count = block_count + pipeline.vocab_estimate = vocab_estimate + pipeline.vocab_size_for_unk = vocab_pad + pipeline.renumber_vocabulary = renumber + pipeline.output_q = collapse_values + + if pipeline.vocab_size_for_unk and not pipeline.initial_probs.interpolate_unigrams: + raise RuntimeError('--vocab_pad requires --interpolate_unigrams be on') + + if skip_symbols: + pipeline.disallowed_symbol_action = _kenlm.COMPLAIN + else: + pipeline.disallowed_symbol_action = _kenlm.THROW_UP + + for i in range(4): + pipeline.discount.fallback.amount[i] = 0.0 + + if discount_fallback is None: + pipeline.discount.bad_action = _kenlm.THROW_UP + else: + if len(discount_fallback) > 3: + raise RuntimeError("Specify at most three fallback discounts: 1, 2, and 3+") + + if len(discount_fallback) == 0: + raise RuntimeError("Fallback discounting enabled, but no discount specified") + + pipeline.discount.fallback.amount[0] = 0.0 + + for i in range(3): + discount = discount_fallback[len(discount_fallback) - 1] + if i < len(discount_fallback): + discount = discount_fallback[i] + discount = float(discount) + + if (discount < 0.0 or discount > (i + 1)): + raise RuntimeError("The discount for count " + str(i+1) + " was parsed as " + discount + " which is not in the range [0, " + str(i+1) + "].") + + pipeline.discount.fallback.amount[i+1] = discount + + pipeline.discount.bad_action = _kenlm.COMPLAIN + + if len(pruning) > 0: + + pipeline.prune_thresholds.reserve(len(pruning)) + + for e in pruning: + pipeline.prune_thresholds.push_back(int(e)) + + if len(pruning) > order: + raise RuntimeError( + "You specified pruning thresholds for orders 1 through " + len(pruning) + + " but the model only has order " + order + ) + + pipeline.prune_thresholds.resize(order, pipeline.prune_thresholds.back()); + + else: + pipeline.prune_thresholds.resize(order, 0) + + if len(limit_vocab_file) == 0: + pipeline.prune_vocab = True + + else: + pipeline.prune_vocab = False + pipeline.prune_vocab_file = limit_vocab_file + + _kenlm.NormalizeTempPrefix(pipeline.sort.temp_prefix) + + pipeline.initial_probs.adder_in.total_memory = 32768; + pipeline.initial_probs.adder_in.block_count = 2; + pipeline.initial_probs.adder_out.total_memory = 32768; + pipeline.initial_probs.adder_out.block_count = 2; + pipeline.read_backoffs.total_memory = 32768; + pipeline.read_backoffs.block_count = 2; + + cdef _kenlm.scoped_fd _in + cdef _kenlm.scoped_fd _out + _in.reset(_kenlm.OpenReadOrThrow(path_text_file)) + _out.reset(_kenlm.CreateOrThrow(path_arpa_file)) + + if intermediate is None: + pipeline.renumber_vocabulary = False + output = Output(pipeline.sort.temp_prefix, False, False) + else: + pipeline.renumber_vocabulary = True + output = Output(intermediate, False, False) + + output.Add(_out.release(), verbose_header) + + Pipeline(pipeline, _in.release(), output) + + cdef class FullScoreReturn: """ Wrapper around FullScoreReturn. diff --git a/setup.py b/setup.py index 9d40c0195..1ec174b9f 100644 --- a/setup.py +++ b/setup.py @@ -10,10 +10,19 @@ def compile_test(header, library): return os.system(command) == 0 -FILES = glob.glob('util/*.cc') + glob.glob('lm/*.cc') + glob.glob('util/double-conversion/*.cc') +FILES = [] + +FILES += glob.glob('lm/*.cc') +FILES += glob.glob('lm/common/*.cc') +FILES += glob.glob('lm/builder/*.cc') + +FILES += glob.glob('util/*.cc') +FILES += glob.glob('util/stream/*.cc') +FILES += glob.glob('util/double-conversion/*.cc') + FILES = [fn for fn in FILES if not (fn.endswith('main.cc') or fn.endswith('test.cc'))] -LIBS = ['stdc++'] +LIBS = ['stdc++', 'boost_thread-mt', 'boost_program_options'] if platform.system() != 'Darwin': LIBS.append('rt') @@ -34,7 +43,7 @@ def compile_test(header, library): ext_modules = [ Extension(name='kenlm', - sources=FILES + ['python/kenlm.cpp'], + sources=FILES + ['python/kenlm.pyx'], language='C++', include_dirs=['.'], libraries=LIBS, @@ -45,4 +54,5 @@ def compile_test(header, library): name='kenlm', ext_modules=ext_modules, include_package_data=True, + version='0.0.1' )