perf: reduce memory usage

Author: winstxnhdw

server/features/translator/nllb.py

@@ -139,7 +139,7 @@ def translate_generator(self, text: str, source_language: Language, target_langu
         results = self.translator.generate_tokens(
             (source_language, *self.tokeniser.encode(text).tokens),
             target_prefix,
-            max_decoding_length=4096,
+            max_decoding_length=1024,
             sampling_temperature=0,
             no_repeat_ngram_size=3,
             suppress_sequences=(target_prefix,),
@@ -169,7 +169,7 @@ def translate(self, text: str, source_language: Language, target_language: Langu
         translated_text (str)
             the translated text
         """
-        return self.tokeniser.decode(list(self.translate_generator(text, source_language, target_language)))
+        return self.tokeniser.decode(tuple(self.translate_generator(text, source_language, target_language)))
 
     def translate_stream(self, text: str, source_language: Language, target_language: Language) -> Iterator[str]:
         """

server/schemas/health.py

@@ -3,7 +3,7 @@
 from server.config import Config
 
 
-class Health(Struct, kw_only=True, gc=False):
+class Health(Struct, kw_only=True, frozen=True, gc=False):
     """
     Summary
     -------

server/schemas/v1/language.py

@@ -5,7 +5,7 @@
 from server.typedefs import Confidence, Language
 
 
-class LanguageResult(Struct, kw_only=True):
+class LanguageResult(Struct, kw_only=True, frozen=True, gc=False):
     """
     Summary
     -------

server/schemas/v1/tokens.py

@@ -3,7 +3,7 @@
 from msgspec import Meta, Struct
 
 
-class Tokens(Struct, kw_only=True):
+class Tokens(Struct, kw_only=True, frozen=True, gc=False):
     """
     Summary
     -------

server/schemas/v1/translated.py

@@ -3,7 +3,7 @@
 from msgspec import Meta, Struct
 
 
-class Translated(Struct, kw_only=True):
+class Translated(Struct, kw_only=True, frozen=True, gc=False):
     """
     Summary
     -------

server/schemas/v1/translation.py

@@ -5,7 +5,7 @@
 from server.typedefs import Language
 
 
-class Translation(Struct, kw_only=True):
+class Translation(Struct, kw_only=True, frozen=True, gc=False):
     """
     Summary
     -------

server/typings/ctranslate2/__init__.pyi

@@ -63,8 +63,8 @@ class Translator:
     @overload
     def translate_batch(
         self,
-        source: Iterable[list[str]],
-        target_prefix: Iterable[list[str] | None] | None = None,
+        source: Iterable[Sequence[str]],
+        target_prefix: Iterable[Sequence[str] | None] | None = None,
         *,
         max_batch_size: int = 0,
         batch_type: BatchTypes = "examples",
@@ -77,7 +77,7 @@ class Translator:
         repetition_penalty: float = 1,
         no_repeat_ngram_size: int = 0,
         disable_unks: bool = False,
-        supress_sequences: list[list[str]] | None = None,
+        suppress_sequences: Sequence[Sequence[str]] | None = None,
         end_token: str | list[str] | list[int] | None = None,
         return_end_token: bool = False,
         prefix_bias_beta: float = 0,
@@ -98,8 +98,8 @@ class Translator:
     @overload
     def translate_batch(
         self,
-        source: Iterable[list[str]],
-        target_prefix: Iterable[list[str] | None] | None = None,
+        source: Iterable[Sequence[str]],
+        target_prefix: Iterable[Sequence[str] | None] | None = None,
         *,
         max_batch_size: int = 0,
         batch_type: BatchTypes = "examples",
@@ -112,7 +112,7 @@ class Translator:
         repetition_penalty: float = 1,
         no_repeat_ngram_size: int = 0,
         disable_unks: bool = False,
-        supress_sequences: list[list[str]] | None = None,
+        suppress_sequences: Sequence[Sequence[str]] | None = None,
         end_token: str | list[str] | list[int] | None = None,
         return_end_token: bool = False,
         prefix_bias_beta: float = 0,
@@ -132,8 +132,8 @@ class Translator:
     ) -> list[AsyncTranslationResult]: ...
     def translate_iterable(
         self,
-        source: Iterable[list[str]],
-        target_prefix: Iterable[list[str]] | None = None,
+        source: Iterable[Sequence[str]],
+        target_prefix: Iterable[Sequence[str]] | None = None,
         max_batch_size: int = 32,
         batch_type: BatchTypes = "examples",
         *,
@@ -145,7 +145,7 @@ class Translator:
         repetition_penalty: float = 1,
         no_repeat_ngram_size: int = 0,
         disable_unks: bool = False,
-        supress_sequences: list[list[str]] | None = None,
+        suppress_sequences: list[list[str]] | None = None,
         end_token: str | list[str] | list[int] | None = None,
         return_end_token: bool = False,
         prefix_bias_beta: float = 0,

server/typings/tokenizers/__init__.pyi

@@ -7,6 +7,7 @@ class Encoding(Sized):
 class Tokenizer:
     @classmethod
     def from_file(cls, path: str) -> Self: ...
+    def token_to_id(self, token: str) -> int: ...
     def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str: ...
     def decode_batch(self, ids: Sequence[Sequence[int]], skip_special_tokens: bool = True) -> list[str]: ...
     def encode(