fix

Author: hiworldwzj

docs/CN/source/tutorial/api_server_args_zh.rst

@@ -274,10 +274,6 @@ attention类型选择参数
 
     多模态资源的缓存服务器容量，默认为 ``200``
 
-.. option:: --cache_reserved_ratio
-
-    缓存服务器清理后的保留容量比例，默认为 ``0.5``
-
 .. option:: --visual_infer_batch_size
 
     每次推理批次中处理的图像数量，默认为 ``1``

docs/EN/source/tutorial/api_server_args_zh.rst

@@ -273,10 +273,6 @@ Multimodal Parameters
 
     Cache server capacity for multimodal resources, default is ``200``
 
-.. option:: --cache_reserved_ratio
-
-    Reserved capacity ratio after cache server cleanup, default is ``0.5``
-
 .. option:: --visual_infer_batch_size
 
     Number of images processed in each inference batch, default is ``1``

lightllm/server/api_cli.py

@@ -288,9 +288,6 @@ def make_argument_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "--cache_capacity", type=int, default=200, help="cache server capacity for multimodal resources"
     )
-    parser.add_argument(
-        "--cache_reserved_ratio", type=float, default=0.5, help="cache server reserved capacity ratio after clear"
-    )
     parser.add_argument(
         "--data_type",
         type=str,

lightllm/server/core/objs/start_args_type.py

@@ -57,7 +57,6 @@ class StartArgs:
     enable_decode_microbatch_overlap: bool = field(default=False)
     enable_prefill_microbatch_overlap: bool = field(default=False)
     cache_capacity: int = field(default=200)
-    cache_reserved_ratio: float = field(default=0.5)
     data_type: Optional[str] = field(
         default=None, metadata={"choices": ["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"]}
     )

lightllm/server/embed_cache/impl/naive_memory_cache.py

@@ -32,8 +32,6 @@ def __init__(self, args) -> None:
         self._records = dict()
         self._md5_to_record = dict()
         self.capacity = max(1, args.cache_capacity)
-        self.reserved = max(0, int(self.capacity * args.cache_reserved_ratio))
-        self.reserved = min(self.reserved, self.capacity - 1)
         self.occupied = 0
         self.expired_secs = 60 * 60
         self.lock = threading.Lock()
@@ -69,9 +67,9 @@ def _check_and_set_new_id_range(self, alloced_token_num):
                         time.sleep(3)
         return
 
-    def _clear(self):
+    def _clear(self, free_max_count: int):
         deleted = 0
-        max_delete = max(1, self.occupied - self.reserved)
+        max_delete = free_max_count
         items = sorted(self._records.items(), key=lambda x: x[1].visittime)
         t = time.time()
         for id, record in items:
@@ -91,10 +89,10 @@ def alloc(self, md5sum_list: list[str], token_num_list: list[int]) -> Optional[l
         now = time.time()
         with self.lock:
             new_md5s = [m for m in md5sum_list if m not in self._md5_to_record]
-            new_needed = len(new_md5s)
+            new_needed = len(set(new_md5s))
 
             if self.occupied + new_needed > self.capacity:
-                self._clear()
+                self._clear(free_max_count=new_needed - (self.capacity - self.occupied))
             if self.occupied + new_needed > self.capacity:
                 return None
 
@@ -127,19 +125,19 @@ def alloc(self, md5sum_list: list[str], token_num_list: list[int]) -> Optional[l
 
     def release(self, ids: list[int]) -> None:
         with self.lock:
-            for id in ids:
-                self._records[id].ref -= 1
+            for id_ in ids:
+                self._records[id_].ref -= 1
 
     def set_items_data(self, ids: list[int]) -> None:
-        for id in ids:
-            self._records[id].data = True
+        for id_ in ids:
+            self._records[id_].data = True
 
     def get_items_data(self, ids: list[int]) -> list[Optional[bool]]:
-        return [self._records.get(i).data if i in self._records else False for i in ids]
+        return [self._records.get(id_).data if id_ in self._records else False for id_ in ids]
 
     def set_items_embed(self, ids: list[int]) -> None:
-        for id in ids:
-            self._records[id].embed = True
+        for id_ in ids:
+            self._records[id_].embed = True
 
     def get_items_embed(self, ids: list[int]) -> list[Optional[bool]]:
-        return [self._records.get(i).embed if i in self._records else False for i in ids]
+        return [self._records.get(id_).embed if id_ in self._records else False for id_ in ids]

lightllm/server/httpserver/manager.py

@@ -115,13 +115,12 @@ def __init__(
         return
 
     async def _alloc_resource(self, items, md5sums, token_nums, datas):
-        wait_time = 1
+
         while True:
             records = obtain(self.cache_client.root.alloc(md5sums, token_nums))
 
             if records is None:
-                await asyncio.sleep(wait_time)
-                wait_time = min(wait_time + 0.5, 2)
+                await asyncio.sleep(0.1)
                 continue
 
             uid_list = []
@@ -155,19 +154,19 @@ async def _alloc_multimodal_resources(self, multimodal_params: MultimodalParams,
                     self.tokenizer.init_imageitem_extral_params(img, multimodal_params, sampling_params)
                     data = img.read()
                     # must after init_imageitem_extral_params
-                    tokens_num = self.tokenizer.get_image_token_length(img)
+                    token_num = self.tokenizer.get_image_token_length(img)
                     md5sum = hashlib.md5(data).hexdigest() + "_" + str(hash(frozendict(img.extra_params)))
                     md5sums.append(md5sum)
-                    tokens_nums.append(tokens_num)
+                    tokens_nums.append(token_num)
                     datas.append(data)
                     items.append(img)
                 for audio in multimodal_params.audios:
                     self.tokenizer.init_audioitem_extral_params(audio, multimodal_params, sampling_params)
                     data = audio.read()
-                    tokens_num = self.tokenizer.get_audio_token_length(audio)
+                    token_num = self.tokenizer.get_audio_token_length(audio)
                     md5sum = hashlib.md5(data).hexdigest() + "_" + str(hash(frozendict(audio.extra_params)))
                     md5sums.append(md5sum)
-                    tokens_nums.append(tokens_num)
+                    tokens_nums.append(token_num)
                     datas.append(data)
                     items.append(audio)