Linboyan-trc · Apr 30, 2025 · Apr 30, 2025 · May 12, 2025 · May 12, 2025 · May 12, 2025
diff --git a/configs/caching/hunyuan_i2v_TaylorSeer.json → configs/caching/hunyuan_i2v_ada.json b/configs/caching/hunyuan_i2v_TaylorSeer.json → configs/caching/hunyuan_i2v_ada.json
@@ -8,5 +8,5 @@
         "mm_type": "W-int8-channel-sym-A-int8-channel-sym-dynamic-Vllm",
         "weight_auto_quant": true
     },
-    "feature_caching": "TaylorSeer"
+    "feature_caching": "Ada"
 }
diff --git a/configs/caching/hunyuan_i2v_custom.json b/configs/caching/hunyuan_i2v_custom.json
@@ -0,0 +1,12 @@
+{
+    "infer_steps": 20,
+    "target_video_length": 33,
+    "i2v_resolution": "720p",
+    "attention_type": "flash_attn3",
+    "seed": 0,
+    "mm_config": {
+        "mm_type": "W-int8-channel-sym-A-int8-channel-sym-dynamic-Vllm",
+        "weight_auto_quant": true
+    },
+    "feature_caching": "Custom"
+}
diff --git a/configs/caching/hunyuan_i2v_taylor.json b/configs/caching/hunyuan_i2v_taylor.json
@@ -0,0 +1,12 @@
+{
+    "infer_steps": 20,
+    "target_video_length": 33,
+    "i2v_resolution": "720p",
+    "attention_type": "flash_attn3",
+    "seed": 0,
+    "mm_config": {
+        "mm_type": "W-int8-channel-sym-A-int8-channel-sym-dynamic-Vllm",
+        "weight_auto_quant": true
+    },
+    "feature_caching": "Taylor"
+}
diff --git a/configs/caching/hunyuan_i2v_Tea.json → configs/caching/hunyuan_i2v_tea.json b/configs/caching/hunyuan_i2v_Tea.json → configs/caching/hunyuan_i2v_tea.json
diff --git a/configs/caching/hunyuan_t2v_ada.json b/configs/caching/hunyuan_t2v_ada.json
@@ -0,0 +1,13 @@
+{
+    "infer_steps": 20,
+    "target_video_length": 33,
+    "target_height": 720,
+    "target_width": 1280,
+    "attention_type": "flash_attn3",
+    "seed": 42,
+    "mm_config": {
+        "mm_type": "W-int8-channel-sym-A-int8-channel-sym-dynamic-Vllm",
+        "weight_auto_quant": true
+    },
+    "feature_caching": "Ada"
+}
diff --git a/configs/caching/hunyuan_t2v_custom.json b/configs/caching/hunyuan_t2v_custom.json
@@ -0,0 +1,13 @@
+{
+    "infer_steps": 20,
+    "target_video_length": 33,
+    "target_height": 720,
+    "target_width": 1280,
+    "attention_type": "flash_attn3",
+    "seed": 42,
+    "mm_config": {
+        "mm_type": "W-int8-channel-sym-A-int8-channel-sym-dynamic-Vllm",
+        "weight_auto_quant": true
+    },
+    "feature_caching": "Custom"
+}
diff --git a/configs/caching/hunyuan_t2v_TaylorSeer.json → configs/caching/hunyuan_t2v_taylor.json b/configs/caching/hunyuan_t2v_TaylorSeer.json → configs/caching/hunyuan_t2v_taylor.json
@@ -5,5 +5,5 @@
     "target_width": 1280,
     "attention_type": "flash_attn3",
     "seed": 42,
-    "feature_caching": "TaylorSeer"
+    "feature_caching": "Taylor"
 }
diff --git a/configs/caching/hunyuan_t2v_Tea.json → configs/caching/hunyuan_t2v_tea.json b/configs/caching/hunyuan_t2v_Tea.json → configs/caching/hunyuan_t2v_tea.json
diff --git a/configs/caching/wan_i2v_ada.json b/configs/caching/wan_i2v_ada.json
@@ -0,0 +1,17 @@
+{
+    "infer_steps": 40,
+    "target_video_length": 81,
+    "target_height": 480,
+    "target_width": 832,
+    "attention_type": "flash_attn3",
+    "seed": 42,
+    "sample_guide_scale": 5,
+    "sample_shift": 5,
+    "enable_cfg": true,
+    "cpu_offload": false,
+    "mm_config": {
+        "mm_type": "W-fp8-channel-sym-A-fp8-channel-sym-dynamic-Sgl",
+        "weight_auto_quant": true
+    },
+    "feature_caching": "Ada"
+}
diff --git a/configs/caching/wan_i2v_custom.json b/configs/caching/wan_i2v_custom.json
@@ -0,0 +1,17 @@
+{
+    "infer_steps": 40,
+    "target_video_length": 81,
+    "target_height": 480,
+    "target_width": 832,
+    "attention_type": "flash_attn3",
+    "seed": 42,
+    "sample_guide_scale": 5,
+    "sample_shift": 5,
+    "enable_cfg": true,
+    "cpu_offload": false,
+    "mm_config": {
+        "mm_type": "W-fp8-channel-sym-A-fp8-channel-sym-dynamic-Sgl",
+        "weight_auto_quant": true
+    },
+    "feature_caching": "Custom"
+}
diff --git a/configs/caching/wan_i2v_taylor.json b/configs/caching/wan_i2v_taylor.json
@@ -0,0 +1,17 @@
+{
+    "infer_steps": 40,
+    "target_video_length": 81,
+    "target_height": 480,
+    "target_width": 832,
+    "attention_type": "flash_attn3",
+    "seed": 42,
+    "sample_guide_scale": 5,
+    "sample_shift": 5,
+    "enable_cfg": true,
+    "cpu_offload": false,
+    "mm_config": {
+        "mm_type": "W-fp8-channel-sym-A-fp8-channel-sym-dynamic-Sgl",
+        "weight_auto_quant": true
+    },
+    "feature_caching": "Taylor"
+}
diff --git a/configs/caching/wan_i2v_Tea.json → configs/caching/wan_i2v_tea.json b/configs/caching/wan_i2v_Tea.json → configs/caching/wan_i2v_tea.json
diff --git a/configs/caching/wan_t2v_ada.json b/configs/caching/wan_t2v_ada.json
@@ -0,0 +1,18 @@
+{
+    "infer_steps": 50,
+    "target_video_length": 81,
+    "text_len": 512,
+    "target_height": 480,
+    "target_width": 832,
+    "attention_type": "flash_attn3",
+    "seed": 42,
+    "sample_guide_scale": 6,
+    "sample_shift": 8,
+    "enable_cfg": true,
+    "cpu_offload": false,
+    "mm_config": {
+        "mm_type": "W-fp8-channel-sym-A-fp8-channel-sym-dynamic-Sgl",
+        "weight_auto_quant": true
+    },
+    "feature_caching": "Ada"
+}
diff --git a/configs/caching/wan_t2v_custom.json b/configs/caching/wan_t2v_custom.json
@@ -0,0 +1,18 @@
+{
+    "infer_steps": 50,
+    "target_video_length": 81,
+    "text_len": 512,
+    "target_height": 480,
+    "target_width": 832,
+    "attention_type": "flash_attn3",
+    "seed": 42,
+    "sample_guide_scale": 6,
+    "sample_shift": 8,
+    "enable_cfg": true,
+    "cpu_offload": false,
+    "mm_config": {
+        "mm_type": "W-fp8-channel-sym-A-fp8-channel-sym-dynamic-Sgl",
+        "weight_auto_quant": true
+    },
+    "feature_caching": "Custom"
+}
diff --git a/configs/caching/wan_t2v_taylor.json b/configs/caching/wan_t2v_taylor.json
@@ -0,0 +1,18 @@
+{
+    "infer_steps": 50,
+    "target_video_length": 81,
+    "text_len": 512,
+    "target_height": 480,
+    "target_width": 832,
+    "attention_type": "flash_attn3",
+    "seed": 42,
+    "sample_guide_scale": 6,
+    "sample_shift": 8,
+    "enable_cfg": true,
+    "cpu_offload": false,
+    "mm_config": {
+        "mm_type": "W-fp8-channel-sym-A-fp8-channel-sym-dynamic-Sgl",
+        "weight_auto_quant": true
+    },
+    "feature_caching": "Taylor"
+}
diff --git a/configs/caching/wan_t2v_Tea.json → configs/caching/wan_t2v_tea.json b/configs/caching/wan_t2v_Tea.json → configs/caching/wan_t2v_tea.json
diff --git a/lightx2v/common/transformer_infer/transformer_infer.py b/lightx2v/common/transformer_infer/transformer_infer.py
@@ -0,0 +1,45 @@
+from abc import ABC, abstractmethod
+import torch
+import math
+
+
+class BaseTransformerInfer(ABC):
+    @abstractmethod
+    def infer(self):
+        pass
+
+    @abstractmethod
+    def infer_calculating(self):
+        pass
+
+    @abstractmethod
+    def infer_using_cache(self):
+        pass
+
+    @abstractmethod
+    def get_taylor_step_diff(self):
+        pass
+
+    def set_scheduler(self, scheduler):
+        self.scheduler = scheduler
+        self.scheduler.transformer_infer = self
+
+    # 1. when fully calcualted, stored in cache
+    def derivative_approximation(self, block_cache, module_name, out):
+        if module_name not in block_cache:
+            block_cache[module_name] = {0: out}
+        else:
+            step_diff = self.get_taylor_step_diff()
+
+            previous_out = block_cache[module_name][0]
+            block_cache[module_name][0] = out
+            block_cache[module_name][1] = (out - previous_out) / step_diff
+
+    def taylor_formula(self, tensor_dict):
+        x = self.get_taylor_step_diff()
+
+        output = 0
+        for i in range(len(tensor_dict)):
+            output += (1 / math.factorial(i)) * tensor_dict[i] * (x**i)
+
+        return output