Skip to content

Commit f837c2a

Browse files
chuangz0jhaotingc
chuangz0
authored and
jhaotingc
committed
chore:[BREAKING CHANGE] use cacheTransceiverConfig as knobs for disagg service (NVIDIA#5234)
Signed-off-by: Chuang Zhu <[email protected]>
1 parent 4a015e3 commit f837c2a

File tree

64 files changed

+646
-286
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

64 files changed

+646
-286
lines changed

‎benchmarks/cpp/disaggServerBenchmark.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -636,6 +636,8 @@ class DisaggExecutorServer
636636
: texec::DecodingMode::Auto(),
637637
benchmarkParams.executorLookaheadConfig, benchmarkParams.medusaChoices));
638638
executorConfig.setExtendedRuntimePerfKnobConfig(extendedRuntimePerfKnobConfig);
639+
executorConfig.setCacheTransceiverConfig(
640+
texec::CacheTransceiverConfig(texec::CacheTransceiverConfig::BackendType::DEFAULT));
639641
constexpr int maxIterationsForRequestStats = 1000;
640642
if (mEnableCollectKvCacheTransferTime)
641643
{

‎cpp/include/tensorrt_llm/batch_manager/cacheTransceiver.h‎

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -70,28 +70,20 @@ class BaseCacheTransceiver
7070
class CacheTransceiver : public BaseCacheTransceiver
7171
{
7272
public:
73-
enum class CommType : std::uint8_t
74-
{
75-
UNKNOWN = 0,
76-
MPI = 1,
77-
UCX = 2,
78-
NIXL = 3
79-
};
80-
81-
CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheManager, CommType commType,
73+
CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheManager,
8274
executor::kv_cache::CacheState::ModelConfig const& cacheStateModelCfg, runtime::WorldConfig const& worldConfig,
8375
nvinfer1::DataType dataType,
8476
executor::kv_cache::CacheState::AttentionType attentionType
8577
= executor::kv_cache::CacheState::AttentionType::kDEFAULT,
8678
std::optional<executor::CacheTransceiverConfig> cacheTransceiverConfig = std::nullopt);
8779

88-
CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheManager, CommType commType,
89-
std::vector<SizeType32> numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock,
90-
runtime::WorldConfig const& worldConfig, nvinfer1::DataType dataType,
80+
CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheManager, std::vector<SizeType32> numKvHeadsPerLayer,
81+
SizeType32 sizePerHead, SizeType32 tokensPerBlock, runtime::WorldConfig const& worldConfig,
82+
nvinfer1::DataType dataType,
9183
executor::kv_cache::CacheState::AttentionType attentionType
9284
= executor::kv_cache::CacheState::AttentionType::kDEFAULT,
9385
std::optional<executor::CacheTransceiverConfig> cacheTransceiverConfig = std::nullopt)
94-
: CacheTransceiver(cacheManager, commType,
86+
: CacheTransceiver(cacheManager,
9587
executor::kv_cache::CacheState::ModelConfig{numKvHeadsPerLayer, sizePerHead, tokensPerBlock}, worldConfig,
9688
dataType, attentionType, cacheTransceiverConfig)
9789
{
@@ -118,7 +110,6 @@ class CacheTransceiver : public BaseCacheTransceiver
118110

119111
void setContextState(LlmRequest* llmRequest);
120112

121-
CommType mCommType;
122113
std::unique_ptr<DataResponder> mDataResponder;
123114
std::unique_ptr<DataRequester> mDataRequester;
124115
std::vector<std::pair<LlmRequest*, std::future<void>>> mResponderFutures;

‎cpp/include/tensorrt_llm/executor/executor.h‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1430,18 +1430,29 @@ class LogitsPostProcessorConfig
14301430
class CacheTransceiverConfig
14311431
{
14321432
public:
1433-
explicit CacheTransceiverConfig(std::optional<size_t> maxNumTokens = std::nullopt);
1433+
enum class BackendType : std::uint8_t
1434+
{
1435+
DEFAULT = 0,
1436+
MPI = 1,
1437+
UCX = 2,
1438+
NIXL = 3
1439+
};
1440+
explicit CacheTransceiverConfig(
1441+
std::optional<BackendType> backendType = std::nullopt, std::optional<size_t> maxNumTokens = std::nullopt);
14341442

14351443
bool operator==(CacheTransceiverConfig const& other) const;
1444+
void setBackendType(std::optional<BackendType> backendType);
1445+
void setMaxTokensInBuffer(std::optional<size_t> maxTokensInBuffer);
14361446

1437-
[[nodiscard]] std::optional<size_t> getMaxNumTokens() const;
1438-
void setMaxNumTokens(size_t maxNumTokens);
1447+
[[nodiscard]] std::optional<size_t> getMaxTokensInBuffer() const;
1448+
[[nodiscard]] std::optional<BackendType> getBackendType() const;
14391449

14401450
private:
1451+
std::optional<BackendType> mBackendType;
14411452
/// @brief The maximum number of tokens that the CacheTransceiver's pre-allocated buffer can hold. If the number of
14421453
/// kvCache tokens to be transferred for a single request is greater than this value, the performance of the cache
14431454
/// transfer may be degraded.
1444-
std::optional<size_t> mMaxNumTokens;
1455+
std::optional<size_t> mMaxTokensInBuffer;
14451456
};
14461457

14471458
/// @brief Configuration class for the model executor

‎cpp/tensorrt_llm/batch_manager/cacheTransBuffer.cpp‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,7 @@ CacheTransBufferManager::CacheTransBufferManager(
210210
{
211211
auto poolIdx = mCacheManager->getBlockManager().getLayerPoolIdx(layerId);
212212
auto windowSize = static_cast<size_t>(mCacheManager->getBlockManager().getPoolWindowSize(poolIdx));
213-
auto validTokenNum = windowSize < maxNumTokens.value() ? windowSize : maxNumTokens.value();
213+
auto validTokenNum = (windowSize < maxNumTokens.value() ? windowSize : maxNumTokens.value());
214214
bufferSizeFromMaxNumToken += validTokenNum * kvCacheByteSizePerTokenPerLayer;
215215
}
216216
}
@@ -230,26 +230,37 @@ CacheTransBufferManager::CacheTransBufferManager(
230230
TLLM_LOG_INFO(
231231
"CacheTransBufferManager: mMaxNumTokens:%ld, mRecvBufferCount:%ld, "
232232
"mSendBufferCount:%ld,mTransferBufferSize:%ld, mPreAllocBufferSize:%ld,mOnlyUseDynamicBuffer:%d "
233-
"mUseFabricMemory:%d",
233+
"mUseFabricMemory:%d mDataType:%d",
234234
maxNumTokens.has_value() ? maxNumTokens.value() : 0, mRecvBufferCount, mSendBufferCount, mTransferBufferSize,
235-
mPreAllocBufferSize, mOnlyUseDynamicBuffer, mUseFabricMemory);
236-
bool to_allocate = common::getEnvUseMPIKvCache() || common::getEnvUseUCXKvCache() || common::getEnvUseNixlKvCache();
235+
mPreAllocBufferSize, mOnlyUseDynamicBuffer, mUseFabricMemory, mDataType);
237236

238-
TLLM_CHECK_WITH_INFO(to_allocate, "CacheTransBufferManager: to_allocate is false");
239237
allocateBuffer();
240238
}
241239

242-
size_t CacheTransBufferManager::preAllocBufferSize(std::optional<size_t> maxNumTokens)
240+
size_t CacheTransBufferManager::preAllocBufferSize(
241+
std::map<SizeType32, SizeType32> const& cacheSizeBytesPerTokenPerWindow,
242+
std::optional<executor::CacheTransceiverConfig> const& cacheTransceiverConfig)
243243
{
244-
bool to_allocate = common::getEnvUseMPIKvCache() || common::getEnvUseUCXKvCache() || common::getEnvUseNixlKvCache();
245-
if (!to_allocate)
244+
if (!cacheTransceiverConfig.has_value())
246245
{
247246
return 0;
248247
}
248+
if (!cacheTransceiverConfig->getBackendType().has_value())
249+
{
250+
return 0;
251+
}
252+
auto maxNumTokens = cacheTransceiverConfig->getMaxTokensInBuffer();
249253
size_t TransferBufferSize = common::getEnvMemSizeForKVCacheTransferBuffer();
250254
if (maxNumTokens.has_value())
251255
{
252-
TransferBufferSize = maxNumTokens.value();
256+
TransferBufferSize = 0;
257+
for (auto const& [windowSize, cacheSizeBytesPerToken] : cacheSizeBytesPerTokenPerWindow)
258+
{
259+
auto validTokenNum
260+
= (static_cast<size_t>(windowSize) < maxNumTokens.value() ? static_cast<size_t>(windowSize)
261+
: maxNumTokens.value());
262+
TransferBufferSize += validTokenNum * cacheSizeBytesPerToken;
263+
}
253264
}
254265
bool useFabricMemory = FabricMemory::supportFbaricMemory()
255266
&& (!(common::getEnvKVCacheTransferUseSyncBuffer() || common::getEnvKVCacheTransferUseAsyncBuffer()));
@@ -329,6 +340,14 @@ std::tuple<std::vector<runtime::ITensor::SharedPtr>, size_t, bool> CacheTransBuf
329340
size_t bufferCoverTargetNum = std::min(
330341
static_cast<size_t>(targetNum), mTransferBufferSize / (targetBufferEleSize * common::getDTypeSize(mDataType)));
331342
TLLM_LOG_DEBUG("getOrAllocateBuffers bufferCoverTargetNum:%d", bufferCoverTargetNum);
343+
if (bufferCoverTargetNum < static_cast<size_t>(targetNum))
344+
{
345+
TLLM_LOG_WARNING(
346+
"CacheTransceiver getOrAllocateBuffers: bufferCoverTargetNum:%d < targetNum:%d, may use dynamic buffer, "
347+
"it's better to increase MaxTokensInBuffer in cacheTransceiverConfig, otherwise, the performance may "
348+
"be degraded",
349+
bufferCoverTargetNum, targetNum);
350+
}
332351
if (bufferId.has_value())
333352
{
334353
TLLM_CHECK(static_cast<size_t>(bufferId.value()) < concurrenceResource.mBuffers.size());

‎cpp/tensorrt_llm/batch_manager/cacheTransBuffer.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#pragma once
1919

2020
#include "tensorrt_llm/batch_manager/kvCacheManager.h"
21+
#include "tensorrt_llm/executor/executor.h"
2122
#include "tensorrt_llm/runtime/bufferManager.h"
2223
#include "tensorrt_llm/runtime/iTensor.h"
2324
#include <atomic>
@@ -59,7 +60,8 @@ class CacheTransBufferManager
5960
CacheTransBufferManager(
6061
KVCacheManager::BaseKVCacheManager* cacheManager, std::optional<size_t> maxNumTokens = std::nullopt);
6162

62-
static size_t preAllocBufferSize(std::optional<size_t> maxNumTokens = std::nullopt);
63+
static size_t preAllocBufferSize(std::map<SizeType32, SizeType32> const& cacheSizeBytesPerTokenPerWindow,
64+
std::optional<executor::CacheTransceiverConfig> const& cacheTransceiverConfig = std::nullopt);
6365

6466
std::optional<int> assignBufferIndexForSend();
6567
void freeBufferIndexForSend(std::optional<int> bufferId);

‎cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp‎

Lines changed: 79 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -62,41 +62,49 @@ std::unique_ptr<BaseCacheTransceiver> CacheTransceiverFactory::createCacheTransc
6262
runtime::WorldConfig const& worldConfig, executor::kv_cache::CacheState::AttentionType attentionType,
6363
std::optional<executor::CacheTransceiverConfig> cacheTransceiverConfig)
6464
{
65-
66-
std::optional<CacheTransceiver::CommType> commType;
67-
if (common::getEnvUseUCXKvCache())
68-
{
69-
commType = CacheTransceiver::CommType::UCX;
70-
TLLM_LOG_INFO("Enable UCX KV cache transport.");
71-
}
72-
else if (common::getEnvUseNixlKvCache())
65+
if (!cacheTransceiverConfig.has_value() || !cacheTransceiverConfig.value().getBackendType().has_value())
7366
{
74-
commType = CacheTransceiver::CommType::NIXL;
75-
TLLM_LOG_INFO("Enable NIXL KV cache transport.");
67+
TLLM_LOG_INFO("CacheTransceiver is disabled.");
68+
return nullptr;
7669
}
77-
else if (common::getEnvUseMPIKvCache())
70+
auto backendType = cacheTransceiverConfig.value().getBackendType();
71+
if (backendType.value() == executor::CacheTransceiverConfig::BackendType::DEFAULT)
7872
{
79-
commType = CacheTransceiver::CommType::MPI;
80-
TLLM_LOG_INFO("Enable MPI KV cache transport.");
73+
if (common::getEnvUseUCXKvCache())
74+
{
75+
backendType = executor::CacheTransceiverConfig::BackendType::UCX;
76+
TLLM_LOG_INFO("Enable UCX KV cache transport.");
77+
}
78+
else if (common::getEnvUseNixlKvCache())
79+
{
80+
backendType = executor::CacheTransceiverConfig::BackendType::NIXL;
81+
TLLM_LOG_INFO("Enable NIXL KV cache transport.");
82+
}
83+
else if (common::getEnvUseMPIKvCache())
84+
{
85+
backendType = executor::CacheTransceiverConfig::BackendType::MPI;
86+
TLLM_LOG_INFO("Enable MPI KV cache transport.");
87+
TLLM_LOG_WARNING("MPI KV cache transport is deprecated, please use UCX or NIXL instead.");
88+
}
89+
else
90+
{
91+
backendType = executor::CacheTransceiverConfig::BackendType::UCX;
92+
}
8193
}
94+
cacheTransceiverConfig.value().setBackendType(backendType);
8295

83-
if (commType)
84-
{
85-
executor::kv_cache::CacheState::ModelConfig cacheStateCfg{
86-
modelConfig.getNumKvHeadsPerLayer(), modelConfig.getSizePerHead(), modelConfig.getTokensPerBlock()};
96+
executor::kv_cache::CacheState::ModelConfig cacheStateCfg{
97+
modelConfig.getNumKvHeadsPerLayer(), modelConfig.getSizePerHead(), modelConfig.getTokensPerBlock()};
8798

88-
return std::make_unique<CacheTransceiver>(cacheManager, commType.value(), cacheStateCfg, worldConfig,
89-
modelConfig.getKvDataType(), attentionType, cacheTransceiverConfig);
90-
}
91-
return nullptr;
99+
return std::make_unique<CacheTransceiver>(
100+
cacheManager, cacheStateCfg, worldConfig, modelConfig.getKvDataType(), attentionType, cacheTransceiverConfig);
92101
}
93102

94-
CacheTransceiver::CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheManager, CommType commType,
103+
CacheTransceiver::CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheManager,
95104
executor::kv_cache::CacheState::ModelConfig const& cacheStateModelCfg, runtime::WorldConfig const& worldConfig,
96105
nvinfer1::DataType dataType, executor::kv_cache::CacheState::AttentionType attentionType,
97106
std::optional<executor::CacheTransceiverConfig> cacheTransceiverConfig)
98-
: mCommType{commType}
99-
, mMpiGroupComm(std::addressof(tensorrt_llm::mpi::MpiComm::session()))
107+
: mMpiGroupComm(std::addressof(tensorrt_llm::mpi::MpiComm::session()))
100108
, mCacheTransceiverConfig{cacheTransceiverConfig}
101109
{
102110
using tensorrt_llm::batch_manager::kv_cache_manager::CacheFormatter;
@@ -138,59 +146,59 @@ CacheTransceiver::CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheMa
138146
}
139147
}
140148
bool isMLA = attentionType == executor::kv_cache::CacheState::AttentionType::kMLA;
141-
if (mCommType == CommType::MPI || mCommType == CommType::UCX || mCommType == CommType::NIXL)
142-
{
143-
std::optional<size_t> maxNumTokens = std::nullopt;
144-
if (mCacheTransceiverConfig.has_value())
145-
{
146-
maxNumTokens = mCacheTransceiverConfig.value().getMaxNumTokens();
147-
}
148-
mCacheTransBufferManager
149-
= std::make_unique<kv_cache_manager::CacheTransBufferManager>(cacheManager, maxNumTokens);
150-
if (mCommType == CommType::UCX)
151-
{
152-
std::lock_guard<std::mutex> lock(mDllMutex);
153-
mWrapperLibHandle = dllOpen(UCX_WRAPPER_LIB_NAME);
154-
TLLM_CHECK_WITH_INFO(mWrapperLibHandle != nullptr, "UCX wrapper library is not open correctly.");
155-
auto load_sym = [](void* handle, char const* name)
156-
{
157-
void* ret = dllGetSym(handle, name);
158-
TLLM_CHECK_WITH_INFO(ret != nullptr,
159-
"Unable to load UCX wrapper library symbol, possible cause is that TensorRT-LLM library is not "
160-
"built with UCX support, please rebuild in UCX-enabled environment.");
161-
return ret;
162-
};
163-
std::unique_ptr<tensorrt_llm::executor::kv_cache::ConnectionManager> (*makeUcxConnectionManager)();
164-
*(void**) (&makeUcxConnectionManager) = load_sym(mWrapperLibHandle, "makeUcxConnectionManager");
165-
mManager = makeUcxConnectionManager();
166-
TLLM_LOG_INFO("UCX Connection Manager created");
167-
}
168-
else if (mCommType == CommType::NIXL)
169-
{
170-
mManager = std::make_unique<tensorrt_llm::executor::kv_cache::AgentConnectionManager>(
171-
mCacheTransBufferManager.get());
172-
TLLM_LOG_INFO("NIXL Connection Manager created");
173-
}
174-
else
175-
{
176-
mMpiWorldComm = std::addressof(tensorrt_llm::mpi::MpiComm::world());
177-
mManager = std::make_unique<executor::kv_cache::MpiConnectionManager>(mMpiWorldComm);
178-
TLLM_LOG_INFO("MPI Connection Manager created");
179-
}
149+
TLLM_CHECK_WITH_INFO(mCacheTransceiverConfig.has_value(), "CacheTransceiverConfig is not set.");
150+
auto backendType = mCacheTransceiverConfig.value().getBackendType();
151+
TLLM_CHECK_WITH_INFO(
152+
backendType.has_value() && (backendType.value() != executor::CacheTransceiverConfig::BackendType::DEFAULT),
153+
" CacheTransceiverConfig::BackendType is not set.");
180154

181-
using tensorrt_llm::batch_manager::kv_cache_manager::MLACacheFormatter;
182-
auto makeFormatter = [cacheManager, isMLA, this]()
183-
{ return createCacheFormatter(cacheManager, mCacheTransBufferManager.get(), isMLA); };
155+
std::optional<size_t> maxNumTokens = mCacheTransceiverConfig.value().getMaxTokensInBuffer();
184156

185-
mDataResponder = std::make_unique<DataResponder>(
186-
std::make_unique<DataSenderImpl>(mManager.get(), *mCacheState, worldConfig.getRank(), makeFormatter()));
187-
mDataRequester = std::make_unique<DataRequester>(
188-
std::make_unique<DataReceiverImpl>(mManager.get(), *mCacheState, worldConfig.getRank(), makeFormatter()));
157+
mCacheTransBufferManager = std::make_unique<kv_cache_manager::CacheTransBufferManager>(cacheManager, maxNumTokens);
158+
if (backendType.value() == executor::CacheTransceiverConfig::BackendType::UCX)
159+
{
160+
std::lock_guard<std::mutex> lock(mDllMutex);
161+
mWrapperLibHandle = dllOpen(UCX_WRAPPER_LIB_NAME);
162+
TLLM_CHECK_WITH_INFO(mWrapperLibHandle != nullptr, "UCX wrapper library is not open correctly.");
163+
auto load_sym = [](void* handle, char const* name)
164+
{
165+
void* ret = dllGetSym(handle, name);
166+
TLLM_CHECK_WITH_INFO(ret != nullptr,
167+
"Unable to load UCX wrapper library symbol, possible cause is that TensorRT-LLM library is not "
168+
"built with UCX support, please rebuild in UCX-enabled environment.");
169+
return ret;
170+
};
171+
std::unique_ptr<tensorrt_llm::executor::kv_cache::ConnectionManager> (*makeUcxConnectionManager)();
172+
*(void**) (&makeUcxConnectionManager) = load_sym(mWrapperLibHandle, "makeUcxConnectionManager");
173+
mManager = makeUcxConnectionManager();
174+
TLLM_LOG_INFO("UCX Connection Manager created");
175+
}
176+
else if (backendType.value() == executor::CacheTransceiverConfig::BackendType::NIXL)
177+
{
178+
mManager = std::make_unique<tensorrt_llm::executor::kv_cache::AgentConnectionManager>(
179+
mCacheTransBufferManager.get());
180+
TLLM_LOG_INFO("NIXL Connection Manager created");
181+
}
182+
else if (backendType.value() == executor::CacheTransceiverConfig::BackendType::MPI)
183+
{
184+
mMpiWorldComm = std::addressof(tensorrt_llm::mpi::MpiComm::world());
185+
mManager = std::make_unique<executor::kv_cache::MpiConnectionManager>(mMpiWorldComm);
186+
TLLM_LOG_INFO("MPI Connection Manager created");
189187
}
190188
else
191189
{
192-
TLLM_THROW("Unsupported communication type.");
190+
TLLM_THROW("Unsupported cache transceiver backend type ");
193191
}
192+
193+
using tensorrt_llm::batch_manager::kv_cache_manager::MLACacheFormatter;
194+
auto makeFormatter = [cacheManager, isMLA, this]()
195+
{ return createCacheFormatter(cacheManager, mCacheTransBufferManager.get(), isMLA); };
196+
197+
mDataResponder = std::make_unique<DataResponder>(
198+
std::make_unique<DataSenderImpl>(mManager.get(), *mCacheState, worldConfig.getRank(), makeFormatter()));
199+
mDataRequester = std::make_unique<DataRequester>(
200+
std::make_unique<DataReceiverImpl>(mManager.get(), *mCacheState, worldConfig.getRank(), makeFormatter()));
201+
194202
initializeCommState();
195203
}
196204

‎cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2235,13 +2235,8 @@ BlocksPerWindow BaseKVCacheManager::calculateMaxNumBlocks(executor::KvCacheConfi
22352235
cacheSizeBytesPerTokenPerWindow[windowSize] = cacheSizeBytesPerToken;
22362236
}
22372237

2238-
auto const extraCostMemoryBytes = extraCostMemory
2239-
* std::accumulate(cacheSizeBytesPerTokenPerWindow.cbegin(), cacheSizeBytesPerTokenPerWindow.cend(),
2240-
SizeType32{0}, [](SizeType32 acc, auto const cost) { return acc + cost.second; });
2241-
2242-
TLLM_LOG_DEBUG(
2243-
"extraCostMemoryBytes [all windows] [Gib]: %0.2f", extraCostMemoryBytes / static_cast<double>(1 << 30));
2244-
2238+
TLLM_LOG_DEBUG("extraCostMemory [Gib]: %0.2f", extraCostMemory / static_cast<double>(1 << 30));
2239+
allottedPrimaryMemBytes = allottedPrimaryMemBytes - extraCostMemory;
22452240
auto const tokensPerBlock = modelConfig.getTokensPerBlock();
22462241
auto const calculatePrimaryBlocks
22472242
= [&](SizeType32 windowSize, float windowSizeShare, SizeType32 cacheSizeBytesPerToken)

0 commit comments

Comments
 (0)