Add parameters max_context_length_per_message and max_total_context_length #244
Description
I would like to add a parameter to the Basic Agent: max_context_length_per_message and max_total_context_length.
I already made a truncation function that achieves this. But the solution is insufficient because it can only be called after the agent is finished with one full invocation (going through its max_iterations).
Better would be to have this directly in the prompt loop of the agent. Instead of having the user call truncate_agent as seen in the code block directly below, this could happen behind the scenes.
@fast.agent(
PRODUCT_GETTER,
instruction=get_general_instructions("getter"),
servers=[
"mcp_browser_use",
"mcp_slack",
"mcp_html_dom",
"mcp_helper_library",
],
# use_history=True,
use_history=False,
request_params=get_request_params(max_iterations=15),
)
async def main(shop):
async with fast.run() as agent:
agent = truncate_agent_history(agent)
response = await agent(f"""
{get_general_instructions('getter')}
{get_shop_prompt(shop)}""")
await asyncio.sleep(2)
if "DONE" in response:
logging.debug("Goal reached.")
return "DONE"
elif "ERROR_ENCOUNTERED" in response:
logging.debug("Error encountered. Please check the agent's response.")
return "ERROR_ENCOUNTERED"
else:
await asyncio.sleep(2)
agent = truncate_agent_history(agent)
response = await agent(f"""
{get_general_instructions('getter')}
{get_shop_prompt(shop)}""")
This issue related, but different in its implementation to #128 as the current issue does not limit the request size, but directly truncates the history.
def truncate_agent_history(agent):
"""
Truncates the agent's message history to ensure it does not exceed the maximum context length.
Cuts individual message content pieces to a maximum length and removes the oldest messages if the total context length exceeds the maximum allowed (MAXIMUM_LENGTH_PER_MESSAGE_CONTENT_PIECE).
Cuts the oldest messages until the total context length is below the maximum allowed (MAXIMUM_TOTAL_CONTEXT_LENGTH)
"""
if not agent._agents[DEFAULT_AGENT]._llm._message_history:
logging.warning(f"""The message history is empty.""")
total_original_length = 0
total_truncated_length = 0
for message_index, message in enumerate(agent._agents[DEFAULT_AGENT]._llm._message_history):
for content_piece_index, content_piece in enumerate(message.content):
original_length = len(content_piece.text)
total_original_length += original_length
truncated_content = content_piece.text[:MAXIMUM_LENGTH_PER_MESSAGE_CONTENT_PIECE]
agent._agents[DEFAULT_AGENT]._llm._message_history[message_index].content[content_piece_index] = TextContent(type="text", text=truncated_content)
total_truncated_length += len(truncated_content)
logging.warning(f"""
content_piece (type: {type(truncated_content)}):
original_length: {original_length}
truncated_length: {len(truncated_content)}
{truncated_content}
""")
def remove_oldest_messages_until_short_enough(agent, recursion_depth=0):
if recursion_depth > 10:
logging.warning(f"""Recursion depth exceeded 10, is currently {recursion_depth}. This is likely an error in the code. Please check the code.""")
return agent
total_context_length = 0
for message_index, message in enumerate(agent._agents[DEFAULT_AGENT]._llm._message_history):
for content_piece_index, content_piece in enumerate(message.content):
total_context_length += len(content_piece.text)
logging.debug(f"""The total context length is: {total_context_length}""")
if total_context_length > MAXIMUM_TOTAL_CONTEXT_LENGTH:
logging.debug(f"""The total context length is too long: {total_context_length}, removing the first message.""")
agent._agents[DEFAULT_AGENT]._llm._message_history.pop(0)
agent = remove_oldest_messages_until_short_enough(agent, recursion_depth=recursion_depth+1)
else:
return agent
return agent
agent = remove_oldest_messages_until_short_enough(agent)
if not agent._agents[DEFAULT_AGENT]._llm._message_history:
# _message_history is a list of messages, each message having a content attribute (which in turn is yet another list of content pieces).
logging.warning(f"""The message history is empty.""")
logging.warning(f"""
truncate_agent_history()
total_original_length: {total_original_length}
total_truncated_length: {total_truncated_length}
""")
return agent