github/MemGPT
7
0
Fork 0
mirror of https://github.com/cpacker/MemGPT.git synced 2025-06-03 04:30:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add llm config to ttfs logging (#1998)

Browse Source
This commit is contained in:
cthomas 2025-05-02 17:41:43 -07:00 committed by GitHub
parent d60b8788fe
commit 16cc387fe1
2 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
letta/server/rest_api/utils.py
View File

@ -20,6 +20,7 @@ from letta.helpers.message_helper import convert_message_creates_to_messages
from letta.log import get_logger from letta.log import get_logger
from letta.schemas.enums import MessageRole from letta.schemas.enums import MessageRole
from letta.schemas.letta_message_content import OmittedReasoningContent, ReasoningContent, RedactedReasoningContent, TextContent from letta.schemas.letta_message_content import OmittedReasoningContent, ReasoningContent, RedactedReasoningContent, TextContent
from letta.schemas.llm_config import LLMConfig
from letta.schemas.message import Message, MessageCreate from letta.schemas.message import Message, MessageCreate
from letta.schemas.usage import LettaUsageStatistics from letta.schemas.usage import LettaUsageStatistics
from letta.schemas.user import User from letta.schemas.user import User
@ -53,6 +54,7 @@ async def sse_async_generator(
usage_task: Optional[asyncio.Task] = None, usage_task: Optional[asyncio.Task] = None,
finish_message=True, finish_message=True,
request_start_timestamp_ns: Optional[int] = None, request_start_timestamp_ns: Optional[int] = None,
llm_config: Optional[LLMConfig] = None,
): ):
""" """
Wraps a generator for use in Server-Sent Events (SSE), handling errors and ensuring a completion message. Wraps a generator for use in Server-Sent Events (SSE), handling errors and ensuring a completion message.
@ -70,6 +72,7 @@ async def sse_async_generator(
ttft_span = None ttft_span = None
if request_start_timestamp_ns is not None: if request_start_timestamp_ns is not None:
ttft_span = tracer.start_span("time_to_first_token", start_time=request_start_timestamp_ns) ttft_span = tracer.start_span("time_to_first_token", start_time=request_start_timestamp_ns)
ttft_span.set_attributes({f"llm_config.{k}": v for k, v in llm_config.model_dump().items()})
try: try:
async for chunk in generator: async for chunk in generator:

1
letta/server/server.py
View File

@ -1725,6 +1725,7 @@ class SyncServer(Server):
usage_task=task, usage_task=task,
finish_message=include_final_message, finish_message=include_final_message,
request_start_timestamp_ns=request_start_timestamp_ns, request_start_timestamp_ns=request_start_timestamp_ns,
llm_config=llm_config,
), ),
media_type="text/event-stream", media_type="text/event-stream",
) )