Reinvent conversation metadata API and implementation (#71)

- Changed the ConversationMetadata table to be a key/value list -- more flexible.
- When initializing a database, after the schema is initialized, also initialize that table
- We're now using schema_version==1

Almost all of this was written by Claude Sonnet 4.5; I also briefly tried Gemini 2.5 Pro and GPT-5-Codex (Preview), all using agent mode in VS Code.

Author: gvanrossum

AGENTS.md

@@ -10,7 +10,7 @@ that make changes to the repository. Not even `git add`**
 When moving, copying or deleting files, use the git commands: `git mv`, `git cp`, `git rm`
 
 - Don't use '!' on the command line, it's some bash magic (even inside single quotes)
-- Activate `.venv`: make venv; source .venv/bin/activate
+- Activate `.venv`: `make venv; source .venv/bin/activate` (run this only once)
 - To get API keys in ad-hoc code, call `typeagent.aitools.utils.load_dotenv()`
 - Use `pytest test` to run tests in test/
 - Use `pyright` to check type annotations in tools/, test/, typeagent/, gmail/

test/test_conversation_metadata.py

@@ -26,7 +26,6 @@
 from typeagent.aitools import utils
 from typeagent.aitools.embeddings import AsyncEmbeddingModel
 from typeagent.storage.utils import create_storage_provider
-from typeagent.storage.sqlite.provider import SqliteStorageProvider
 from typeagent.transcripts.transcript_ingest import (
     get_transcript_duration,
     get_transcript_speakers,

tools/ingest_vtt.py

@@ -4,6 +4,7 @@
 """Base class for conversations with incremental indexing support."""
 
 from dataclasses import dataclass
+from datetime import datetime, timezone
 from typing import Generic, Self, TypeVar
 
 import typechat
@@ -160,6 +161,11 @@ async def add_messages_with_indexing(
                 - start_points.semref_count,
             )
 
+            # Update the updated_at timestamp
+            storage.update_conversation_timestamps(
+                updated_at=datetime.now(timezone.utc)
+            )
+
             return result
 
     async def _add_metadata_knowledge_incremental(

typeagent/knowpro/conversation_base.py

@@ -777,6 +777,24 @@ class ConversationDataWithIndexes[TMessageData](ConversationData[TMessageData]):
 # --------
 
 
+@dataclass
+class ConversationMetadata:
+    """Storage-provider-agnostic metadata for a conversation.
+
+    This dataclass represents metadata that can be read from and written to
+    any storage provider (SQLite, in-memory, etc.). Providers may store this
+    internally in different formats (e.g., key-value pairs), but this provides
+    a uniform interface for accessing conversation metadata.
+    """
+
+    name_tag: str
+    schema_version: int
+    created_at: Datetime
+    updated_at: Datetime
+    tags: list[str]
+    extra: dict[str, str]  # All extra values stored as strings
+
+
 class IReadonlyCollection[T, TOrdinal](AsyncIterable[T], Protocol):
     async def size(self) -> int: ...
 
@@ -832,6 +850,39 @@ async def get_related_terms_index(self) -> ITermToRelatedTermsIndex: ...
 
     async def get_conversation_threads(self) -> IConversationThreads: ...
 
+    # Metadata management
+    def get_conversation_metadata(self) -> Any | None:
+        """Get conversation metadata.
+
+        Returns ConversationMetadata or None if no metadata exists.
+        Return type is Any to avoid circular import with storage layer.
+        """
+        ...
+
+    def set_conversation_metadata(self, **kwds: str | list[str] | None) -> None:
+        """Set or update conversation metadata key-value pairs.
+
+        Args:
+            **kwds: Metadata keys and values where:
+                - str value: Sets a single key-value pair (replaces existing)
+                - list[str] value: Sets multiple values for the same key
+                - None value: Deletes all rows for the given key
+        """
+        ...
+
+    def update_conversation_timestamps(
+        self,
+        created_at: Datetime | None = None,
+        updated_at: Datetime | None = None,
+    ) -> None:
+        """Update conversation timestamps.
+
+        Args:
+            created_at: Optional creation timestamp
+            updated_at: Optional last updated timestamp
+        """
+        ...
+
     # Transaction management
     async def __aenter__(self) -> Self:
         """Enter transaction context. Calls begin_transaction()."""

typeagent/knowpro/interfaces.py

@@ -18,33 +18,14 @@
 from ..storage.memory.convthreads import ConversationThreads
 from ..knowpro.convsettings import ConversationSettings
 from ..knowpro.interfaces import (
-    AddMessagesResult,
     ConversationDataWithIndexes,
-    Datetime,
-    ICollection,
-    IConversation,
-    IConversationSecondaryIndexes,
-    IKnowledgeSource,
-    IMessage,
-    IMessageCollection,
-    IMessageMetadata,
-    ISemanticRefCollection,
-    IStorageProvider,
-    ITermToSemanticRefIndex,
-    IndexingStartPoints,
-    MessageOrdinal,
     SemanticRef,
     Term,
-    Timedelta,
 )
 from ..storage.memory.messageindex import MessageTextIndex
 from ..storage.memory.reltermsindex import TermToRelatedTermsMap
 from ..storage.utils import create_storage_provider
 from ..knowpro import serialization
-from ..storage.memory.collections import (
-    MemoryMessageCollection,
-    MemorySemanticRefCollection,
-)
 from ..knowpro.universal_message import (
     ConversationMessage,
     ConversationMessageMeta,

typeagent/podcasts/podcast.py

@@ -1,7 +1,12 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-"""Memory storage provider implementation."""
+"""In-memory storage provider implementation."""
+
+
+from datetime import datetime
+
+from ...knowpro import interfaces
 
 from .collections import MemoryMessageCollection, MemorySemanticRefCollection
 from .semrefindex import TermToSemanticRefIndex
@@ -95,3 +100,37 @@ async def get_semantic_ref_collection(self) -> MemorySemanticRefCollection:
     async def close(self) -> None:
         """Close the storage provider."""
         pass
+
+    def get_conversation_metadata(self) -> None:
+        """Get conversation metadata (no-op for in-memory storage).
+
+        Returns None since in-memory storage doesn't persist metadata.
+        """
+        return None
+
+    def set_conversation_metadata(self, **kwds: str | list[str] | None) -> None:
+        """Set conversation metadata (no-op for in-memory storage).
+
+        This method exists for API compatibility with SqliteStorageProvider
+        but does nothing since in-memory storage doesn't persist metadata.
+
+        Args:
+            **kwds: Metadata keys and values (ignored)
+        """
+        pass
+
+    def update_conversation_timestamps(
+        self,
+        created_at: datetime | None = None,
+        updated_at: datetime | None = None,
+    ) -> None:
+        """Update conversation timestamps (no-op for in-memory storage).
+
+        This method exists for API compatibility with SqliteStorageProvider
+        but does nothing since in-memory storage doesn't persist metadata.
+
+        Args:
+            created_at: Optional creation timestamp (ignored)
+            updated_at: Optional last updated timestamp (ignored)
+        """
+        pass

typeagent/storage/memory/provider.py

@@ -6,26 +6,24 @@
 from .collections import SqliteMessageCollection, SqliteSemanticRefCollection
 from .messageindex import SqliteMessageTextIndex
 from .propindex import SqlitePropertyIndex
-from .reltermsindex import SqliteRelatedTermsIndex
-from .semrefindex import SqliteTermToSemanticRefIndex
-from .timestampindex import SqliteTimestampToTextRangeIndex
 from .provider import SqliteStorageProvider
+from .reltermsindex import SqliteRelatedTermsIndex
 from .schema import (
-    ConversationMetadata,
     init_db_schema,
     get_db_schema_version,
 )
+from .semrefindex import SqliteTermToSemanticRefIndex
+from .timestampindex import SqliteTimestampToTextRangeIndex
 
 __all__ = [
+    "get_db_schema_version",
+    "init_db_schema",
     "SqliteMessageCollection",
-    "SqliteSemanticRefCollection",
     "SqliteMessageTextIndex",
     "SqlitePropertyIndex",
     "SqliteRelatedTermsIndex",
+    "SqliteSemanticRefCollection",
+    "SqliteStorageProvider",
     "SqliteTermToSemanticRefIndex",
     "SqliteTimestampToTextRangeIndex",
-    "SqliteStorageProvider",
-    "ConversationMetadata",
-    "init_db_schema",
-    "get_db_schema_version",
 ]