diff --git a/dev/benchmark/env.template b/dev/benchmark/env.template deleted file mode 100644 index ea7473b2..00000000 --- a/dev/benchmark/env.template +++ /dev/null @@ -1,3 +0,0 @@ -TENANT_ID= -APP_ID= -APP_SECRET= \ No newline at end of file diff --git a/dev/benchmark/src/config.py b/dev/benchmark/src/config.py deleted file mode 100644 index 403fbafc..00000000 --- a/dev/benchmark/src/config.py +++ /dev/null @@ -1,23 +0,0 @@ -import os -from dotenv import load_dotenv - -load_dotenv() - - -class BenchmarkConfig: - """Configuration class for benchmark settings.""" - - TENANT_ID: str = "" - APP_ID: str = "" - APP_SECRET: str = "" - AGENT_API_URL: str = "" - - @classmethod - def load_from_env(cls) -> None: - """Loads configuration values from environment variables.""" - cls.TENANT_ID = os.environ.get("TENANT_ID", "") - cls.APP_ID = os.environ.get("APP_ID", "") - cls.APP_SECRET = os.environ.get("APP_SECRET", "") - cls.AGENT_URL = os.environ.get( - "AGENT_API_URL", "http://localhost:3978/api/messages" - ) diff --git a/dev/benchmark/src/generate_token.py b/dev/benchmark/src/generate_token.py deleted file mode 100644 index 19c0e93e..00000000 --- a/dev/benchmark/src/generate_token.py +++ /dev/null @@ -1,34 +0,0 @@ -import requests -from .config import BenchmarkConfig - -URL = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" - - -def generate_token(app_id: str, app_secret: str) -> str: - """Generate a token using the provided app credentials.""" - - url = URL.format(tenant_id=BenchmarkConfig.TENANT_ID) - - res = requests.post( - url, - headers={ - "Content-Type": "application/x-www-form-urlencoded", - }, - data={ - "grant_type": "client_credentials", - "client_id": app_id, - "client_secret": app_secret, - "scope": f"{app_id}/.default", - }, - timeout=10, - ) - return res.json().get("access_token") - - -def generate_token_from_env() -> str: - """Generates a token using environment variables.""" - app_id = BenchmarkConfig.APP_ID - app_secret = BenchmarkConfig.APP_SECRET - if not app_id or not app_secret: - raise ValueError("APP_ID and APP_SECRET must be set in the BenchmarkConfig.") - return generate_token(app_id, app_secret) diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_SendsText_ReturnsPoem.yaml b/dev/integration/tests/basic_agent/directline/SendActivity_SendsText_ReturnsPoem.yaml deleted file mode 100644 index 12999ce3..00000000 --- a/dev/integration/tests/basic_agent/directline/SendActivity_SendsText_ReturnsPoem.yaml +++ /dev/null @@ -1,28 +0,0 @@ -test: -- type: input - activity: - type: message - channelId: directline - from: - id: user1 - name: User - conversation: - id: conversation-abc123 - recipient: - id: bot1 - name: Bot - text: poem - locale: en-US -- type: skip -# - type: assertion -# selector: -# activity: -# type: typing -# activity: -# text: ["CONTAINS", "Hold on for an awesome poem about Apollo"] -# - type: assertion -# selector: -# index: -1 -# activity: -# text: ["CONTAINS", "Apollo"] -# - type: breakpoint \ No newline at end of file diff --git a/dev/integration/tests/quickstart/test_quickstart_sample.py b/dev/integration/tests/quickstart/test_quickstart_sample.py deleted file mode 100644 index afd45e6c..00000000 --- a/dev/integration/tests/quickstart/test_quickstart_sample.py +++ /dev/null @@ -1,20 +0,0 @@ -import pytest - -from microsoft_agents.testing import ( - ddt, - Integration, - AiohttpEnvironment, -) - -from ...samples import QuickstartSample - - -@ddt("tests/quickstart/directline") -class TestQuickstartDirectline(Integration): - _sample_cls = QuickstartSample - _environment_cls = AiohttpEnvironment - - -@ddt("tests/quickstart/directline") -@pytest.mark.skipif(True, reason="Skipping external agent tests for now.") -class TestQuickstartExternalDirectline(Integration): ... diff --git a/dev/benchmark/README.md b/dev/microsoft-agents-testing/BENCHMARK.md similarity index 100% rename from dev/benchmark/README.md rename to dev/microsoft-agents-testing/BENCHMARK.md diff --git a/dev/microsoft-agents-testing/README.md b/dev/microsoft-agents-testing/README.md index 2c52b935..d8c91648 100644 --- a/dev/microsoft-agents-testing/README.md +++ b/dev/microsoft-agents-testing/README.md @@ -1,1311 +1,738 @@ -# Microsoft 365 Agents SDK for Python - Testing Framework +# microsoft-agents-testing -A comprehensive testing framework designed specifically for Microsoft 365 Agents SDK, providing essential utilities and abstractions to streamline integration testing, authentication, data-driven testing, and end-to-end agent validation. +A comprehensive testing framework for Microsoft Agents in Python. This package provides powerful tools for integration testing, data-driven testing, assertion helpers, authentication utilities, and performance benchmarking for agents built with the Microsoft Agents SDK. ## Table of Contents -- [Why This Package Exists](#why-this-package-exists) -- [Key Features](#key-features) - - [Authentication Utilities](#authentication-utilities) - - [Integration Test Framework](#integration-test-framework) - - [Agent Communication Clients](#agent-communication-clients) - - [Data-Driven Testing](#data-driven-testing) - - [Advanced Assertions Framework](#advanced-assertions-framework) - - [Testing Utilities](#testing-utilities) - [Installation](#installation) +- [Features](#features) - [Quick Start](#quick-start) -- [Usage Guide](#usage-guide) -- [Advanced Examples](#advanced-examples) +- [Core Components](#core-components) + - [Integration Testing](#integration-testing) + - [Data-Driven Testing (DDT)](#data-driven-testing-ddt) + - [Assertions](#assertions) + - [Authentication](#authentication) + - [SDK Configuration](#sdk-configuration) +- [CLI Tools](#cli-tools) +- [Usage Examples](#usage-examples) - [API Reference](#api-reference) -- [CI/CD Integration](#cicd-integration) +- [Future Goals](#future-goals) - [Contributing](#contributing) -## Why This Package Exists - -Building and testing conversational agents presents unique challenges that standard testing frameworks don't address. This package eliminates these pain points by providing powerful abstractions specifically designed for agent testing scenarios, including support for data-driven testing with YAML/JSON configurations. - -**Key Benefits:** -- Write tests once in YAML/JSON, run them everywhere -- Reduce boilerplate code with pre-built fixtures and clients -- Validate complex conversation flows with declarative assertions -- Maintain test suites that are easy to read and maintain -- Integrate seamlessly with pytest and CI/CD pipelines - -## Key Features - -### ๐Ÿ” Authentication Utilities - -Generate OAuth2 access tokens for testing secured agents with Microsoft Authentication Library (MSAL) integration. +## Installation -**Features:** -- Client credentials flow support -- Environment variable configuration -- SDK config integration +### Standard Installation -**Example:** +```bash +pip install microsoft-agents-testing +``` -```python -from microsoft_agents.testing import generate_token, generate_token_from_config +### Development Installation (Editable Mode) -# Generate token directly -token = generate_token( - app_id="your-app-id", - app_secret="your-secret", - tenant_id="your-tenant" -) +For active development: -# Or from SDK config -token = generate_token_from_config(sdk_config) +```bash +pip install -e ./microsoft-agents-testing/ --config-settings editable_mode=compat ``` -### ๐Ÿงช Integration Test Framework +### Requirements -Pre-built pytest fixtures and abstractions for agent integration testing. +- Python >= 3.10 +- Dependencies: + - `microsoft-agents-activity` + - `microsoft-agents-hosting-core` + - `microsoft-agents-authentication-msal` + - `microsoft-agents-hosting-aiohttp` + - `pyjwt>=2.10.1` + - `isodate>=0.6.1` + - `azure-core>=1.30.0` + - `python-dotenv>=1.1.1` -**Features:** -- Pytest fixture integration -- Environment abstraction for different hosting configurations -- Sample management for test organization -- Application lifecycle management -- Automatic setup and teardown +## Features -**Example:** - -```python -from microsoft_agents.testing import Integration, AiohttpEnvironment, Sample - -class MyAgentSample(Sample): - async def init_app(self): - self.app = create_my_agent_app(self.env) - - @classmethod - async def get_config(cls): - return {"service_url": "http://localhost:3978"} +โœ… **Integration Testing Framework** - Full-featured integration testing with pytest support +โœ… **Data-Driven Testing** - YAML-based test definitions for declarative testing +โœ… **Flexible Assertions** - Advanced model and field assertion capabilities +โœ… **Authentication Helpers** - OAuth token generation for Azure Bot Service +โœ… **CLI Tools** - Command-line interface for testing and benchmarking +โœ… **Performance Benchmarking** - Load testing with concurrent workers +โœ… **Response Mocking** - Built-in mock service for testing agent responses +โœ… **Activity Utilities** - Helper functions for activity manipulation -class MyAgentTests(Integration): - _sample_cls = MyAgentSample - _environment_cls = AiohttpEnvironment - - @pytest.mark.asyncio - async def test_conversation_flow(self, agent_client, sample): - # Client and sample are automatically set up via fixtures - response = await agent_client.send_activity("Hello") - assert response is not None -``` +## Quick Start -### ๐Ÿค– Agent Communication Clients +### 1. Set Up Environment -High-level clients for sending and receiving activities from agents under test. +Create a `.env` file with your Azure Bot Service credentials: -**Features:** -- Simple text message sending -- Full Activity object support -- Automatic token management -- Support for `expectReplies` delivery mode -- Response collection and management +```env +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID=your-client-id +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID=your-tenant-id +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET=your-client-secret +``` -**AgentClient Example:** +### 2. Basic Integration Test ```python -from microsoft_agents.testing import AgentClient -from microsoft_agents.activity import Activity, ActivityTypes - -client = AgentClient( - agent_url="http://localhost:3978", - cid="conversation-id", - client_id="your-client-id", - tenant_id="your-tenant-id", - client_secret="your-secret" -) +import pytest +from microsoft_agents.testing import Integration, ddt -# Send simple text message -response = await client.send_activity("What's the weather?") +@ddt("tests/my_agent/directline") +class TestMyAgent(Integration): + _agent_url = "http://localhost:3978/" + _service_url = "http://localhost:8001/" + _config_path = ".env" +``` -# Send full Activity object -activity = Activity(type=ActivityTypes.message, text="Hello") -response = await client.send_activity(activity) +### 3. Run Data-Driven Tests via CLI -# Send with expectReplies delivery mode -replies = await client.send_expect_replies("What can you do?") -for reply in replies: - print(reply.text) +```bash +aclip --env_path .env ddt ./tests/my_test.yaml ``` -**ResponseClient Example:** +### 4. Generate Authentication Token ```python -from microsoft_agents.testing import ResponseClient +from microsoft_agents.testing import generate_token -# Create response client to collect agent responses -async with ResponseClient(host="localhost", port=9873) as response_client: - # ... send activities with agent_client ... - - # Collect all responses - responses = await response_client.pop() - assert len(responses) > 0 +token = generate_token( + app_id="your-app-id", + app_secret="your-secret", + tenant_id="your-tenant-id" +) ``` -### ๐Ÿ“‹ Data-Driven Testing +## Core Components -Write test scenarios in YAML or JSON files and execute them automatically. Perfect for creating reusable test suites, regression tests, and living documentation. +### Integration Testing -**Features:** -- Declarative test definition in YAML/JSON -- Parent/child file inheritance for shared defaults -- Multiple step types (input, assertion, sleep, breakpoint) -- Flexible assertions with selectors and quantifiers -- Automatic test discovery and generation -- Field-level assertion operators +The `Integration` class provides a complete pytest-based integration testing framework with fixtures for environment setup, agent clients, and response handling. -#### Using the @ddt Decorator +#### Key Features: +- Automatic environment initialization +- Agent client management with authentication +- Response client for mocking service endpoints +- Configurable service and agent URLs +- Support for multiple test environments -The @ddt (data-driven tests) decorator automatically loads test files and generates pytest test methods: +#### Example: ```python -from microsoft_agents.testing import Integration, AiohttpEnvironment, ddt +import pytest +from microsoft_agents.testing import Integration, ddt -@ddt("tests/my_agent/test_cases", recursive=True) -class TestMyAgent(Integration): - _sample_cls = MyAgentSample - _environment_cls = AiohttpEnvironment - _agent_url = "http://localhost:3978" - _cid = "test-conversation" +@ddt("tests/basic_agent/directline", prefix="directline") +class TestBasicAgent(Integration): + _agent_url = "http://localhost:3978/" + _service_url = "http://localhost:8001/" + _config_path = "agents/basic_agent/.env" + + # Tests are automatically generated from YAML files ``` -This will: -1. Load all `.yaml` and `.json` files from `tests/my_agent/test_cases` (and subdirectories if `recursive=True`) -2. Create a pytest test method for each file (e.g., `test_data_driven__greeting_test`) -3. Execute the test flow defined in each file +### Data-Driven Testing (DDT) -#### Test File Format +Data-driven testing allows you to define test scenarios in YAML files, making tests declarative, maintainable, and easy to understand. -**Shared Defaults (parent.yaml):** +#### YAML Test Structure: ```yaml -name: directline +name: SendActivity_ConversationUpdate_ReturnsWelcomeMessage +description: Tests that a conversation update activity triggers a welcome message + defaults: input: activity: channelId: directline locale: en-US - serviceUrl: http://localhost:56150 - deliveryMode: expectReplies - conversation: - id: conv1 - from: - id: user1 - name: User - recipient: - id: bot - name: Bot -``` - -**Test File (greeting_test.yaml):** + assertion: + quantifier: all -```yaml -parent: parent.yaml -name: greeting_test -description: Test basic greeting conversation test: - type: input activity: - type: message - text: hello world - - - type: assertion - selector: - activity: - type: message - activity: - type: message - text: "[0] You said: hello world" - - - type: input - activity: - type: message - text: hello again + type: conversationUpdate + from: + id: user1 + conversation: + id: conversation-001 + membersAdded: + - id: bot@serviceurl + name: bot + - id: user1 - type: assertion selector: - index: -1 # Select the last matching activity - activity: - type: message + index: -1 activity: type: message - text: "[1] You said: hello again" -``` - -#### Test Step Types - -##### Input Steps - -Send activities to the agent under test: - -```yaml -- type: input - activity: - type: message - text: "What's the weather?" + text: ["CONTAINS", "Hello and Welcome!"] ``` -With overrides: +#### Test Step Types: -```yaml -- type: input - activity: - type: message - text: "Hello" - locale: "fr-FR" # Override default locale - channelData: - custom: "value" -``` +- **`input`** - Send an activity to the agent +- **`assertion`** - Assert expected responses +- **`sleep`** - Wait for a specified duration +- **`breakpoint`** - Trigger a debugger breakpoint +- **`skip`** - Skip the current step -##### Assertion Steps +#### Defaults System: -Verify agent responses with flexible matching: +You can define defaults for inputs, assertions, and sleep durations to reduce repetition: ```yaml -- type: assertion - quantifier: all # Options: all, any, one, none - selector: - index: 0 # Optional: select by index (0, -1, etc.) +defaults: + input: activity: - type: message # Filter by activity fields - activity: - type: message - text: ["CONTAINS", "sunny"] # Use operators for flexible matching + channelId: directline + locale: en-US + textFormat: plain + assertion: + quantifier: all + sleep: + duration: 0.5 ``` -**Quantifiers:** -- `all` (default): Every selected activity must match -- `any`: At least one activity must match -- `one`: Exactly one activity must match -- `none`: No activities should match - -**Selectors:** -- `activity`: Filter activities by field values -- `index`: Select specific activity by index (supports negative indices) - -**Field Assertion Operators:** -- `["CONTAINS", "substring"]`: Check if string contains substring -- `["NOT_CONTAINS", "substring"]`: Check if string doesn't contain substring -- `["RE_MATCH", "pattern"]`: Check if string matches regex pattern -- `["IN", [list]]`: Check if value is in list -- `["NOT_IN", [list]]`: Check if value is not in list -- `["EQUALS", value]`: Explicit equality check -- `["NOT_EQUALS", value]`: Explicit inequality check -- `["GREATER_THAN", number]`: Numeric comparison -- `["LESS_THAN", number]`: Numeric comparison -- Direct value: Implicit equality check - -##### Sleep Steps - -Add delays between operations: +#### Parent/Child Test Inheritance: -```yaml -- type: sleep - duration: 0.5 # seconds -``` - -With default duration: +Tests can inherit defaults from parent test files: ```yaml -defaults: - sleep: - duration: 0.2 +parent: _parent.yaml +name: ChildTest test: - - type: sleep # Uses default duration -``` - -##### Breakpoint Steps - -Pause execution for debugging: - -```yaml -- type: breakpoint + - type: input + activity: + text: "Hello" ``` -When the test reaches this step, it will trigger a Python breakpoint, allowing you to inspect state in a debugger. +### Assertions -#### Loading Tests Programmatically +Powerful assertion system for validating agent responses with support for nested object validation and flexible matching. -Load and run tests manually without the decorator: +#### Field Assertions ```python -from microsoft_agents.testing import load_ddts, DataDrivenTest +from microsoft_agents.testing import assert_field, FieldAssertionType -# Load all test files from a directory -tests = load_ddts("tests/my_agent", recursive=True) +# Exact match +assert_field(activity.text, "Hello", FieldAssertionType.EQUALS) -# Run specific tests -for test in tests: - print(f"Running: {test.name}") - await test.run(agent_client, response_client) -``` +# Contains check +assert_field(activity.text, "Hello", FieldAssertionType.CONTAINS) -Load from specific file: - -```python -tests = load_ddts("tests/greeting_test.yaml", recursive=False) -test = tests[0] -await test.run(agent_client, response_client) +# Exists check +assert_field(activity.text, None, FieldAssertionType.EXISTS) ``` -### โœ… Advanced Assertions Framework - -Powerful assertion system for validating agent responses with flexible matching criteria. - -#### ModelAssertion - -Create assertions for validating lists of activities: +#### Model Assertions ```python -from microsoft_agents.testing import ModelAssertion, Selector, AssertionQuantifier +from microsoft_agents.testing import assert_model, ModelAssertion +from microsoft_agents.activity import Activity -# Create an assertion +# Simple assertion +expected = Activity(type="message", text="Hello") +assert_model(actual_activity, expected) + +# Advanced assertion with selector assertion = ModelAssertion( - assertion={"type": "message", "text": "Hello"}, - selector=Selector(selector={"type": "message"}), + assertion={"type": "message", "text": ["CONTAINS", "Hello"]}, + selector=ModelSelector(index=-1), quantifier=AssertionQuantifier.ALL ) -# Test activities -activities = [...] # List of Activity objects -passes, error = assertion.check(activities) - -# Or use as callable (raises AssertionError on failure) -assertion(activities) -``` - -From configuration dictionary: - -```python -config = { - "activity": {"type": "message", "text": "Hello"}, - "selector": {"activity": {"type": "message"}}, - "quantifier": "all" -} -assertion = ModelAssertion.from_config(config) -``` - -#### Selectors - -Filter activities before validation: - -```python -from microsoft_agents.testing import Selector - -# Select all message activities -selector = Selector(selector={"type": "message"}) -messages = selector(activities) - -# Select the first message activity -selector = Selector(selector={"type": "message"}, index=0) -first_message = selector.select_first(activities) - -# Select the last message activity -selector = Selector(selector={"type": "message"}, index=-1) -last_message = selector(activities)[0] - -# Select by multiple fields -selector = Selector(selector={ - "type": "message", - "locale": "en-US", - "channelId": "directline" -}) +# Check multiple activities +passes, error = assertion.check(activity_list) +assert passes, error ``` -From configuration: - -```python -config = { - "activity": {"type": "message"}, - "index": -1 -} -selector = Selector.from_config(config) -``` +#### Assertion Quantifiers: -#### Quantifiers +- **`ALL`** - All selected items must match +- **`ONE`** - Exactly one item must match +- **`NONE`** - No items should match -Control how many activities must match the assertion: +#### Model Selector: ```python -from microsoft_agents.testing import AssertionQuantifier +from microsoft_agents.testing import ModelSelector -# ALL: Every selected activity must match (default) -quantifier = AssertionQuantifier.ALL +# Select by index +selector = ModelSelector(index=-1) # Last item -# ANY: At least one activity must match -quantifier = AssertionQuantifier.ANY +# Select by model properties +selector = ModelSelector(model={"type": "message"}) -# ONE: Exactly one activity must match -quantifier = AssertionQuantifier.ONE - -# NONE: No activities should match -quantifier = AssertionQuantifier.NONE - -# From string -quantifier = AssertionQuantifier.from_config("all") +# Select first match +first_match = selector.select_first(activities) ``` -#### Field Assertions +### Authentication -Test individual fields with operators: +Generate OAuth tokens for testing against Azure Bot Service. ```python -from microsoft_agents.testing import check_field, FieldAssertionType - -# String contains -result = check_field("Hello world", ["CONTAINS", "world"]) # True - -# Regex match -result = check_field("ID-12345", ["RE_MATCH", r"ID-\d+"]) # True - -# Value in list -result = check_field(5, ["IN", [1, 3, 5, 7]]) # True - -# Value not in list -result = check_field(2, ["NOT_IN", [1, 3, 5, 7]]) # True - -# Numeric comparisons -result = check_field(10, ["GREATER_THAN", 5]) # True -result = check_field(3, ["LESS_THAN", 10]) # True - -# String doesn't contain -result = check_field("Hello", ["NOT_CONTAINS", "world"]) # True - -# Exact equality -result = check_field("test", "test") # True -result = check_field(42, ["EQUALS", 42]) # True - -# Inequality -result = check_field("foo", ["NOT_EQUALS", "bar"]) # True -``` +from microsoft_agents.testing import generate_token, generate_token_from_config +from microsoft_agents.testing import SDKConfig -Verbose checking with error details: +# Direct token generation +token = generate_token( + app_id="your-app-id", + app_secret="your-secret", + tenant_id="your-tenant-id" +) -```python -from microsoft_agents.testing import check_field_verbose - -passes, error_data = check_field_verbose("Hello", ["CONTAINS", "world"]) -if not passes: - print(f"Field: {error_data.field_path}") - print(f"Actual: {error_data.actual_value}") - print(f"Expected: {error_data.assertion}") - print(f"Type: {error_data.assertion_type}") +# Token from configuration +config = SDKConfig(env_path=".env") +token = generate_token_from_config(config) ``` -#### Activity Assertions +### SDK Configuration -Check entire activities: +The `SDKConfig` class loads and provides access to SDK configuration from `.env` files or environment variables. ```python -from microsoft_agents.testing import check_model, assert_model - -activity = Activity(type="message", text="Hello", locale="en-US") - -# Check without raising exception -assertion = {"type": "message", "text": ["CONTAINS", "Hello"]} -result = check_activity(activity, assertion) # True +from microsoft_agents.testing import SDKConfig -# Check with detailed error information -passes, error_data = check_activity_verbose(activity, assertion) +# Load configuration +config = SDKConfig(env_path=".env") -# Assert with exception on failure -assert_model(activity, assertion) # Raises AssertionError if fails -``` - -Nested field checking: +# Get connection settings +connection = config.get_connection("SERVICE_CONNECTION") -```python -assertion = { - "type": "message", - "channelData": { - "user": { - "id": ["RE_MATCH", r"user-\d+"] - } - } -} -assert_model(activity, assertion) +# Access configuration dictionary +config_dict = config.config ``` -### ๐Ÿ› ๏ธ Testing Utilities +## CLI Tools -Helper functions for common testing operations. +The package includes a powerful CLI tool accessible via the `aclip` command. -#### populate_activity +### Available Commands -Fill activity objects with default values: - -```python -from microsoft_agents.testing import populate_activity -from microsoft_agents.activity import Activity +#### 1. Data-Driven Testing -defaults = { - "service_url": "http://localhost", - "channel_id": "test", - "locale": "en-US" -} +Run data-driven tests from YAML files: -activity = Activity(type="message", text="Hello") -activity = populate_activity(activity, defaults) - -# activity now has service_url, channel_id, and locale set -``` - -#### get_host_and_port - -Parse URLs to extract host and port: - -```python -from microsoft_agents.testing import get_host_and_port - -host, port = get_host_and_port("http://localhost:3978/api/messages") -# Returns: ("localhost", 3978) - -host, port = get_host_and_port("https://myagent.azurewebsites.net") -# Returns: ("myagent.azurewebsites.net", 443) +```bash +aclip --env_path .env ddt ./tests/my_test.yaml --service_url http://localhost:8001/ ``` -## Installation +Options: +- `--env_path` - Path to environment file (default: `.env`) +- `--service_url` - Service URL for responses (default: `http://localhost:8001/`) +- `--pytest-args` - Arguments to pass to pytest (default: `-v -s`) -```bash -pip install microsoft-agents-testing -``` +#### 2. Authentication Test Server -For development: +Run a test authentication server: ```bash -pip install microsoft-agents-testing[dev] +aclip --env_path .env auth --port 3978 ``` -## Quick Start - -### Traditional Integration Testing - -```python -import pytest -from microsoft_agents.testing import Integration, AiohttpEnvironment, Sample -from microsoft_agents.activity import Activity - -class MyAgentSample(Sample): - async def init_app(self): - # Initialize your agent application - from my_agent import create_app - self.app = create_app(self.env) - - @classmethod - async def get_config(cls): - return { - "service_url": "http://localhost:3978", - "app_id": "test-app-id", - } - -class TestMyAgent(Integration): - _sample_cls = MyAgentSample - _environment_cls = AiohttpEnvironment - - _agent_url = "http://localhost:3978" - _cid = "test-conversation" - - @pytest.mark.asyncio - async def test_greeting(self, agent_client): - response = await agent_client.send_activity("Hello") - assert "Hi there" in response - - @pytest.mark.asyncio - async def test_conversation(self, agent_client): - replies = await agent_client.send_expect_replies("What can you do?") - assert len(replies) > 0 - assert replies[0].type == "message" -``` +Options: +- `--port` - Port to run the server on (default: `3978`) -### Data-Driven Testing +#### 3. Post Activity -**Step 1:** Create test YAML files in `tests` directory +Send a single activity to an agent: -```yaml -# tests/greeting.yaml -name: greeting_test -description: Test basic greeting functionality -defaults: - input: - activity: - type: message - locale: en-US - channelId: directline -test: - - type: input - activity: - text: Hello - - - type: assertion - activity: - type: message - text: ["CONTAINS", "Hi"] +```bash +aclip --env_path .env post --payload_path ./payload.json ``` -**Step 2:** Add the @ddt decorator to your test class - -```python -from microsoft_agents.testing import Integration, AiohttpEnvironment, ddt +Options: +- `--payload_path` / `-p` - Path to payload JSON file (default: `./payload.json`) +- `--verbose` / `-v` - Enable verbose logging +- `--async_mode` / `-a` - Run with coroutine workers -@ddt("tests", recursive=True) -class TestMyAgent(Integration): - _sample_cls = MyAgentSample - _environment_cls = AiohttpEnvironment - _agent_url = "http://localhost:3978" -``` +#### 4. Benchmarking -**Step 3:** Run tests with pytest +Run performance benchmarks against your agent: ```bash -pytest tests/ -v +aclip --env_path .env benchmark --payload_path ./payload.json --num_workers 10 ``` -Output: -``` -tests/test_my_agent.py::TestMyAgent::test_data_driven__greeting_test PASSED -``` +Options: +- `--payload_path` / `-p` - Path to payload JSON file +- `--num_workers` / `-n` - Number of concurrent workers (default: `1`) +- `--verbose` / `-v` - Enable verbose logging +- `--async_mode` / `-a` - Use coroutine workers instead of threads -## Usage Guide +The benchmark command provides: +- Aggregated results with min/max/mean/median duration +- Success/failure rates +- Timeline visualization +- Throughput metrics -### Setting Up Authentication +## Usage Examples -#### From Environment Variables +### Complete Integration Test Example ```python -import os -from microsoft_agents.testing import generate_token - -token = generate_token( - app_id=os.getenv("CLIENT_ID"), - app_secret=os.getenv("CLIENT_SECRET"), - tenant_id=os.getenv("TENANT_ID") +import pytest +from microsoft_agents.testing import ( + Integration, + AgentClient, + ResponseClient, + ddt, ) -``` - -#### From SDK Config - -```python -from microsoft_agents.testing import SDKConfig, generate_token_from_config - -config = SDKConfig() -# config loads from environment or config file -token = generate_token_from_config(config) -``` -### Creating Custom Environments +@ddt("tests/my_agent/directline") +class TestMyAgentIntegration(Integration): + _agent_url = "http://localhost:3978/" + _service_url = "http://localhost:8001/" + _config_path = ".env" -```python -from microsoft_agents.testing import Environment -from aiohttp import web - -class MyCustomEnvironment(Environment): - async def init_env(self, config: dict): - # Custom initialization - self.config = config - # Set up any required services, databases, etc. - - def create_runner(self, host: str, port: int): - # Return application runner - from my_agent import create_app - app = create_app(self) - return MyAppRunner(app, host, port) -``` - -### Writing Complex Assertions - -```yaml -test: - - type: input - activity: - type: message - text: "Get user profile for user123" - - - type: assertion - quantifier: one - selector: - activity: - type: message - activity: - type: message - text: ["RE_MATCH", ".*user123.*"] - attachments: - - contentType: "application/vnd.microsoft.card.adaptive" - channelData: - userId: "user123" -``` - -## Advanced Examples - -### Complex Weather Conversation - -```yaml -name: weather_conversation -description: Test multi-turn weather conversation flow -defaults: - input: - activity: - type: message - channelId: directline - locale: en-US - conversation: - id: weather-conv-1 - assertion: - quantifier: all -test: - # Initial weather query - - type: input - activity: - text: "What's the weather in Seattle?" - - - type: assertion - selector: - activity: - type: message - activity: - type: message - text: ["CONTAINS", "Seattle"] - - # Wait for async processing - - type: sleep - duration: 0.2 - - # Follow-up question - - type: input - activity: - text: "What about tomorrow?" - - - type: assertion - selector: - activity: - type: message - activity: - type: message - text: ["RE_MATCH", "tomorrow.*forecast"] - - # Verify we got exactly one final response - - type: assertion - quantifier: one - selector: - index: -1 - activity: - type: message - activity: - type: message -``` - -### Testing Invoke Activities - -```yaml -parent: parent.yaml -name: test_invoke_profile -test: - - type: input - activity: - type: invoke - name: getUserProfile - value: - userId: "12345" - - # Ensure we don't get error responses - - type: assertion - quantifier: none - activity: - type: invokeResponse - value: - status: ["IN", [400, 404, 500]] - - # Verify successful response - - type: assertion - selector: - activity: - type: invokeResponse - activity: - type: invokeResponse - value: - status: 200 - body: - userId: "12345" - name: ["CONTAINS", "John"] - email: ["RE_MATCH", ".*@example\\.com"] -``` - -### Testing Conversation Update - -```yaml -parent: parent.yaml -name: conversation_update_test -test: - - type: input - activity: - type: conversationUpdate - membersAdded: - - id: bot-id - name: bot - - id: user - from: - id: user - recipient: - id: bot-id - name: bot - channelData: - clientActivityId: "123" - - - type: assertion - selector: - activity: - type: message - activity: - type: message - text: ["CONTAINS", "Hello and Welcome!"] -``` - -### Conditional Responses - -```yaml -test: - - type: input - activity: - text: "Show me options" - - # Verify at least one message was sent - - type: assertion - quantifier: any - selector: - activity: - type: message - activity: - type: message - - # Verify adaptive card was included - - type: assertion - quantifier: one - selector: - activity: - attachments: - - contentType: "application/vnd.microsoft.card.adaptive" - activity: - type: message -``` - -### Testing with Message Reactions - -```yaml -parent: parent.yaml -test: - # Send initial message - - type: input - activity: - type: message - text: "Great job!" - id: "msg-123" - - # Add a reaction - - type: input - activity: - type: messageReaction - reactionsAdded: - - type: like - replyToId: "msg-123" - - - type: assertion - selector: - activity: - type: message - activity: - type: message - text: ["CONTAINS", "Thanks for the reaction"] + @pytest.mark.asyncio + async def test_custom_scenario( + self, + agent_client: AgentClient, + response_client: ResponseClient + ): + # Send activity + await agent_client.send_activity("Hello") + + # Get responses + responses = await response_client.pop() + + # Assert + assert len(responses) > 0 + assert responses[0].text == "Hello! How can I help you?" ``` -## API Reference - -### Classes - -#### Integration -Base class for integration tests with pytest fixtures. +### Manual Agent Client Usage ```python -class Integration: - _sample_cls: type[Sample] - _environment_cls: type[Environment] - _agent_url: str - _service_url: str - _cid: str - _client_id: str - _tenant_id: str - _client_secret: str - - @pytest.fixture - async def environment(self) -> Environment: ... - - @pytest.fixture - async def sample(self, environment) -> Sample: ... - - @pytest.fixture - async def agent_client(self, sample, environment) -> AgentClient: ... - - @pytest.fixture - async def response_client(self) -> ResponseClient: ... -``` - -#### AgentClient -Client for sending activities to agents. +import asyncio +from microsoft_agents.testing import AgentClient +from microsoft_agents.activity import Activity -```python -class AgentClient: - def __init__( - self, - agent_url: str, - cid: str, - client_id: str, - tenant_id: str, - client_secret: str, - service_url: Optional[str] = None, - default_timeout: float = 5.0, - default_activity_data: Optional[Activity | dict] = None - ): ... +async def test_agent(): + client = AgentClient( + agent_url="http://localhost:3978/", + cid="conversation-id", + client_id="your-client-id", + tenant_id="your-tenant-id", + client_secret="your-secret", + service_url="http://localhost:8001/" + ) - async def send_activity( - self, - activity_or_text: Activity | str, - sleep: float = 0, - timeout: Optional[float] = None - ) -> str: ... + try: + # Send expect-replies activity + replies = await client.send_expect_replies( + Activity(text="Hello", type="message") + ) + + for reply in replies: + print(f"Reply: {reply.text}") - async def send_expect_replies( - self, - activity_or_text: Activity | str, - sleep: float = 0, - timeout: Optional[float] = None - ) -> list[Activity]: ... - - async def close(self) -> None: ... -``` - -#### ResponseClient -Client for receiving activities from agents. + finally: + await client.close() -```python -class ResponseClient: - def __init__(self, host: str = "localhost", port: int = 9873): ... - - async def pop(self) -> list[Activity]: ... - - async def __aenter__(self) -> ResponseClient: ... - async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: ... +asyncio.run(test_agent()) ``` -#### DataDrivenTest -Runner for YAML/JSON test definitions. +### Custom Sample and Environment ```python -class DataDrivenTest: - def __init__(self, test_flow: dict) -> None: ... - - @property - def name(self) -> str: ... - - async def run( - self, - agent_client: AgentClient, - response_client: ResponseClient - ) -> None: ... -``` - -#### ModelAssertion -Assertion engine for validating activities. +from microsoft_agents.testing import ( + Sample, + Environment, + AiohttpEnvironment, +) +from aiohttp import web -```python -class ModelAssertion: - def __init__( - self, - assertion: dict | Activity | None = None, - selector: Selector | None = None, - quantifier: AssertionQuantifier = AssertionQuantifier.ALL - ): ... - - def check(self, activities: list[Activity]) -> tuple[bool, Optional[str]]: ... - - def __call__(self, activities: list[Activity]) -> None: ... +class MySample(Sample): + async def init_app(self): + # Initialize your application + self.app = web.Application() + # Configure routes, etc. + return self.app - @staticmethod - def from_config(config: dict) -> ModelAssertion: ... -``` - -#### Selector -Filter activities based on criteria. + @classmethod + async def get_config(cls) -> dict: + return { + "CLIENT_ID": "your-client-id", + "TENANT_ID": "your-tenant-id", + "CLIENT_SECRET": "your-secret", + } -```python -class Selector: - def __init__( - self, - selector: dict | Activity | None = None, - index: int | None = None - ): ... - - def select(self, activities: list[Activity]) -> list[Activity]: ... - - def select_first(self, activities: list[Activity]) -> Activity | None: ... - - def __call__(self, activities: list[Activity]) -> list[Activity]: ... - - @staticmethod - def from_config(config: dict) -> Selector: ... +# Use in tests +class TestMySample(Integration): + _sample_cls = MySample + _environment_cls = AiohttpEnvironment + _agent_url = "http://localhost:3978/" + _service_url = "http://localhost:8001/" ``` -#### AssertionQuantifier -Quantifiers for assertions. +### Utility Functions ```python -class AssertionQuantifier(str, Enum): - ALL = "ALL" - ANY = "ANY" - ONE = "ONE" - NONE = "NONE" - - @staticmethod - def from_config(value: str) -> AssertionQuantifier: ... -``` +from microsoft_agents.testing import populate_activity, get_host_and_port +from microsoft_agents.activity import Activity -#### FieldAssertionType -Types of field assertions. +# Populate activity with defaults +defaults = Activity(channelId="directline", locale="en-US") +activity = Activity(type="message", text="Hello") +populated = populate_activity(activity, defaults) -```python -class FieldAssertionType(str, Enum): - EQUALS = "EQUALS" - NOT_EQUALS = "NOT_EQUALS" - GREATER_THAN = "GREATER_THAN" - LESS_THAN = "LESS_THAN" - CONTAINS = "CONTAINS" - NOT_CONTAINS = "NOT_CONTAINS" - IN = "IN" - NOT_IN = "NOT_IN" - RE_MATCH = "RE_MATCH" +# Parse URL +host, port = get_host_and_port("http://localhost:3978/") +# host = "localhost", port = 3978 ``` -### Decorators +## API Reference -#### @ddt -Load and execute data-driven tests. +### Classes -```python -def ddt(test_path: str, recursive: bool = True) -> Callable: - """ - Decorator to add data-driven tests to an integration test class. - - :param test_path: Path to test files directory - :param recursive: Load tests from subdirectories - """ -``` +#### `Integration` +Base class for integration tests with pytest fixtures. -### Functions +**Properties:** +- `service_url` - Service URL for responses +- `agent_url` - Agent URL for sending activities -#### generate_token -Generate OAuth2 access token. +**Fixtures:** +- `environment()` - Test environment instance +- `sample()` - Sample application instance +- `agent_client()` - Agent client for sending activities +- `response_client()` - Response client for receiving activities -```python -def generate_token(app_id: str, app_secret: str, tenant_id: str) -> str: ... -``` +**Methods:** +- `setup_method()` - Initialize test configuration +- `create_agent_client()` - Create agent client instance -#### generate_token_from_config -Generate token from SDK config. +#### `AgentClient` +Client for sending activities to an agent. -```python -def generate_token_from_config(sdk_config: SDKConfig) -> str: ... -``` +**Constructor Parameters:** +- `agent_url` - Agent endpoint URL +- `cid` - Conversation ID +- `client_id` - Azure AD client ID +- `tenant_id` - Azure AD tenant ID +- `client_secret` - Azure AD client secret +- `service_url` - Service URL for callbacks +- `default_activity_data` - Default activity values +- `default_sleep` - Default sleep duration after sending -#### load_ddts -Load data-driven test files. +**Methods:** +- `send_activity(activity_or_text, sleep)` - Send an activity +- `send_expect_replies(activity, sleep)` - Send and expect replies +- `send_invoke_activity(activity, sleep)` - Send invoke activity +- `close()` - Close the client session -```python -def load_ddts( - path: str | Path | None = None, - recursive: bool = False -) -> list[DataDrivenTest]: ... -``` +#### `ResponseClient` +Mock service for receiving agent responses. -#### populate_activity -Fill activity with default values. +**Constructor Parameters:** +- `host` - Host address (default: `localhost`) +- `port` - Port number (default: `9873`) -```python -def populate_activity( - activity: Activity, - defaults: dict | Activity -) -> Activity: ... -``` +**Methods:** +- `pop()` - Retrieve and clear received activities +- `__aenter__()` / `__aexit__()` - Async context manager support -#### get_host_and_port -Parse host and port from URL. +#### `DataDrivenTest` +Runner for YAML-based data-driven tests. -```python -def get_host_and_port(url: str) -> tuple[str, int]: ... -``` +**Constructor Parameters:** +- `test_flow` - Dictionary containing test configuration -#### check_activity -Check if activity matches assertion. +**Properties:** +- `name` - Test name -```python -def check_activity(activity: Activity, assertion: dict | Activity) -> bool: ... -``` +**Methods:** +- `run(agent_client, response_client)` - Execute the test -#### check_activity_verbose -Check activity with detailed error information. +#### `ModelAssertion` +Advanced assertion for model validation. -```python -def check_activity_verbose( - activity: Activity, - assertion: dict | Activity -) -> tuple[bool, Optional[AssertionErrorData]]: ... -``` +**Constructor Parameters:** +- `assertion` - Expected model or dict +- `selector` - Model selector for filtering +- `quantifier` - Assertion quantifier (ALL, ONE, NONE) -#### check_field -Check if field value matches assertion. +**Methods:** +- `check(items)` - Check items against assertion +- `from_config(config)` - Create from configuration dict -```python -def check_field(value: Any, assertion: Any) -> bool: ... -``` +#### `ModelSelector` +Selector for filtering models. -#### check_field_verbose -Check field with detailed error information. +**Constructor Parameters:** +- `model` - Model pattern to match +- `index` - Index to select -```python -def check_field_verbose( - value: Any, - assertion: Any, - field_path: str = "" -) -> tuple[bool, Optional[AssertionErrorData]]: ... -``` +**Methods:** +- `select(items)` - Select matching items +- `select_first(items)` - Select first match -#### assert_model -Assert activity matches, raise on failure. +#### `SDKConfig` +Configuration loader for SDK settings. -```python -def assert_model(activity: Activity, assertion: dict | Activity) -> None: ... -``` +**Constructor Parameters:** +- `env_path` - Path to .env file +- `load_into_environment` - Load into environment variables -#### assert_field -Assert field matches, raise on failure. +**Properties:** +- `config` - Configuration dictionary (read-only copy) -```python -def assert_field(value: Any, assertion: Any, field_path: str = "") -> None: ... -``` +**Methods:** +- `get_connection(connection_name)` - Get connection settings -## CI/CD Integration +### Functions -### GitHub Actions +#### `generate_token(app_id, app_secret, tenant_id)` +Generate OAuth token for Azure Bot Service. -```yaml -name: Agent Tests +#### `generate_token_from_config(sdk_config)` +Generate token from SDK configuration. -on: [push, pull_request] +#### `assert_field(actual_value, assertion, assertion_type)` +Assert a specific field value. -jobs: - test: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Install dependencies - run: | - pip install -r requirements.txt - pip install microsoft-agents-testing pytest pytest-asyncio - - - name: Run integration tests - run: pytest tests/integration/ -v - env: - CLIENT_ID: ${{ secrets.AGENT_CLIENT_ID }} - CLIENT_SECRET: ${{ secrets.AGENT_CLIENT_SECRET }} - TENANT_ID: ${{ secrets.TENANT_ID }} - - - name: Run data-driven tests - run: pytest tests/data_driven/ -v -``` +#### `assert_model(model, assertion)` +Assert an entire model matches expected structure. -### Azure DevOps +#### `check_field(actual_value, assertion, assertion_type)` +Check field value without asserting. -```yaml -trigger: -- main - -pool: - vmImage: 'ubuntu-latest' - -steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: '3.11' - -- script: | - pip install -r requirements.txt - pip install microsoft-agents-testing pytest pytest-asyncio - displayName: 'Install dependencies' - -- script: | - pytest tests/ -v --junitxml=test-results.xml - displayName: 'Run tests' - env: - CLIENT_ID: $(CLIENT_ID) - CLIENT_SECRET: $(CLIENT_SECRET) - TENANT_ID: $(TENANT_ID) - -- task: PublishTestResults@2 - inputs: - testResultsFiles: 'test-results.xml' - testRunTitle: 'Agent Integration Tests' -``` +#### `check_model(model, assertion)` +Check model without asserting. -## Who Should Use This Package +#### `populate_activity(original, defaults)` +Populate activity with default values. -- **Agent Developers**: Testing agents built with `microsoft-agents-hosting-core` and related packages -- **QA Engineers**: Writing integration, E2E, and regression tests for conversational AI systems -- **DevOps Teams**: Automating agent validation in CI/CD pipelines -- **Sample Authors**: Creating reproducible examples and living documentation -- **Test Engineers**: Building comprehensive test suites with data-driven testing -- **Product Managers**: Writing human-readable test specifications in YAML +#### `get_host_and_port(url)` +Parse host and port from URL. -## Related Packages +#### `ddt(path, prefix="")` +Decorator for data-driven test classes. + +### Enums + +#### `FieldAssertionType` +- `EQUALS` - Exact match +- `CONTAINS` - Contains value +- `EXISTS` - Field exists +- `NOT_EXISTS` - Field does not exist +- `GREATER_THAN` - Greater than value +- `LESS_THAN` - Less than value + +#### `AssertionQuantifier` +- `ALL` - All items must match +- `ONE` - Exactly one item must match +- `NONE` - No items should match + +## Future Goals + +The following features and improvements are planned to enhance the usability and power of the microsoft-agents-testing package: + +### 1. Enhanced Test Recording and Playback +- **Interactive Test Recorder**: Capture live agent interactions and automatically generate YAML test definitions +- **Conversation Replay**: Record entire conversations and replay them for regression testing +- **Smart Diff Tools**: Detect changes between recorded and actual responses with intelligent comparison + +### 2. Advanced Mocking Capabilities +- **External Service Mocking**: Built-in support for mocking external APIs and services that agents depend on +- **Channel Simulators**: More realistic channel-specific behavior simulation (Teams, Slack, etc.) +- **Network Condition Simulation**: Test agents under various network conditions (latency, packet loss) + +### 3. Improved Assertion Framework +- **Visual Assertions**: Assert on rich content like Adaptive Cards with visual diff tools +- **Fuzzy Matching**: Support for approximate string matching and similarity scores +- **Custom Assertion Plugins**: Allow users to define custom assertion types +- **Assertion Templates**: Pre-built assertion patterns for common scenarios + +### 4. Performance and Scalability +- **Distributed Load Testing**: Run benchmarks across multiple machines +- **Real-time Metrics Dashboard**: Live visualization of benchmark results +- **Memory Profiling**: Built-in memory usage tracking during tests +- **Async/Await Optimization**: Better support for free-threaded Python and async workloads + +### 5. Better Developer Experience +- **VS Code Extension**: Integrated test runner and YAML editor with IntelliSense +- **Test Generation Wizard**: Interactive CLI tool to scaffold new test suites +- **Enhanced Error Messages**: More detailed and actionable error messages with suggested fixes +- **Auto-completion**: Schema-based auto-completion for YAML test files + +### 6. CI/CD Integration +- **Test Report Formats**: Support for JUnit XML, TAP, and other standard formats +- **GitHub Actions Integration**: Pre-built actions for running agent tests +- **Azure DevOps Tasks**: Custom pipeline tasks for Azure Pipelines +- **Test Result Analytics**: Track test performance over time with trend analysis + +### 7. Multi-Agent Testing +- **Agent-to-Agent Testing**: Test communication between multiple agents +- **Orchestration Testing**: Test complex multi-agent workflows +- **State Management**: Better support for testing stateful conversations across agents + +### 8. Security Testing +- **Authentication Flow Testing**: Comprehensive OAuth and SSO flow validation +- **Permission Testing**: Verify proper authorization checks +- **Security Scan Integration**: Integrate with security scanning tools +- **PII Detection**: Automatically detect and flag potential PII leaks + +### 9. Documentation and Learning +- **Interactive Tutorial**: Step-by-step guide with executable examples +- **Best Practices Guide**: Comprehensive testing patterns and anti-patterns +- **Video Tutorials**: Video content for common testing scenarios +- **Sample Test Repository**: Curated collection of example tests + +### 10. Advanced DDT Features +- **Parameterized Tests**: Support for test parameters and matrix testing +- **Conditional Execution**: Execute test steps based on conditions +- **Dynamic Test Generation**: Generate tests programmatically from schemas +- **Test Composition**: Compose larger tests from reusable test fragments + +### 11. Telemetry and Observability +- **OpenTelemetry Integration**: Export test traces to observability platforms +- **Test Coverage Metrics**: Track which agent capabilities are tested +- **Flaky Test Detection**: Identify and mark unstable tests +- **Performance Regression Detection**: Automatically detect performance degradation + +### 12. Cross-Platform Support +- **Browser-based Testing**: Test web-based bot interfaces +- **Mobile Emulation**: Test agents in mobile contexts +- **Multi-Language Support**: Better support for testing agents in different languages and locales -This package complements the Microsoft 365 Agents SDK ecosystem: +## Contributing -- **`microsoft-agents-activity`**: Activity types and protocols -- **`microsoft-agents-hosting-core`**: Core hosting framework -- **`microsoft-agents-hosting-aiohttp`**: aiohttp hosting integration -- **`microsoft-agents-hosting-fastapi`**: FastAPI hosting integration -- **`microsoft-agents-hosting-teams`**: Teams-specific hosting features -- **`microsoft-agents-authentication-msal`**: MSAL authentication -- **`microsoft-agents-storage-blob`**: Azure Blob storage for agent state -- **`microsoft-agents-storage-cosmos`**: Azure Cosmos DB storage for agent state +Contributions are welcome! This is an experimental development package designed to improve testing workflows for Microsoft Agents. -## Contributing +### Development Setup -This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit [https://cla.opensource.microsoft.com](https://cla.opensource.microsoft.com). +1. Clone the repository +2. Install in editable mode: + ```bash + pip install -e ./microsoft-agents-testing/ --config-settings editable_mode=compat + ``` +3. Run tests: + ```bash + pytest tests/ + ``` -When you submit a pull request, a CLA bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA. +### Guidelines -This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. +- Follow existing code style and patterns +- Add tests for new features +- Update documentation for API changes +- Use type hints for better IDE support ## License -MIT License - -Copyright (c) Microsoft Corporation. +MIT License - see [LICENSE](LICENSE) file for details. ## Support -For issues, questions, or contributions: -- **GitHub Issues**: [https://github.com/microsoft/Agents-for-python/issues](https://github.com/microsoft/Agents-for-python/issues) -- **Documentation**: [https://github.com/microsoft/Agents-for-python](https://github.com/microsoft/Agents-for-python) -- **Stack Overflow**: Tag your questions with `microsoft-agents-sdk` +For issues, questions, or contributions, please visit the [GitHub repository](https://github.com/microsoft/Agents). -## Changelog +--- -See CHANGELOG.md for version history and release notes. +**Note**: This package is part of the Microsoft Agents SDK development tools and is intended for testing and development purposes. For production agent hosting, use the core Microsoft Agents packages. diff --git a/dev/benchmark/__init__.py b/dev/microsoft-agents-testing/_manual_test/ddt/child.yaml similarity index 100% rename from dev/benchmark/__init__.py rename to dev/microsoft-agents-testing/_manual_test/ddt/child.yaml diff --git a/dev/benchmark/src/__init__.py b/dev/microsoft-agents-testing/_manual_test/ddt/parent.yaml similarity index 100% rename from dev/benchmark/src/__init__.py rename to dev/microsoft-agents-testing/_manual_test/ddt/parent.yaml diff --git a/dev/integration/__init__.py b/dev/microsoft-agents-testing/_manual_test/ddt/standalone.yaml similarity index 100% rename from dev/integration/__init__.py rename to dev/microsoft-agents-testing/_manual_test/ddt/standalone.yaml diff --git a/dev/microsoft-agents-testing/_manual_test/main.py b/dev/microsoft-agents-testing/_manual_test/main.py index 7201dfef..b81d447d 100644 --- a/dev/microsoft-agents-testing/_manual_test/main.py +++ b/dev/microsoft-agents-testing/_manual_test/main.py @@ -1,13 +1,13 @@ import os import asyncio +from dotenv import load_dotenv + from microsoft_agents.testing import ( AiohttpEnvironment, AgentClient, ) -from ..samples import QuickstartSample - -from dotenv import load_dotenv +from .quickstart_sample import QuickstartSample async def main(): diff --git a/dev/benchmark/payload.json b/dev/microsoft-agents-testing/_manual_test/payload.json similarity index 100% rename from dev/benchmark/payload.json rename to dev/microsoft-agents-testing/_manual_test/payload.json diff --git a/dev/integration/samples/quickstart_sample.py b/dev/microsoft-agents-testing/_manual_test/quickstart_sample.py similarity index 100% rename from dev/integration/samples/quickstart_sample.py rename to dev/microsoft-agents-testing/_manual_test/quickstart_sample.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py index d2b52a63..032c7dbd 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py @@ -4,20 +4,20 @@ from .sdk_config import SDKConfig from .assertions import ( - ModelAssertion, - Selector, - AssertionQuantifier, - assert_model, - assert_field, - check_model, - check_model_verbose, - check_field, - check_field_verbose, - FieldAssertionType, + ModelQuery, + DynamicObject, + Assertions, + Fixtures, + SafeObject, + Unset ) -from .auth import generate_token, generate_token_from_config -from .utils import populate_activity, get_host_and_port +from .utils import ( + generate_token, + update_with_defaults, + populate_activity, + get_host_and_port +) from .integration import ( Sample, @@ -27,14 +27,13 @@ ResponseClient, AiohttpEnvironment, Integration, - ddt, - DataDrivenTest, ) +from .cli import cli + __all__ = [ "SDKConfig", "generate_token", - "generate_token_from_config", "Sample", "Environment", "ApplicationRunner", @@ -42,18 +41,14 @@ "ResponseClient", "AiohttpEnvironment", "Integration", + "update_with_defaults", "populate_activity", "get_host_and_port", - "ModelAssertion", - "Selector", - "AssertionQuantifier", - "assert_model", - "assert_field", - "check_model", - "check_model_verbose", - "check_field", - "check_field_verbose", - "FieldAssertionType", - "ddt", - "DataDrivenTest", + "cli", + "ModelQuery", + "DynamicObject", + "SafeObject", + "Assertions", + "Fixtures", + "Unset" ] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py index c51c1f98..3fbbcadc 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py @@ -1,26 +1,17 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -from .model_assertion import ModelAssertion -from .assertions import ( - assert_model, - assert_field, +from .assertions import Assertions +from .fixtures import Fixtures +from .model_query import ModelQuery +from .types import ( + DynamicObject, + SafeObject, + Unset ) -from .check_model import check_model, check_model_verbose -from .check_field import check_field, check_field_verbose -from .type_defs import FieldAssertionType, AssertionQuantifier, UNSET_FIELD -from .model_selector import ModelSelector __all__ = [ - "ModelAssertion", - "assert_model", - "assert_field", - "check_model", - "check_model_verbose", - "check_field", - "check_field_verbose", - "FieldAssertionType", - "ModelSelector", - "AssertionQuantifier", - "UNSET_FIELD", -] + "Assertions", + "Fixtures", + "ModelQuery", + "DynamicObject", + "SafeObject", + "Unset" +] \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertion_context.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertion_context.py new file mode 100644 index 00000000..479f757b --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertion_context.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import inspect +from typing import Callable, Any + +from .types import ( + SafeObject, + resolve, + parent +) + +from .types import DynamicObject + +class AssertionContext: + """Context for assertions, providing access to actual and baseline data, + as well as the current path and additional context information.""" + + def __init__( + self, + actual_source: SafeObject, + baseline_source: Any, + actual: SafeObject | None = None, + baseline: Any | None = None, + context: DynamicObject | None = None, + path: str = "" + ): + """Initialize an AssertionContext. + + :param actual_source: The source of the actual data. + :param baseline_source: The source of the baseline data. + :param actual: The actual data for this context. + :param baseline: The baseline data for this context. + :param context: Additional context information. + :param path: The current path within the data structures. + """ + + self._actual_source = actual_source + if baseline_source is None: + baseline_source = {} + + self._baseline_source = baseline_source + + if actual is None: + actual = actual_source + if baseline is None: + baseline = baseline_source + + self._actual = actual + self._baseline = baseline + + if context is None: + context = DynamicObject({}) + self._context = context + + self._path = path + + def next(self, key: Any) -> AssertionContext: + """Create a new AssertionContext for the next level in the data structure. + + :param key: The key for the next level. + :return: A new AssertionContext for the next level. + """ + next_path = f"{self._path}.{key}" if self._path else str(key) + assert self._baseline is not None + return AssertionContext( + self._actual_source, + self._baseline_source, + self._actual[key], + self._baseline[key], + self._context, + next_path + ) + + def resolve_args(self, query_function: Callable) -> Callable: + """Resolve the arguments for a query function based on the current context.\ + + :param query_function: The query function to resolve arguments for. + :return: A callable with the resolved arguments. + """ + sig = inspect.getfullargspec(query_function) + args = {} + + args_map = { + "actual": DynamicObject(self._actual_source), + "path": self._path, + "value": self._actual, + "parent": parent(self._actual), + "context": self._context, + "ctx": self._context + } + + for arg in sig.args: + if arg in args_map: + args[arg] = args_map[arg] + else: + raise RuntimeError(f"Unknown argument '{arg}' in query function") + + output_func = query_function(**args) + output_func.__name__ = query_function.__name__ + return output_func diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py index 04955fcd..ff0e117b 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py @@ -1,35 +1,143 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. +from __future__ import annotations -from typing import Any +from typing import Any, Callable -from microsoft_agents.activity import AgentsModel +from pydantic import BaseModel -from .type_defs import FieldAssertionType -from .check_model import check_model_verbose -from .check_field import check_field_verbose +from .types import SafeObject, DynamicObject, resolve, parent +from .assertion_context import AssertionContext +class Assertions: -def assert_field( - actual_value: Any, assertion: Any, assertion_type: FieldAssertionType -) -> None: - """Asserts that a specific field in the target matches the baseline. + _EVAL_META_FIELD = "__call" - :param key_in_baseline: The key of the field to be tested. - :param target: The target dictionary containing the actual values. - :param assertion: The baseline dictionary containing the expected values. - """ - res, assertion_error_message = check_field_verbose( - actual_value, assertion, assertion_type - ) - assert res, assertion_error_message + @staticmethod + def expand(data: dict) -> dict: + """Expand a flattened dictionary into a nested dictionary. + + :param data: The flattened dictionary to expand. + :return: The expanded nested dictionary. + """ + if not isinstance(data, dict): + return data -def assert_model(model: AgentsModel | dict, assertion: AgentsModel | dict) -> None: - """Asserts that the given model matches the baseline model. + new_data = {} - :param model: The model to be tested. - :param assertion: The baseline model or a dictionary representing the expected model data. - """ - res, assertion_error_data = check_model_verbose(model, assertion) - assert res, str(assertion_error_data) + # flatten + for key, value in data.items(): + if "." in key: + index = key.index(".") + root = key[:index] + path = key[index + 1 :] + + if root in new_data and path in new_data[root]: + raise RuntimeError() + elif root in new_data and not isinstance(new_data[root], (dict, list)): + raise RuntimeError() + + if root not in new_data: + new_data[root] = {} + + new_data[root][path] = value + + else: + root = key + if root in new_data: + raise RuntimeError() + + new_data[root] = value + + # expand + for key, value in new_data.items(): + new_data[key] = Assertions.expand(value) + + return new_data + + @staticmethod + def invoke( + actual: SafeObject[Any], + query_function: Callable, + context: AssertionContext + ) -> tuple[bool, str]: + """Invoke a query function with resolved arguments. + + :param actual: The actual data to pass to the query function. + :param query_function: The query function to invoke. + :param context: The current assertion context. + :return: A tuple containing the result of the query function and a message. + """ + + res = context.resolve_args(query_function)() + + if isinstance(res, tuple) and len(res) == 2: + return res + else: + return bool(res), f"Assertion failed for query function: '{query_function.__name__}'" + + @staticmethod + def _check_verbose(actual: SafeObject[Any], baseline: Any, context: AssertionContext) -> tuple[bool, str]: + """Recursively check the actual data against the baseline data with verbose output. + + :param actual: The actual data to check. + :param baseline: The baseline data to check against. + :param context: The current assertion context. + :return: A tuple containing the overall result and a detailed message. + """ + + results = [] + + if isinstance(baseline, dict): + for key, value in baseline.items(): + check, msg = Assertions._check_verbose(actual[key], value, context.next(key)) + results.append((check, msg)) + elif isinstance(baseline, list): + for i, value in enumerate(baseline): + check, msg = Assertions._check_verbose(actual[i], value, context.next(i)) + results.append((check, msg)) + elif callable(baseline): + results.append(Assertions.invoke(actual, baseline, context)) + else: + check = resolve(actual) == baseline + msg = f"Values do not match: {actual} != {baseline}" if not check else "" + results.append((check, msg)) + + return (all(check for check, msg in results), "\n".join(msg for check, msg in results if not check)) + + @staticmethod + def check_verbose(actual: Any, baseline: Any) -> tuple[bool, str]: + """Check the actual data against the baseline data with verbose output. + + :param actual: The actual data to check. + :param baseline: The baseline data to check against. + :return: A tuple containing the overall result and a detailed message. + """ + if isinstance(actual, BaseModel): + actual = actual.model_dump(exclude_unset=True) + if isinstance(baseline, BaseModel): + baseline = baseline.model_dump(exclude_unset=True) + + + actual = SafeObject(actual) + context = AssertionContext(actual, baseline) + return Assertions._check_verbose(actual, baseline, context) + + @staticmethod + def check(actual: Any, baseline: Any) -> bool: + """Check the actual data against the baseline data. + + :param actual: The actual data to check. + :param baseline: The baseline data to check against. + :return: True if the actual data matches the baseline data, False otherwise. + """ + return Assertions.check_verbose(actual, baseline)[0] + + @staticmethod + def validate(actual: Any, baseline: Any) -> None: + """Validate the actual data against the baseline data, raising an assertion error if they do not match. + + :param actual: The actual data to validate. + :param baseline: The baseline data to validate against." + """ + check, msg = Assertions.check_verbose(actual, baseline) + assert check, msg \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py deleted file mode 100644 index 6693f706..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import re -from typing import Any, Optional - -from .type_defs import FieldAssertionType, UNSET_FIELD - - -_OPERATIONS = { - FieldAssertionType.EQUALS: lambda a, b: a == b or (a is UNSET_FIELD and b is None), - FieldAssertionType.NOT_EQUALS: lambda a, b: a != b - or (a is UNSET_FIELD and b is not None), - FieldAssertionType.GREATER_THAN: lambda a, b: a > b, - FieldAssertionType.LESS_THAN: lambda a, b: a < b, - FieldAssertionType.CONTAINS: lambda a, b: b in a, - FieldAssertionType.NOT_CONTAINS: lambda a, b: b not in a, - FieldAssertionType.RE_MATCH: lambda a, b: re.match(b, a) is not None, -} - - -def _parse_assertion(field: Any) -> tuple[Any, Optional[FieldAssertionType]]: - """Parses the assertion information and returns the assertion type and baseline value. - - :param assertion_info: The assertion information to be parsed. - :return: A tuple containing the assertion type and baseline value. - """ - - assertion_type = FieldAssertionType.EQUALS - assertion = None - - if ( - isinstance(field, dict) - and "assertion_type" in field - and "assertion" in field - and field["assertion_type"] in FieldAssertionType.__members__ - ): - # format: - # {"assertion_type": "__EQ__", "assertion": "value"} - assertion_type = FieldAssertionType[field["assertion_type"]] - assertion = field.get("assertion") - - elif ( - isinstance(field, list) - and len(field) >= 2 - and isinstance(field[0], str) - and field[0] in FieldAssertionType.__members__ - ): - # format: - # ["__EQ__", "assertion"] - assertion_type = FieldAssertionType[field[0]] - assertion = field[1] - elif isinstance(field, list) or isinstance(field, dict): - assertion_type = None - else: - # default format: direct value - assertion = field - - return assertion, assertion_type - - -def check_field( - actual_value: Any, assertion: Any, assertion_type: FieldAssertionType -) -> bool: - """Checks if the actual value satisfies the given assertion based on the assertion type. - - :param actual_value: The value to be checked. - :param assertion: The expected value or pattern to check against. - :param assertion_type: The type of assertion to perform. - :return: True if the assertion is satisfied, False otherwise. - """ - - operation = _OPERATIONS.get(assertion_type) - if not operation: - raise ValueError(f"Unsupported assertion type: {assertion_type}") - return operation(actual_value, assertion) - - -def check_field_verbose( - actual_value: Any, assertion: Any, assertion_type: FieldAssertionType -) -> tuple[bool, Optional[str]]: - """Checks if the actual value satisfies the given assertion based on the assertion type. - - :param actual_value: The value to be checked. - :param assertion: The expected value or pattern to check against. - :param assertion_type: The type of assertion to perform. - :return: A tuple containing a boolean indicating if the assertion is satisfied and an optional error message. - """ - - operation = _OPERATIONS.get(assertion_type) - if not operation: - raise ValueError(f"Unsupported assertion type: {assertion_type}") - - result = operation(actual_value, assertion) - if result: - return True, None - else: - return ( - False, - f"Assertion failed: actual value '{actual_value}' does not satisfy '{assertion_type.name}' with assertion '{assertion}'", - ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_model.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_model.py deleted file mode 100644 index e88564be..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_model.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -from typing import Any, Optional - -from microsoft_agents.activity import AgentsModel -from microsoft_agents.testing.utils import normalize_model_data - -from .check_field import check_field, _parse_assertion -from .type_defs import UNSET_FIELD, FieldAssertionType, AssertionErrorData - - -def _check( - actual: Any, baseline: Any, field_path: str = "" -) -> tuple[bool, Optional[AssertionErrorData]]: - """Recursively checks the actual data against the baseline data. - - :param actual: The actual data to be tested. - :param baseline: The baseline data to compare against. - :param field_path: The current field path being checked (for error reporting). - :return: A tuple containing a boolean indicating success and optional assertion error data. - """ - - assertion, assertion_type = _parse_assertion(baseline) - - if assertion_type is None: - if isinstance(baseline, dict): - for key in baseline: - new_field_path = f"{field_path}.{key}" if field_path else key - new_actual = actual.get(key, UNSET_FIELD) - new_baseline = baseline[key] - - res, assertion_error_data = _check( - new_actual, new_baseline, new_field_path - ) - if not res: - return False, assertion_error_data - return True, None - - elif isinstance(baseline, list): - for index, item in enumerate(baseline): - new_field_path = ( - f"{field_path}[{index}]" if field_path else f"[{index}]" - ) - new_actual = actual[index] if index < len(actual) else UNSET_FIELD - new_baseline = item - - res, assertion_error_data = _check( - new_actual, new_baseline, new_field_path - ) - if not res: - return False, assertion_error_data - return True, None - else: - raise ValueError("Unsupported baseline type for complex assertion.") - else: - assert isinstance(assertion_type, FieldAssertionType) - res = check_field(actual, assertion, assertion_type) - if res: - return True, None - else: - assertion_error_data = AssertionErrorData( - field_path=field_path, - actual_value=actual, - assertion=assertion, - assertion_type=assertion_type, - ) - return False, assertion_error_data - - -def check_model(actual: dict | AgentsModel, baseline: dict | AgentsModel) -> bool: - """Asserts that the given activity matches the baseline activity. - - :param activity: The activity to be tested. - :param baseline: The baseline activity or a dictionary representing the expected activity data. - """ - return check_model_verbose(actual, baseline)[0] - - -def check_model_verbose( - actual: dict | AgentsModel, baseline: dict | AgentsModel -) -> tuple[bool, Optional[AssertionErrorData]]: - """Asserts that the given activity matches the baseline activity. - - :param actual: The actual data to be tested. - :param baseline: The baseline data or a dictionary representing the expected data. - """ - actual = normalize_model_data(actual) - baseline = normalize_model_data(baseline) - return _check(actual, baseline, "model") diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/fixtures.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/fixtures.py new file mode 100644 index 00000000..202f5d32 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/fixtures.py @@ -0,0 +1,22 @@ +from .types import Unset, FutureVar + +class Fixtures: + + actual = FutureVar("actual") + expected = FutureVar("expected") + + @staticmethod + def exists(actual): + return actual is not Unset + + @staticmethod + def not_exists(actual): + return actual is Unset + + @staticmethod + def first(actual): + return actual[0] + + @staticmethod + def last(actual): + return actual[-1] \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/model_assertion.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/model_assertion.py deleted file mode 100644 index f01abdae..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/model_assertion.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -from __future__ import annotations - -from typing import Optional - -from microsoft_agents.activity import AgentsModel - -from .check_model import check_model_verbose -from .model_selector import ModelSelector -from .type_defs import AssertionQuantifier, AssertionErrorData - - -class ModelAssertion: - """Class for asserting activities based on a selector and assertion criteria.""" - - _selector: ModelSelector - _quantifier: AssertionQuantifier - _assertion: dict | AgentsModel - - def __init__( - self, - assertion: dict | None = None, - selector: ModelSelector | None = None, - quantifier: AssertionQuantifier = AssertionQuantifier.ALL, - ) -> None: - """Initializes the ModelAssertion with the given configuration. - - :param config: The configuration dictionary containing quantifier, selector, and assertion. - """ - - self._assertion = assertion or {} - self._selector = selector or ModelSelector() - self._quantifier = quantifier - - @staticmethod - def _combine_assertion_errors(errors: list[AssertionErrorData]) -> str: - """Combines multiple assertion errors into a single string representation. - - :param errors: The list of assertion errors to be combined. - :return: A string representation of the combined assertion errors. - """ - return "\n".join(str(error) for error in errors) - - def check(self, items: list[dict]) -> tuple[bool, Optional[str]]: - """Asserts that the given items match the assertion criteria. - - :param items: The list of items to be tested. - :return: A tuple containing a boolean indicating if the assertion passed and an optional error message. - """ - - items = self._selector(items) - - count = 0 - for item in items: - res, assertion_error_data = check_model_verbose(item, self._assertion) - if self._quantifier == AssertionQuantifier.ALL and not res: - return ( - False, - f"Item did not match the assertion: {item}\nError: {assertion_error_data}", - ) - if self._quantifier == AssertionQuantifier.NONE and res: - return ( - False, - f"Item matched the assertion when none were expected: {item}", - ) - if res: - count += 1 - - passes = True - if self._quantifier == AssertionQuantifier.ONE and count != 1: - return ( - False, - f"Expected exactly one item to match the assertion, but found {count}.", - ) - - return passes, None - - def __call__(self, items: list[dict]) -> None: - """Allows the ModelAssertion instance to be called directly. - - :param items: The list of items to be tested. - :return: A tuple containing a boolean indicating if the assertion passed and an optional error message. - """ - passes, error = self.check(items) - assert passes, error - - @staticmethod - def from_config(config: dict) -> ModelAssertion: - """Creates a ModelAssertion instance from a configuration dictionary. - - :param config: The configuration dictionary containing quantifier, selector, and assertion. - :return: A ModelAssertion instance. - """ - assertion = config.get("assertion", {}) - selector = ModelSelector.from_config(config.get("selector", {})) - quantifier = AssertionQuantifier.from_config(config.get("quantifier", "all")) - - return ModelAssertion( - assertion=assertion, - selector=selector, - quantifier=quantifier, - ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/model_query.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/model_query.py new file mode 100644 index 00000000..c2e2e29d --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/model_query.py @@ -0,0 +1,73 @@ +from typing import Protocol, TypeVar, overload, Iterable, Callable +from pydantic import BaseModel + +from .assertions import Assertions + +BaseModelT = TypeVar("BaseModelT", bound=BaseModel) + +def create_base(model: dict | BaseModel | Callable | None = None, **kwargs) -> dict: + if model is None: + return {**kwargs} + elif isinstance(model, dict): + return dict(Assertions.expand(model), **kwargs) + elif isinstance(model, BaseModel): + return { + **model.model_dump(exclude_unset=True), + **kwargs + } + elif isinstance(model, Callable): + return { + Assertions._EVAL_META_FIELD: model, + **kwargs + } + else: + raise TypeError("model must be a dict, BaseModel, or Callable") + +class ModelQuery: + + def __init__(self, _query: dict | BaseModel | Callable | None = None, **kwargs): + self._query = create_base(_query, **kwargs) + + def check(self, actual: dict | BaseModel) -> bool: + return Assertions.check(actual, self._query) + + def __call__(self, actual: dict | BaseModel) -> bool: + return self.check(actual) + + def check_verbose(self, actual: dict | BaseModel) -> tuple[bool, str]: + return Assertions.check_verbose(actual, self._query) + + def validate(self, actual: dict | BaseModel) -> None: + Assertions.validate(actual, self._query) + + @overload + def select(self, lst: list[dict]) -> list[dict]: ... + @overload + def select(self, lst: list[BaseModelT]) -> list[BaseModelT]: ... + def select(self, lst: list[dict] | list[BaseModelT]) -> list[dict] | list[BaseModelT]: + res = [] + for item in lst: + if self.check(item): + res.append(item) + return res + + @overload + def first(self, lst: Iterable[dict]) -> dict | None: ... + @overload + def first(self, lst: Iterable[BaseModelT]) -> BaseModelT | None: ... + def first(self, lst: Iterable[dict] | Iterable[BaseModelT]) -> dict | BaseModelT | None: + for item in lst: + if self.check(item): + return item + return None + + @overload + def last(self, lst: Iterable[dict]) -> dict | None: ... + @overload + def last(self, lst: Iterable[BaseModelT]) -> BaseModelT | None: ... + def last(self, lst: Iterable[dict] | Iterable[BaseModelT]) -> dict | BaseModelT | None: + last = None + for item in lst: + if self.check(item): + last = item + return last \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/model_selector.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/model_selector.py deleted file mode 100644 index 5a2c3dca..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/model_selector.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -from __future__ import annotations - -from .check_model import check_model - - -class ModelSelector: - """Class for selecting activities based on a model and an index.""" - - _model: dict - _index: int | None - - def __init__( - self, - model: dict | None = None, - index: int | None = None, - ) -> None: - """Initializes the ModelSelector with the given configuration. - - :param model: The model to use for selecting activities. - The model is an object holding the fields to match and assertions to pass. - :param index: The index of the item to select when quantifier is ONE. - """ - - if model is None: - model = {} - - self._model = model - self._index = index - - def select_first(self, items: list[dict]) -> dict | None: - """Selects the first item from the list of items. - - :param items: The list of items to select from. - :return: The first item, or None if no items exist. - """ - res = self.select(items) - if res: - return res[0] - return None - - def select(self, items: list[dict]) -> list[dict]: - """Selects items based on the selector configuration. - - :param items: The list of items to select from. - :return: A list of selected items. - """ - if self._index is None: - return list( - filter( - lambda item: check_model(item, self._model), - items, - ) - ) - else: - filtered_list = [] - for item in items: - if check_model(item, self._model): - filtered_list.append(item) - - if self._index < 0 and abs(self._index) <= len(filtered_list): - return [filtered_list[self._index]] - elif self._index >= 0 and self._index < len(filtered_list): - return [filtered_list[self._index]] - else: - return [] - - def __call__(self, items: list[dict]) -> list[dict]: - """Allows the Selector instance to be called as a function. - - :param items: The list of items to select from. - :return: A list of selected items. - """ - return self.select(items) - - @staticmethod - def from_config(config: dict) -> ModelSelector: - """Creates a ModelSelector instance from a configuration dictionary. - - :param config: The configuration dictionary containing selector, and index. - :return: A Selector instance. - """ - model = config.get("model", {}) - index = config.get("index", None) - - return ModelSelector( - model=model, - index=index, - ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py deleted file mode 100644 index 97c4be49..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -from __future__ import annotations - -from enum import Enum -from dataclasses import dataclass -from typing import Any - - -class UNSET_FIELD: - """Singleton to represent an unset field in activity comparisons.""" - - @staticmethod - def get(*args, **kwargs): - """Returns the singleton instance.""" - return UNSET_FIELD - - -class FieldAssertionType(str, Enum): - """Defines the types of assertions that can be made on fields.""" - - EQUALS = "EQUALS" - NOT_EQUALS = "NOT_EQUALS" - GREATER_THAN = "GREATER_THAN" - LESS_THAN = "LESS_THAN" - CONTAINS = "CONTAINS" - NOT_CONTAINS = "NOT_CONTAINS" - IN = "IN" - NOT_IN = "NOT_IN" - RE_MATCH = "RE_MATCH" - - -class AssertionQuantifier(str, Enum): - """Defines quantifiers for assertions on activities.""" - - ANY = "ANY" - ALL = "ALL" - ONE = "ONE" - NONE = "NONE" - - @staticmethod - def from_config(value: str) -> AssertionQuantifier: - """Creates an AssertionQuantifier from a configuration string. - - :param value: The configuration string. - :return: The corresponding AssertionQuantifier. - """ - value = value.upper() - if value not in AssertionQuantifier: - raise ValueError(f"Invalid AssertionQuantifier value: {value}") - return AssertionQuantifier(value) - - -@dataclass -class AssertionErrorData: - """Data class to hold information about assertion errors.""" - - field_path: str - actual_value: Any - assertion: Any - assertion_type: FieldAssertionType - - def __str__(self) -> str: - return ( - f"Assertion failed at '{self.field_path}': " - f"actual value '{self.actual_value}' " - f"does not satisfy assertion '{self.assertion}' " - f"of type '{self.assertion_type}'." - ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/__init__.py new file mode 100644 index 00000000..54be570e --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/__init__.py @@ -0,0 +1,13 @@ +from .dynamic_object import DynamicObject +from .future_var import FutureVar +from .safe_object import SafeObject, resolve, parent +from .unset import Unset + +__all__ = [ + "DynamicObject", + "SafeObject", + "FutureVar", + "Unset", + "resolve", + "parent" +] \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/_readonly.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/_readonly.py new file mode 100644 index 00000000..147a74be --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/_readonly.py @@ -0,0 +1,18 @@ +class _Readonly: + """A mixin class that makes all attributes of a class readonly.""" + + def __setattr__(self, name, value): + """Prevent setting attributes on the readonly object.""" + raise AttributeError(f"Cannot set attribute '{name}' on {type(self).__name__}") + + def __delattr__(self, name): + """Prevent deleting attributes on the readonly object.""" + raise AttributeError(f"Cannot delete attribute '{name}' on {type(self).__name__}") + + def __setitem__(self, key, value): + """Prevent setting items on the readonly object.""" + raise AttributeError(f"Cannot set item '{key}' on {type(self).__name__}") + + def __delitem__(self, key): + """Prevent deleting items on the readonly object.""" + raise AttributeError(f"Cannot delete item '{key}' on {type(self).__name__}") \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/dynamic_object.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/dynamic_object.py new file mode 100644 index 00000000..8be21f11 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/dynamic_object.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from typing import Any, TypeVar, Sized + +from .safe_object import SafeObject, resolve, parent +from .unset import Unset + +T = TypeVar("T") + +PRIMITIVE_TYPES = (int, float, str, bool) +PRIMITIVES = (None, Unset) + +class DynamicObject(SafeObject[T]): + """A wrapper around an object that provides dynamic access to its attributes + and items, while maintaining a reference to its parent object.""" + + def __init__(self, value: Any, parent_object: SafeObject | None = None): + """Initialize a SafeObject with a value and an optional parent SafeObject. + + :param value: The value to wrap. + :param parent: The parent SafeObject, if any. + """ + + object.__setattr__(self, "__value__", value) + if parent_object is not None: + parent_value = resolve(parent_object) + if parent_value is Unset or parent_value is None: + parent_object = None + else: + parent_object = None + object.__setattr__(self, "__parent__", parent_object) + + def __new__(cls, value: Any, parent_object: SafeObject | None = None) -> Any: + """Create a new DynamicObject or return the value directly if it's a primitive type.""" + if isinstance(value, PRIMITIVE_TYPES): + return value + elif value in PRIMITIVES: + return value + elif isinstance(value, SafeObject) and not isinstance(value, DynamicObject): + resolved_value = resolve(value) + parent_object = parent(value) + return cls.__new__(cls, resolved_value, parent_object) + return super().__new__(cls, value, parent_object) + + + def __contains__(self, key): + """Check if the wrapped object contains the given key.""" + value = resolve(self) + if hasattr(value, "__contains__"): + return key in value + raise TypeError(f"{type(value)} object is not iterable") + + def __in__(self, other) -> bool: + """Check if the wrapped object is in another object.""" + value = resolve(self) + other_value = other + if isinstance(other, SafeObject): + other_value = resolve(other) + return value in other_value + + def __bool__(self) -> bool: + """Get the boolean value of the wrapped object.""" + return bool(resolve(self)) + + def __len__(self) -> int: + """Get the length of the wrapped object.""" + value = resolve(self) + if isinstance(value, Sized): + return len(value) + raise TypeError(f"{type(value)} object has no length") \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/future_var.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/future_var.py new file mode 100644 index 00000000..5e3a8a0c --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/future_var.py @@ -0,0 +1,34 @@ +from typing import Callable + +class FutureVar: + """A class representing a future variable for deferred evaluation in assertions.""" + + def __init__(self, name: str): + self.name = name + + def __eq__(self, other) -> Callable[..., bool]: + return lambda ctx: ctx.get(self.name) == other + + def __ne__(self, other) -> Callable[..., bool]: + return lambda ctx: ctx.get(self.name) != other + + def __lt__(self, other) -> Callable[..., bool]: + return lambda ctx: ctx.get(self.name) < other + + def __le__(self, other) -> Callable[..., bool]: + return lambda ctx: ctx.get(self.name) <= other + + def __gt__(self, other) -> Callable[..., bool]: + return lambda ctx: ctx.get(self.name) > other + + def __ge__(self, other) -> Callable[..., bool]: + return lambda ctx: ctx.get(self.name) >= other + + def __contains__(self, item) -> Callable[..., bool]: + return lambda ctx: item in ctx.get(self.name, "") + + def __str__(self) -> str: + return f"FutureVar(name={self.name})" + + def __repr__(self) -> str: + return self.__str__() \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/safe_object.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/safe_object.py new file mode 100644 index 00000000..a9f90226 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/safe_object.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +from typing import Any, Generic, TypeVar, overload, cast + +from ._readonly import _Readonly +from .unset import Unset + +T = TypeVar("T") +P = TypeVar("P") + +@overload +def resolve(obj: SafeObject[T]) -> T: ... +@overload +def resolve(obj: P) -> P: ... +def resolve(obj: SafeObject[T] | P) -> T | P: + """Resolve the value of a SafeObject or return the object itself if it's not a SafeObject.""" + if isinstance(obj, SafeObject): + return object.__getattribute__(obj, "__value__") + return obj + +def parent(obj: SafeObject[T]) -> SafeObject | None: + """Get the parent SafeObject of the given SafeObject, or None if there is no parent.""" + return object.__getattribute__(obj, "__parent__") + +class SafeObject(Generic[T], _Readonly): + """A wrapper around an object that provides safe access to its attributes + and items, while maintaining a reference to its parent object.""" + + def __init__(self, value: Any, parent_object: SafeObject | None = None): + """Initialize a SafeObject with a value and an optional parent SafeObject. + + :param value: The value to wrap. + :param parent: The parent SafeObject, if any. + """ + + if isinstance(value, SafeObject): + return + + object.__setattr__(self, "__value__", value) + if parent_object is not None: + parent_value = resolve(parent_object) + if parent_value is Unset or parent_value is None: + parent_object = None + else: + parent_object = None + object.__setattr__(self, "__parent__", parent_object) + + + def __new__(cls, value: Any, parent_object: SafeObject | None = None): + """Create a new SafeObject or return the value directly if it's already a SafeObject. + + :param value: The value to wrap. + :param parent: The parent SafeObject, if any. + + :return: A SafeObject instance or the original value. + """ + # breakpoint()f + if isinstance(value, SafeObject): + return value + return super().__new__(cls) + + def __getattr__(self, name: str) -> Any: + """Get an attribute of the wrapped object safely. + + :param name: The name of the attribute to access. + :return: The attribute value wrapped in a SafeObject. + """ + # breakpoint() + + value = resolve(self) + cls = object.__getattribute__(self, "__class__") + if isinstance(value, dict): + return cls(value.get(name, Unset), self) + attr = getattr(value, name, Unset) + return cls(attr, self) + + def __getitem__(self, key) -> Any: + """Get an item of the wrapped object safely. + + :param key: The key or index of the item to access. + :return: The item value wrapped in a SafeObject. + """ + # breakpoint() + + value = resolve(self) + value = cast(dict, value) + if isinstance(value, list): + cls = object.__getattribute__(self, "__class__") + return cls(value[key], self) + return type(self)(value.get(key, Unset), self) + + def __str__(self) -> str: + """Get the string representation of the wrapped object.""" + # breakpoint() + return str(resolve(self)) + + def __repr__(self) -> str: + """Get the detailed string representation of the SafeObject.""" + value = resolve(self) + # breakpoint() + cls = object.__getattribute__(self, "__class__") + return f"{cls.__name__}({value!r})" + + def __eq__(self, other) -> bool: + """Check if the wrapped object is equal to another object.""" + value = resolve(self) + other_value = other + if isinstance(other, SafeObject): + other_value = resolve(other) + return value == other_value \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/unset.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/unset.py new file mode 100644 index 00000000..c676869a --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/types/unset.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from ._readonly import _Readonly + +class _Unset(_Readonly): + """A class representing an unset value.""" + + def get(self, *args, **kwargs): + """Returns the singleton instance when accessed as a method.""" + return self + + def __getattr__(self, name, *args, **kwargs): + """Returns the singleton instance when accessed as an attribute.""" + return self + + def __getitem__(self, key, *args, **kwargs): + """Returns the singleton instance when accessed as an item.""" + return self + + def __bool__(self): + """Returns False when converted to a boolean.""" + return False + + def __repr__(self): + """Returns 'Unset' when represented.""" + return "Unset" + + def __str__(self): + """Returns 'Unset' when converted to a string.""" + return repr(self) + +Unset = _Unset() \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py deleted file mode 100644 index 80bb0402..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -from .generate_token import generate_token, generate_token_from_config - -__all__ = ["generate_token", "generate_token_from_config"] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/cli/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/__init__.py new file mode 100644 index 00000000..c7cb19c1 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/__init__.py @@ -0,0 +1,3 @@ +from .cli import cli + +__all__ = ["cli"] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/cli/cli.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/cli.py new file mode 100644 index 00000000..77f21659 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/cli.py @@ -0,0 +1,41 @@ +from pathlib import Path + +import click +from dotenv import load_dotenv + +from microsoft_agents.testing.utils import resolve_env + +from .cli_config import cli_config +from .commands import COMMAND_LIST + +@click.group() +@click.option("--env_path", default=".env", help="Environment file path") +@click.option("--connection_name", default=None, help="Connection name") +@click.pass_context +def cli(ctx, env_path, connection_name): + """A simple CLI tool for managing tasks.""" + + click.echo("-"*80) + click.echo("Welcome to the CLI for the microsoft-agents-testing package for Python.") + + ctx.ensure_object(dict) + + env_path = Path(env_path) + + if not env_path.exists(): + raise FileNotFoundError(f"Environment file not found at: {env_path.absolute()}") + + + env_path = str(env_path.resolve()) + load_dotenv(env_path, override=True) + click.echo("\tUsing environment file at: " + env_path) + click.echo() + + ctx.obj["env_path"] = env_path + + cli_config.load_from_config(connection_name) + + + +for command in COMMAND_LIST: + cli.add_command(command) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/cli/cli_config.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/cli_config.py new file mode 100644 index 00000000..4b908943 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/cli_config.py @@ -0,0 +1,83 @@ +import os +from dataclasses import dataclass + +_UNSET = object() + +def add_trailing_slash(url: str) -> str: + """Add a trailing slash to the URL if it doesn't already have one.""" + if not url.endswith("/"): + url += "/" + return url + +@dataclass +class _CLIConfig: + """Configuration class for benchmark settings.""" + + tenant_id: str = "" + app_id: str = "" + app_secret: str = "" + _agent_url: str = "http://localhost:3978/" + _service_url: str = "http://localhost:8001/" + + @property + def service_url(self) -> str: + """Return the service URL""" + return self._service_url + + @service_url.setter + def service_url(self, value: str) -> None: + """Set the service URL""" + self._service_url = add_trailing_slash(value) + + @property + def agent_url(self) -> str: + """Return the agent URL""" + return self._agent_url + + @agent_url.setter + def agent_url(self, value: str) -> None: + """Set the agent URL""" + self._agent_url = add_trailing_slash(value) + + @property + def agent_endpoint(self) -> str: + """Return the agent messaging endpoint""" + return f"{self.agent_url}api/messages/" + + def load_from_config(self, config: dict | None = None) -> None: + """Load configuration from a dictionary""" + + config = config or dict(os.environ) + config = {key.upper(): value for key, value in config.items()} + + self.tenant_id = config.get("TENANT_ID", self.tenant_id) + self.app_id = config.get("APP_ID", self.app_id) + self.app_secret = config.get("APP_SECRET", self.app_secret) + self.agent_url = config.get("AGENT_URL", self.agent_url) + + def load_from_connection( + self, connection_name: str = "SERVICE_CONNECTION", config: dict | None = None + ) -> None: + """Load configuration from a connection dictionary.""" + + config = config or dict(os.environ) + + config = { + "app_id": os.environ.get( + f"CONNECTIONS__{connection_name}__SETTINGS__CLIENTID", _UNSET + ), + "app_secret": os.environ.get( + f"CONNECTIONS__{connection_name}__SETTINGS__CLIENTSECRET", _UNSET + ), + "tenant_id": os.environ.get( + f"CONNECTIONS__{connection_name}__SETTINGS__TENANTID", _UNSET + ), + } + + config = {key: value for key, value in config.items() if value is not _UNSET} + + self.load_from_config(config) + + +cli_config = _CLIConfig() +cli_config.load_from_config() diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/__init__.py new file mode 100644 index 00000000..d32741d7 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/__init__.py @@ -0,0 +1,15 @@ +from click import Command + +from .benchmark import benchmark +from .post import post +from .auth import auth + +COMMAND_LIST: list[Command] = [ + benchmark, + post, + auth, +] + +__all__ = [ + "COMMAND_LIST", +] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/auth/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/auth/__init__.py new file mode 100644 index 00000000..6d7318b9 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/auth/__init__.py @@ -0,0 +1,3 @@ +from .auth import auth + +__all__ = ["auth"] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/auth/auth.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/auth/auth.py new file mode 100644 index 00000000..6e1c827d --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/auth/auth.py @@ -0,0 +1,28 @@ +import asyncio +import click + +from microsoft_agents.testing.integration import AiohttpEnvironment + +from .auth_sample import AuthSample + +async def _auth(port: int): + # Initialize the environment + environment = AiohttpEnvironment() + config = await AuthSample.get_config() + await environment.init_env(config) + + sample = AuthSample(environment) + await sample.init_app() + + host = "localhost" + async with environment.create_runner(host, port): + click.echo(f"\nServer running at http://{host}:{port}/api/messages\n") + while True: + await asyncio.sleep(10) + + +@click.command() +@click.option("--port", type=int, default=3978, help="Port to run the bot on.") +def auth(port: int): + """Run the authentication testing sample from a configuration file.""" + asyncio.run(_auth(port)) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/auth/auth_sample.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/auth/auth_sample.py new file mode 100644 index 00000000..0b18490b --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/auth/auth_sample.py @@ -0,0 +1,49 @@ +import os +import click + +from microsoft_agents.activity import ActivityTypes + +from microsoft_agents.hosting.core import AgentApplication, TurnContext, TurnState + +from microsoft_agents.testing.integration import Sample + + +def create_auth_route(auth_handler_id: str, agent: AgentApplication): + """Create a dynamic function to handle authentication routes.""" + + async def dynamic_function(context: TurnContext, state: TurnState): + token = await agent.auth.get_token(context, auth_handler_id) + await context.send_activity(f"Hello from {auth_handler_id}! Token: {token}") + + dynamic_function.__name__ = f"auth_route_{auth_handler_id}".lower() + click.echo(f"Creating route: {dynamic_function.__name__} for handler {auth_handler_id}") + return dynamic_function + + +class AuthSample(Sample): + """A quickstart sample implementation.""" + + @classmethod + async def get_config(cls) -> dict: + """Retrieve the configuration for the sample.""" + return dict(os.environ) + + async def init_app(self): + """Initialize the application for the quickstart sample.""" + + app: AgentApplication[TurnState] = self.env.agent_application + + assert app._auth + assert app._auth._handlers + + for authorization_handler in app._auth._handlers.values(): + auth_handler = authorization_handler._handler + app.message( + auth_handler.name.lower(), + auth_handlers=[auth_handler.name], + )(create_auth_route(auth_handler.name, app)) + + async def handle_message(context: TurnContext, state: TurnState): + await context.send_activity("Hello from the auth testing sample! Enter the name of an auth handler to test it.") + + app.activity(ActivityTypes.message)(handle_message) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/benchmark/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/benchmark/__init__.py new file mode 100644 index 00000000..c0a77364 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/benchmark/__init__.py @@ -0,0 +1,5 @@ +from .benchmark import benchmark + +__all__ = [ + "benchmark", +] diff --git a/dev/benchmark/src/aggregated_results.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/benchmark/aggregated_results.py similarity index 96% rename from dev/benchmark/src/aggregated_results.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/benchmark/aggregated_results.py index b1edaa5e..d3609d6c 100644 --- a/dev/benchmark/src/aggregated_results.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/benchmark/aggregated_results.py @@ -1,4 +1,4 @@ -from .executor import ExecutionResult +from microsoft_agents.testing.cli.common import ExecutionResult class AggregatedResults: diff --git a/dev/benchmark/src/main.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/benchmark/benchmark.py similarity index 69% rename from dev/benchmark/src/main.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/benchmark/benchmark.py index d8a31c83..7e835292 100644 --- a/dev/benchmark/src/main.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/benchmark/benchmark.py @@ -4,17 +4,19 @@ import click -from .payload_sender import create_payload_sender -from .executor import Executor, CoroutineExecutor, ThreadExecutor +from microsoft_agents.testing.cli.common import ( + Executor, + CoroutineExecutor, + ThreadExecutor, + create_payload_sender, +) + from .aggregated_results import AggregatedResults -from .config import BenchmarkConfig from .output import output_results LOG_FORMAT = "%(asctime)s: %(message)s" logging.basicConfig(format=LOG_FORMAT, level=logging.INFO, datefmt="%H:%M:%S") -BenchmarkConfig.load_from_env() - @click.command() @click.option( @@ -28,18 +30,18 @@ is_flag=True, help="Run coroutine workers rather than thread workers.", ) -def main(payload_path: str, num_workers: int, verbose: bool, async_mode: bool): - """Main function to run the benchmark.""" +def benchmark(payload_path: str, num_workers: int, verbose: bool, async_mode: bool): + """Run a benchmark against an agent with a custom payload.""" with open(payload_path, "r", encoding="utf-8") as f: payload = json.load(f) - func = create_payload_sender(payload) + payload_sender = create_payload_sender(payload) executor: Executor = CoroutineExecutor() if async_mode else ThreadExecutor() start_time = datetime.now(timezone.utc).timestamp() - results = executor.run(func, num_workers=num_workers) + results = executor.run(payload_sender, num_workers=num_workers) end_time = datetime.now(timezone.utc).timestamp() if verbose: output_results(results) @@ -47,7 +49,3 @@ def main(payload_path: str, num_workers: int, verbose: bool, async_mode: bool): agg = AggregatedResults(results) agg.display(start_time, end_time) agg.display_timeline() - - -if __name__ == "__main__": - main() # pylint: disable=no-value-for-parameter diff --git a/dev/benchmark/src/output.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/benchmark/output.py similarity index 85% rename from dev/benchmark/src/output.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/benchmark/output.py index a0d3d76a..a1caecbd 100644 --- a/dev/benchmark/src/output.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/benchmark/output.py @@ -1,4 +1,4 @@ -from .executor import ExecutionResult +from microsoft_agents.testing.cli.common import ExecutionResult def output_results(results: list[ExecutionResult]) -> None: diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/ddt/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/ddt/__init__.py new file mode 100644 index 00000000..33d11268 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/ddt/__init__.py @@ -0,0 +1,5 @@ +from .ddt import ddt + +__all__ = [ + "ddt" +] \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/ddt/ddt.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/ddt/ddt.py new file mode 100644 index 00000000..4b6523b3 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/ddt/ddt.py @@ -0,0 +1,99 @@ +# from contextlib import contextmanager +# from pathlib import Path +# import logging +# import tempfile +# import io + +# import click +# import pytest + +# from microsoft_agents.testing.cli.cli_config import cli_config + +# # agents-cli --env_path .\agents\basic_agent\python\.env ddt .\tests\basic_agent\directline\SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml --pytest-args -xvs + +# @contextmanager +# def log_context(): +# # Setup log capture for non-pytest logs +# log_stream = io.StringIO() +# log_handler = logging.StreamHandler(log_stream) +# log_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +# log_handler.setFormatter(log_formatter) + +# # Add handler to root logger to capture all logs +# root_logger = logging.getLogger() +# original_level = root_logger.level +# original_handlers = root_logger.handlers.copy() + +# # Remove all existing handlers to prevent duplicate output +# for handler in original_handlers: +# root_logger.removeHandler(handler) + +# # Add our capture handler +# root_logger.addHandler(log_handler) + +# yield + +# # Remove our handler and restore original handlers +# root_logger.removeHandler(log_handler) +# for handler in original_handlers: +# root_logger.addHandler(handler) +# root_logger.setLevel(original_level) + +# # Output captured logs +# log_contents = log_stream.getvalue() +# if log_contents: +# click.echo("\n" + "="*80) +# click.echo("CAPTURED LOGS:") +# click.echo("="*80) +# click.echo(log_contents) +# click.echo("="*80 + "\n") + +# log_stream.close() + +# @click.command() +# @click.argument("test_path", default="./") +# @click.option("--service_url", default="http://localhost:8001/", help='Service URL to reply to') +# @click.option("--pytest-args", default="-v -s", help='Arguments to pass to pytest as a string') +# @click.pass_context +# def ddt(ctx, test_path: str, service_url: str, pytest_args: str): + +# env_path = ctx.obj["env_path"] + +# test_path = str(Path(test_path).absolute()) + +# agent_url = cli_config.agent_url + +# # Write the test class as actual Python code +# test_code = f''' +# from microsoft_agents.testing.integration import ddt as ddt_decorator, Integration + +# @ddt_decorator(r"{test_path}") +# class Test(Integration): +# _agent_url = r"{agent_url}" +# _service_url = r"{service_url}" +# _config_path = r"{env_path}" +# ''' + +# # Create temp file in a known directory to avoid pytest scanning issues +# temp_dir = Path(tempfile.gettempdir()) / "microsoft_agents_cli" +# temp_dir = temp_dir.absolute() +# temp_dir.mkdir(exist_ok=True) + +# temp_file = temp_dir / f"test_ddt_{Path(test_path).stem}.py" +# temp_file.write_text(test_code) + +# with log_context(): +# try: +# # Use --override-ini to prevent pytest from using parent configs +# # and --rootdir to set a specific root +# exit_code = pytest.main([ +# *pytest_args.split(), +# "--override-ini=testpaths=.", +# "--asyncio-mode=auto", +# f"--rootdir={temp_dir}", +# str(temp_file) +# ]) +# finally: +# temp_file.unlink(missing_ok=True) + +# return exit_code \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/post/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/post/__init__.py new file mode 100644 index 00000000..bb0d264a --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/post/__init__.py @@ -0,0 +1,3 @@ +from .post import post + +__all__ = ["post"] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/post/post.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/post/post.py new file mode 100644 index 00000000..f9ae4909 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/commands/post/post.py @@ -0,0 +1,40 @@ +import json + +import click + +from microsoft_agents.testing.cli.common import ( + Executor, + CoroutineExecutor, + ThreadExecutor, + create_payload_sender, +) + + +@click.command() +@click.option( + "--payload_path", "-p", default="./payload.json", help="Path to the payload file." +) +@click.option("--verbose", "-v", is_flag=True, help="Enable verbose logging.") +@click.option( + "--async_mode", + "-a", + is_flag=True, + help="Run coroutine workers rather than thread workers.", +) +def post(payload_path: str, async_mode: bool): + """Send an activity to an agent.""" + + with open(payload_path, "r", encoding="utf-8") as f: + payload = json.load(f) + + payload_sender = create_payload_sender(payload) + + executor: Executor = CoroutineExecutor() if async_mode else ThreadExecutor() + + result = executor.run(payload_sender)[0] + + status = "Success" if result.success else "Failure" + print( + f"Execution ID: {result.exe_id}, Duration: {result.duration:.4f} seconds, Status: {status}" + ) + print(result.result) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/cli/common/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/common/__init__.py new file mode 100644 index 00000000..2ed1fa99 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/common/__init__.py @@ -0,0 +1,10 @@ +from .executor import Executor, ExecutionResult, CoroutineExecutor, ThreadExecutor +from .create_payload_sender import create_payload_sender + +__all__ = [ + "Executor", + "ExecutionResult", + "CoroutineExecutor", + "ThreadExecutor", + "create_payload_sender", +] diff --git a/dev/benchmark/src/payload_sender.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/common/create_payload_sender.py similarity index 66% rename from dev/benchmark/src/payload_sender.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/cli/common/create_payload_sender.py index a27f87c0..4a7fd9cc 100644 --- a/dev/benchmark/src/payload_sender.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/common/create_payload_sender.py @@ -5,8 +5,9 @@ import requests from typing import Callable, Awaitable, Any -from .config import BenchmarkConfig -from .generate_token import generate_token_from_env +from microsoft_agents.testing.utils import generate_token + +from microsoft_agents.testing.cli.cli_config import cli_config def create_payload_sender( @@ -19,13 +20,20 @@ def create_payload_sender( :return: A callable that sends the payload when invoked. """ - token = generate_token_from_env() - endpoint = BenchmarkConfig.AGENT_URL + token = generate_token( + cli_config.app_id, + cli_config.app_secret, + cli_config.tenant_id, + ) headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} async def payload_sender() -> Any: response = await asyncio.to_thread( - requests.post, endpoint, headers=headers, json=payload, timeout=timeout + requests.post, + cli_config.agent_endpoint, + headers=headers, + json=payload, + timeout=timeout, ) return response.content diff --git a/dev/benchmark/src/executor/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/common/executor/__init__.py similarity index 100% rename from dev/benchmark/src/executor/__init__.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/cli/common/executor/__init__.py diff --git a/dev/benchmark/src/executor/coroutine_executor.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/common/executor/coroutine_executor.py similarity index 100% rename from dev/benchmark/src/executor/coroutine_executor.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/cli/common/executor/coroutine_executor.py diff --git a/dev/benchmark/src/executor/execution_result.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/common/executor/execution_result.py similarity index 100% rename from dev/benchmark/src/executor/execution_result.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/cli/common/executor/execution_result.py diff --git a/dev/benchmark/src/executor/executor.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/common/executor/executor.py similarity index 100% rename from dev/benchmark/src/executor/executor.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/cli/common/executor/executor.py diff --git a/dev/benchmark/src/executor/thread_executor.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli/common/executor/thread_executor.py similarity index 100% rename from dev/benchmark/src/executor/thread_executor.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/cli/common/executor/thread_executor.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py index 77a605ae..10e29643 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py @@ -1,30 +1,24 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from .core import ( +from .application_runner import ApplicationRunner +from .aiohttp import AiohttpEnvironment, AiohttpRunner +from .client import ( AgentClient, - ApplicationRunner, - AiohttpEnvironment, ResponseClient, - Environment, - Integration, - Sample, -) -from .data_driven import ( - DataDrivenTest, - ddt, - load_ddts, ) +from .environment import Environment +from .integration import Integration +from .sample import Sample + __all__ = [ "AgentClient", "ApplicationRunner", "AiohttpEnvironment", + "AiohttpRunner", "ResponseClient", "Environment", "Integration", "Sample", - "DataDrivenTest", - "ddt", - "load_ddts", ] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/aiohttp/__init__.py similarity index 60% rename from dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/integration/aiohttp/__init__.py index 4625620e..08a65aba 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/aiohttp/__init__.py @@ -3,5 +3,6 @@ from .aiohttp_environment import AiohttpEnvironment from .aiohttp_runner import AiohttpRunner +from .aiohttp_async_runner import AiohttpAsyncRunner -__all__ = ["AiohttpEnvironment", "AiohttpRunner"] +__all__ = ["AiohttpEnvironment", "AiohttpRunner", "AiohttpAsyncRunner"] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/aiohttp/aiohttp_async_runner.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/aiohttp/aiohttp_async_runner.py new file mode 100644 index 00000000..4d05a100 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/aiohttp/aiohttp_async_runner.py @@ -0,0 +1,102 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import Optional +import asyncio + +from aiohttp import ClientSession +from aiohttp.web import Application, Request, Response +from aiohttp.web_runner import AppRunner, TCPSite + +from ..application_runner import ApplicationRunner + + +class AiohttpAsyncRunner(ApplicationRunner): + """A runner for aiohttp applications.""" + + def __init__(self, app: Application, host: str = "localhost", port: int = 8000): + assert isinstance(app, Application) + super().__init__(app) + + url = f"{host}:{port}" + self._host = host + self._port = port + if "http" not in url: + url = f"http://{url}" + self._url = url + + self._app.router.add_get("/shutdown", self._shutdown_route) + + self._server_task: Optional[asyncio.Task] = None + self._shutdown_event = asyncio.Event() + self._runner: Optional[AppRunner] = None + self._site: Optional[TCPSite] = None + + @property + def url(self) -> str: + return self._url + + async def _start_server(self) -> None: + assert isinstance(self._app, Application) + + self._runner = AppRunner(self._app) + await self._runner.setup() + self._site = TCPSite(self._runner, self._host, self._port) + await self._site.start() + + # Wait for shutdown signal + await self._shutdown_event.wait() + + # Cleanup + await self._site.stop() + await self._runner.cleanup() + + async def __aenter__(self): + if self._server_task: + raise RuntimeError("AiohttpRunner is already running.") + + self._shutdown_event.clear() + + # Create a background task instead of a thread + self._server_task = asyncio.create_task(self._start_server()) + + # Wait a moment to ensure the server starts + await asyncio.sleep(0.5) + + return self + + async def _stop_server(self): + try: + async with ClientSession() as session: + async with session.get( + f"http://{self._host}:{self._port}/shutdown" + ) as response: + pass # Just trigger the shutdown + except Exception: + pass # Ignore errors during shutdown request + + # Set shutdown event as fallback + self._shutdown_event.set() + + async def _shutdown_route(self, request: Request) -> Response: + """Handle shutdown request by setting the shutdown event""" + self._shutdown_event.set() + return Response(status=200, text="Shutdown initiated") + + async def __aexit__(self, exc_type, exc, tb): + if not self._server_task: + raise RuntimeError("AiohttpRunner is not running.") + + await self._stop_server() + + # Wait for the server task to complete + try: + await asyncio.wait_for(self._server_task, timeout=5.0) + except asyncio.TimeoutError: + self._server_task.cancel() + try: + await self._server_task + except asyncio.CancelledError: + pass + + self._server_task = None diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/aiohttp/aiohttp_environment.py similarity index 86% rename from dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/integration/aiohttp/aiohttp_environment.py index cd630697..e5a933df 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/aiohttp/aiohttp_environment.py @@ -25,6 +25,10 @@ class AiohttpEnvironment(Environment): """An environment for aiohttp-hosted agents.""" + def __init__(self, use_jwt_middleware: bool = True) -> None: + super().__init__() + self._use_jwt_middleware = use_jwt_middleware + async def init_env(self, environ_config: dict) -> None: environ_config = environ_config or {} @@ -51,7 +55,11 @@ async def entry_point(req: Request) -> Response: adapter: CloudAdapter = req.app["adapter"] return await start_agent_process(req, agent, adapter) - APP = Application(middlewares=[jwt_authorization_middleware]) + middlewares = [] + if self._use_jwt_middleware: + middlewares.append(jwt_authorization_middleware) + + APP = Application(middlewares=middlewares) APP.router.add_post("/api/messages", entry_point) APP["agent_configuration"] = ( self.connection_manager.get_default_connection_configuration() diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/aiohttp/aiohttp_runner.py similarity index 100% rename from dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/integration/aiohttp/aiohttp_runner.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/application_runner.py similarity index 100% rename from dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/integration/application_runner.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/client/__init__.py similarity index 100% rename from dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/integration/client/__init__.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/client/agent_client.py similarity index 63% rename from dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/integration/client/agent_client.py index 7fdf5e79..9c115bda 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/client/agent_client.py @@ -7,15 +7,15 @@ from aiohttp import ClientSession from msal import ConfidentialClientApplication +from pydantic import ValidationError from microsoft_agents.activity import ( Activity, ActivityTypes, DeliveryModes, ChannelAccount, - ConversationAccount, + InvokeResponse, ) -from microsoft_agents.testing.utils import populate_activity _DEFAULT_ACTIVITY_VALUES = { "service_url": "http://localhost", @@ -36,8 +36,8 @@ def __init__( tenant_id: str, client_secret: str, service_url: Optional[str] = None, - default_timeout: float = 5.0, default_activity_data: Optional[Activity | dict] = None, + default_sleep: float = 0.1, ): self._agent_url = agent_url self._cid = cid @@ -46,13 +46,13 @@ def __init__( self._client_secret = client_secret self._service_url = service_url self._headers = None - self._default_timeout = default_timeout self._client: Optional[ClientSession] = None self._default_activity_data: Activity | dict = ( default_activity_data or _DEFAULT_ACTIVITY_VALUES ) + self._default_sleep = default_sleep @property def agent_url(self) -> str: @@ -91,7 +91,14 @@ async def _init_client(self) -> None: base_url=self._agent_url, headers=self._headers ) - async def send_request(self, activity: Activity, sleep: float = 0) -> str: + async def _send( + self, + activity: Activity, + sleep: float | None = None, + ) -> str: + + if sleep is None: + sleep = self._default_sleep await self._init_client() assert self._client @@ -100,7 +107,7 @@ async def send_request(self, activity: Activity, sleep: float = 0) -> str: activity.service_url = self.service_url # activity = populate_activity(activity, self._default_activity_data) - + async with self._client.post( "api/messages", headers=self._headers, @@ -125,36 +132,71 @@ def _to_activity(self, activity_or_text: Activity | str) -> Activity: return cast(Activity, activity_or_text) async def send_activity( - self, - activity_or_text: Activity | str, - sleep: float = 0, - timeout: Optional[float] = None, + self, activity_or_text: Activity | str, sleep: float | None = None ) -> str: - timeout = timeout or self._default_timeout activity = self._to_activity(activity_or_text) - content = await self.send_request(activity, sleep=sleep) + content = await self._send(activity, sleep=sleep) return content + # async def send_stream( + # self, activity_or_text: Activity | str, sleep: float | None = None + # ) -> list[Activity]: + + # activity = self._to_activity(activity_or_text) + # if isinstance(activity_or_text, str): + # activity.delivery_mode = DeliveryModes.stream + + # if not activity.delivery_mode == DeliveryModes.stream: + # raise ValueError( + # "Activity delivery_mode must be 'stream' for send_stream method." + # ) + + # content = await self._send(activity, sleep=sleep) + + # await asyncio.sleep(5) # Allow time for all activities to be processed + + # activities_data = json.loads(content).get("activities", []) + # activities = [Activity.model_validate(act) for act in activities_data] + + # return activities + async def send_expect_replies( - self, - activity_or_text: Activity | str, - sleep: float = 0, - timeout: Optional[float] = None, + self, activity_or_text: Activity | str, sleep: float | None = None ) -> list[Activity]: - timeout = timeout or self._default_timeout + activity = self._to_activity(activity_or_text) - activity.delivery_mode = DeliveryModes.expect_replies - activity.service_url = ( - activity.service_url or "http://localhost" - ) # temporary fix + if isinstance(activity_or_text, str): + activity.delivery_mode = DeliveryModes.expect_replies + + if not activity.delivery_mode == DeliveryModes.expect_replies: + raise ValueError( + "Activity delivery_mode must be 'expect_replies' for send_expect_replies method." + ) - content = await self.send_request(activity, sleep=sleep) + content = await self._send(activity, sleep=sleep) activities_data = json.loads(content).get("activities", []) activities = [Activity.model_validate(act) for act in activities_data] return activities + async def send_invoke_activity( + self, activity: Activity, sleep: float | None = None + ) -> InvokeResponse: + + if not activity.type == ActivityTypes.invoke: + raise ValueError("Activity type must be 'invoke' for send_invoke method.") + + content = await self._send(activity, sleep=sleep) + + try: + response_data = json.loads(content) + return InvokeResponse.model_validate(response_data) + except ValidationError: + raise ValueError( + "Error when sending invoke activity: InvokeResponse not returned or invalid format." + ) + async def close(self) -> None: if self._client: await self._client.close() diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/client/response_client.py similarity index 65% rename from dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/integration/client/response_client.py index 280195d1..8ab66019 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/client/response_client.py @@ -10,10 +10,7 @@ from aiohttp.web import Application, Request, Response -from microsoft_agents.activity import ( - Activity, - ActivityTypes, -) +from microsoft_agents.activity import Activity, ActivityTypes, Entity from ..aiohttp import AiohttpRunner @@ -60,37 +57,28 @@ async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: await self._app_runner.__aexit__(exc_type, exc_val, exc_tb) + async def _add(self, activity: Activity) -> None: + with self._activities_list_lock: + self._activities_list.append(activity) + async def _handle_conversation(self, request: Request) -> Response: try: data = await request.json() activity = Activity.model_validate(data) - # conversation_id = ( - # activity.conversation.id if activity.conversation else None - # ) - - with self._activities_list_lock: - self._activities_list.append(activity) - - if any(map(lambda x: x.type == "streaminfo", activity.entities or [])): - await self._handle_streamed_activity(activity) - return Response(status=200, text="Stream info handled") - else: - if activity.type != ActivityTypes.typing: - await asyncio.sleep(0.1) # Simulate processing delay - return Response( - status=200, - content_type="application/json", - text='{"message": "Activity received"}', - ) + await self._add(activity) + if activity.type != ActivityTypes.typing: + await asyncio.sleep(0.05) # Simulate processing delay + + return Response( + status=200, + content_type="application/json", + text='{"message": "Activity received"}', + ) + except Exception as e: return Response(status=500, text=str(e)) - async def _handle_streamed_activity( - self, activity: Activity, *args, **kwargs - ) -> bool: - raise NotImplementedError("_handle_streamed_activity is not implemented yet.") - async def pop(self) -> list[Activity]: with self._activities_list_lock: activities = self._activities_list[:] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py deleted file mode 100644 index a1161336..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -from .application_runner import ApplicationRunner -from .aiohttp import AiohttpEnvironment -from .client import ( - AgentClient, - ResponseClient, -) -from .environment import Environment -from .integration import Integration -from .sample import Sample - - -__all__ = [ - "AgentClient", - "ApplicationRunner", - "AiohttpEnvironment", - "ResponseClient", - "Environment", - "Integration", - "Sample", -] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/auto_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/auto_client.py deleted file mode 100644 index dcea531b..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/auto_client.py +++ /dev/null @@ -1,18 +0,0 @@ -# from microsoft_agents.activity import Activity - -# from ..agent_client import AgentClient - -# class AutoClient: - -# def __init__(self, agent_client: AgentClient): -# self._agent_client = agent_client - -# async def generate_message(self) -> str: -# pass - -# async def run(self, max_turns: int = 10, time_between_turns: float = 2.0) -> None: - -# for i in range(max_turns): -# await self._agent_client.send_activity( -# Activity(type="message", text=self.generate_message()) -# ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py deleted file mode 100644 index a0ddd2e7..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -from .data_driven_test import DataDrivenTest -from .ddt import ddt -from .load_ddts import load_ddts - -__all__ = ["DataDrivenTest", "ddt", "load_ddts"] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py deleted file mode 100644 index 051042cc..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License.s - -import pytest -import asyncio - -import yaml - -from copy import deepcopy - -from microsoft_agents.activity import Activity - -from microsoft_agents.testing.assertions import ModelAssertion -from microsoft_agents.testing.utils import ( - update_with_defaults, -) - -from ..core import AgentClient, ResponseClient - - -class DataDrivenTest: - """Data driven test runner.""" - - def __init__(self, test_flow: dict) -> None: - self._name: str = test_flow.get("name", "") - if not self._name: - raise ValueError("Test flow must have a 'name' field.") - self._description = test_flow.get("description", "") - - defaults = test_flow.get("defaults", {}) - self._input_defaults = defaults.get("input", {}) - self._assertion_defaults = defaults.get("assertion", {}) - self._sleep_defaults = defaults.get("sleep", {}) - - parent = test_flow.get("parent") - if parent: - parent_input_defaults = parent.get("defaults", {}).get("input", {}) - parent_sleep_defaults = parent.get("defaults", {}).get("sleep", {}) - parent_assertion_defaults = parent.get("defaults", {}).get("assertion", {}) - - update_with_defaults(self._input_defaults, parent_input_defaults) - update_with_defaults(self._sleep_defaults, parent_sleep_defaults) - update_with_defaults(self._assertion_defaults, parent_assertion_defaults) - - self._test = test_flow.get("test", []) - - @property - def name(self) -> str: - """Get the name of the data driven test.""" - return self._name - - def _load_input(self, input_data: dict) -> Activity: - defaults = deepcopy(self._input_defaults) - update_with_defaults(input_data, defaults) - return Activity.model_validate(input_data.get("activity", {})) - - def _load_assertion(self, assertion_data: dict) -> ModelAssertion: - defaults = deepcopy(self._assertion_defaults) - update_with_defaults(assertion_data, defaults) - return ModelAssertion.from_config(assertion_data) - - async def _sleep(self, sleep_data: dict) -> None: - duration = sleep_data.get("duration") - if duration is None: - duration = self._sleep_defaults.get("duration", 0) - await asyncio.sleep(duration) - - def _pre_process(self) -> None: - """Compile the data driven test to ensure all steps are valid.""" - for step in self._test: - if step.get("type") == "assertion": - if "assertion" not in step: - if "activity" in step: - step["assertion"] = step["activity"] - selector = step.get("selector") - if selector is not None: - if isinstance(selector, int): - step["selector"] = {"index": selector} - elif isinstance(selector, dict): - if "selector" not in selector: - if "activity" in selector: - selector["selector"] = selector["activity"] - - async def run( - self, agent_client: AgentClient, response_client: ResponseClient - ) -> None: - """Run the data driven test. - - :param agent_client: The agent client to send activities to. - """ - - self._pre_process() - - responses = [] - for step in self._test: - step_type = step.get("type") - if not step_type: - raise ValueError("Each step must have a 'type' field.") - - if step_type == "input": - input_activity = self._load_input(step) - if input_activity.delivery_mode == "expectReplies": - replies = await agent_client.send_expect_replies(input_activity) - responses.extend(replies) - else: - await agent_client.send_activity(input_activity) - - elif step_type == "assertion": - activity_assertion = self._load_assertion(step) - responses.extend(await response_client.pop()) - - res, err = activity_assertion.check(responses) - - if not res: - err = "Assertion failed: {}\n\n{}".format(step, err) - assert res, err - - elif step_type == "sleep": - await self._sleep(step) - - elif step_type == "breakpoint": - breakpoint() - - elif step_type == "skip": - pytest.skip("Skipping step as per test definition.") diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py deleted file mode 100644 index 57ae7129..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -from typing import Callable, TypeVar - -import pytest - -from microsoft_agents.testing.integration.core import Integration - -from .data_driven_test import DataDrivenTest -from .load_ddts import load_ddts - -IntegrationT = TypeVar("IntegrationT", bound=type[Integration]) - - -def _add_test_method( - test_cls: type[Integration], data_driven_test: DataDrivenTest -) -> None: - """Add a test method to the test class for the given data driven test. - - :param test_cls: The test class to add the test method to. - :param data_driven_test: The data driven test to add as a method. - """ - - test_case_name = ( - f"test_data_driven__{data_driven_test.name.replace('/', '_').replace('.', '_')}" - ) - - @pytest.mark.asyncio - async def _func(self, agent_client, response_client) -> None: - await data_driven_test.run(agent_client, response_client) - - setattr(test_cls, test_case_name, _func) - - -def ddt( - test_path: str, recursive: bool = True, prefix: str = "" -) -> Callable[[IntegrationT], IntegrationT]: - """Decorator to add data driven tests to an integration test class. - - :param test_path: The path to the data driven test files. - :param recursive: Whether to load data driven tests recursively from subdirectories. - :return: The decorated test class. - """ - - ddts = load_ddts(test_path, recursive=recursive, prefix=prefix) - if not ddts: - raise RuntimeError(f"No data driven tests found in path: {test_path}") - - def decorator(test_cls: IntegrationT) -> IntegrationT: - for data_driven_test in ddts: - # scope data_driven_test to avoid late binding in loop - _add_test_method(test_cls, data_driven_test) - return test_cls - - return decorator diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/load_ddts.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/load_ddts.py deleted file mode 100644 index c0341a59..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/load_ddts.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import json, yaml -from glob import glob -from pathlib import Path -from .data_driven_test import DataDrivenTest - - -def _resolve_parent(path: str, test_modules: dict) -> None: - """Resolve the parent test flow for a given test flow data. - - :param data: The test flow data. - :param tests: A dictionary of all test flows keyed by their file paths. - """ - - module = test_modules[str(path)] - parent_field = module.get("parent") - if parent_field and isinstance(parent_field, str): - # resolve a parent path reference to the data itself - parent_path = Path(path).parent / parent_field - parent_path_str = str(parent_path) - if parent_path_str not in test_modules: - raise RuntimeError("Parent module not found in tests collection.") - module["parent"] = test_modules[parent_path_str] - - -_resolve_name_seen_set = set() - - -def _resolve_name(module: dict) -> str: - """Resolve the name for a given test flow data. - - :param data: The test flow data. - :param tests: A dictionary of all test flows keyed by their file paths. - :return: The resolved name. - """ - - if id(module) in _resolve_name_seen_set: - return module.get("name", module["path"]) - _resolve_name_seen_set.add(id(module)) - - parent = module.get("parent") - if parent: - return f"{_resolve_name(parent)}.{module.get('name', module['path'])}" - else: - return module.get("name", module["path"]) - - -def load_ddts( - path: str | Path | None = None, recursive: bool = True, prefix: str = "" -) -> list[DataDrivenTest]: - """Load data driven tests from JSON and YAML files in a given path. - - :param path: The path to load test files from. If None, the current working directory is used. - :param recursive: Whether to search for test files recursively in subdirectories. - :return: A list of DataDrivenTest instances. - """ - - if not path: - path = Path.cwd() - - # collect test file paths - if recursive: - json_file_paths = glob(f"{path}/**/*.json", recursive=True) - yaml_file_paths = glob(f"{path}/**/*.yaml", recursive=True) - else: - json_file_paths = glob(f"{path}/*.json") - yaml_file_paths = glob(f"{path}/*.yaml") - - # load files - tests_json = dict() - for json_file_path in json_file_paths: - with open(json_file_path, "r", encoding="utf-8") as f: - tests_json[str(Path(json_file_path).absolute())] = json.load(f) - - tests_yaml = dict() - for yaml_file_path in yaml_file_paths: - with open(yaml_file_path, "r", encoding="utf-8") as f: - tests_yaml[str(Path(yaml_file_path).absolute())] = yaml.safe_load(f) - - test_modules = {**tests_json, **tests_yaml} - - for file_path, module in test_modules.items(): - _resolve_parent(file_path, test_modules) - module["path"] = Path(file_path).stem # store path for name resolution - for file_path, module in test_modules.items(): - module["name"] = _resolve_name(module) - if prefix: - module["name"] = f"{prefix}.{module['name']}" - - return [ - DataDrivenTest(test_flow=data) - for data in test_modules.values() - if "test" in data - ] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py new file mode 100644 index 00000000..b7a38ee3 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py @@ -0,0 +1,4 @@ +class DataDrivenTest + + def __init__(self, test_seq: list[DDTComponent]): + pass \ No newline at end of file diff --git a/dev/integration/agents/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/ddt/ddt.py similarity index 100% rename from dev/integration/agents/__init__.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/integration/ddt/ddt.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/ddt/ddt_component.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/ddt/ddt_component.py new file mode 100644 index 00000000..c4960f43 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/ddt/ddt_component.py @@ -0,0 +1,13 @@ +from microsoft_agents.testing.assertions import ( + ModelQuery +) + +class DDTComponent: + pass + +class Send(DDTComponent): + pass + +class Receive(DDTComponent): + + def __init__(self, ) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/environment.py similarity index 100% rename from dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/integration/environment.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/integration.py similarity index 97% rename from dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/integration/integration.py index ce56da9c..55440c68 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/integration.py @@ -33,7 +33,7 @@ class Integration: _service_url: Optional[str] = "http://localhost:9378" _agent_url: Optional[str] = "http://localhost:3978" - _config_path: Optional[str] = "./src/tests/.env" + _config_path: Optional[str] = ".env" _cid: Optional[str] = None _client_id: Optional[str] = None _tenant_id: Optional[str] = None @@ -56,7 +56,7 @@ def setup_method(self): if not self._config: self._config = {} - load_dotenv(self._config_path) + load_dotenv(self._config_path, override=True) self._config.update( { "client_id": os.getenv( diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/sample.py similarity index 100% rename from dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/integration/sample.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py index eddb25de..d24ec68b 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py @@ -2,11 +2,16 @@ # Licensed under the MIT License. from .populate import update_with_defaults, populate_activity -from .misc import get_host_and_port, normalize_model_data +from .misc import pdb_breakpoint, get_host_and_port, normalize_model_data +from .resolve_env import resolve_env +from .generate_token import generate_token __all__ = [ "update_with_defaults", "populate_activity", + "pdb_breakpoint", "get_host_and_port", "normalize_model_data", + "resolve_env", + "generate_token", ] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/generate_token.py similarity index 100% rename from dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/utils/generate_token.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py index 66771de5..458979d6 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py @@ -6,6 +6,13 @@ from microsoft_agents.activity import AgentsModel +def pdb_breakpoint() -> None: + """Set a breakpoint using pdb.""" + import pdb + + pdb.set_trace() + + def get_host_and_port(url: str) -> tuple[str, int]: """Extract host and port from a URL.""" diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/resolve_env.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/resolve_env.py new file mode 100644 index 00000000..efd28f4a --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/resolve_env.py @@ -0,0 +1,23 @@ +from glob import glob +from pathlib import Path + +from dotenv import dotenv_values + + +def resolve_env(path: str | Path) -> dict: + """Resolves a .env file from a given path, which can be a file or directory. + + :param path: Path to a .env file or a directory containing a .env file. + :return: A dictionary containing the key-value pairs from the .env file. + """ + + path = Path(path) + + if path.is_dir(): + env_files = glob(str(path / ".env")) + if not env_files: + raise FileNotFoundError(f"No .env file found in directory: {path}") + path = Path(env_files[0]) + + config = dotenv_values(path) + return config diff --git a/dev/microsoft-agents-testing/pyproject.toml b/dev/microsoft-agents-testing/pyproject.toml index 5557ac38..d45f80ec 100644 --- a/dev/microsoft-agents-testing/pyproject.toml +++ b/dev/microsoft-agents-testing/pyproject.toml @@ -23,3 +23,6 @@ classifiers = [ [project.urls] "Homepage" = "https://github.com/microsoft/Agents" + +[project.scripts] +aclip = "microsoft_agents.testing.cli:cli" \ No newline at end of file diff --git a/dev/microsoft-agents-testing/pytest.ini b/dev/microsoft-agents-testing/pytest.ini index fee2ab83..686ad28f 100644 --- a/dev/microsoft-agents-testing/pytest.ini +++ b/dev/microsoft-agents-testing/pytest.ini @@ -12,6 +12,7 @@ filterwarnings = ignore::PendingDeprecationWarning # pytest-asyncio warnings that are safe to ignore ignore:.*deprecated.*asyncio.*:DeprecationWarning:pytest_asyncio.* + ignore:pytest.PytestUnraisableExceptionWarning # Test discovery configuration testpaths = tests diff --git a/dev/benchmark/requirements.txt b/dev/microsoft-agents-testing/requirements.txt similarity index 100% rename from dev/benchmark/requirements.txt rename to dev/microsoft-agents-testing/requirements.txt diff --git a/dev/microsoft-agents-testing/tests/assertions/_common.py b/dev/microsoft-agents-testing/tests/assertions/_common.py deleted file mode 100644 index 83e666e4..00000000 --- a/dev/microsoft-agents-testing/tests/assertions/_common.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import pytest - -from microsoft_agents.activity import Activity - - -@pytest.fixture -def activity(): - return Activity(type="message", text="Hello, World!") - - -@pytest.fixture( - params=[ - Activity(type="message", text="Hello, World!"), - {"type": "message", "text": "Hello, World!"}, - ] -) -def baseline(request): - return request.param diff --git a/dev/microsoft-agents-testing/tests/assertions/test_assert_model.py b/dev/microsoft-agents-testing/tests/assertions/test_assert_model.py deleted file mode 100644 index 870500a0..00000000 --- a/dev/microsoft-agents-testing/tests/assertions/test_assert_model.py +++ /dev/null @@ -1,261 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -from microsoft_agents.activity import Activity, Attachment -from microsoft_agents.testing.assertions import assert_model, check_model - - -class TestAssertModel: - """Tests for assert_model function.""" - - def test_assert_model_with_matching_simple_fields(self): - """Test that activity matches baseline with simple equal fields.""" - activity = Activity(type="message", text="Hello, World!") - baseline = {"type": "message", "text": "Hello, World!"} - assert_model(activity, baseline) - - def test_assert_model_with_non_matching_fields(self): - """Test that activity doesn't match baseline with different field values.""" - activity = Activity(type="message", text="Hello") - baseline = {"type": "message", "text": "Goodbye"} - assert not check_model(activity, baseline) - - def test_assert_model_with_activity_baseline(self): - """Test that baseline can be an Activity object.""" - activity = Activity(type="message", text="Hello") - baseline = Activity(type="message", text="Hello") - assert_model(activity, baseline) - - def test_assert_model_with_partial_baseline(self): - """Test that only fields in baseline are checked.""" - activity = Activity( - type="message", - text="Hello", - channel_id="test-channel", - conversation={"id": "conv123"}, - ) - baseline = {"type": "message", "text": "Hello"} - assert_model(activity, baseline) - - def test_assert_model_with_missing_field(self): - """Test that activity with missing field doesn't match baseline.""" - activity = Activity(type="message") - baseline = {"type": "message", "text": "Hello"} - assert not check_model(activity, baseline) - - def test_assert_model_with_none_values(self): - """Test that None values are handled correctly.""" - activity = Activity(type="message") - baseline = {"type": "message", "text": None} - assert_model(activity, baseline) - - def test_assert_model_with_empty_baseline(self): - """Test that empty baseline always matches.""" - activity = Activity(type="message", text="Hello") - baseline = {} - assert_model(activity, baseline) - - def test_assert_model_with_dict_assertion_format(self): - """Test using dict format for assertions.""" - activity = Activity(type="message", text="Hello, World!") - baseline = { - "type": "message", - "text": {"assertion_type": "CONTAINS", "assertion": "Hello"}, - } - assert_model(activity, baseline) - - def test_assert_model_with_list_assertion_format(self): - """Test using list format for assertions.""" - activity = Activity(type="message", text="Hello, World!") - baseline = {"type": "message", "text": ["CONTAINS", "World"]} - assert_model(activity, baseline) - - def test_assert_model_with_not_equals_assertion(self): - """Test NOT_EQUALS assertion type.""" - activity = Activity(type="message", text="Hello") - baseline = { - "type": "message", - "text": {"assertion_type": "NOT_EQUALS", "assertion": "Goodbye"}, - } - assert_model(activity, baseline) - - def test_assert_model_with_contains_assertion(self): - """Test CONTAINS assertion type.""" - activity = Activity(type="message", text="Hello, World!") - baseline = {"text": {"assertion_type": "CONTAINS", "assertion": "World"}} - assert_model(activity, baseline) - - def test_assert_model_with_not_contains_assertion(self): - """Test NOT_CONTAINS assertion type.""" - activity = Activity(type="message", text="Hello") - baseline = {"text": {"assertion_type": "NOT_CONTAINS", "assertion": "Goodbye"}} - assert_model(activity, baseline) - - def test_assert_model_with_regex_assertion(self): - """Test RE_MATCH assertion type.""" - activity = Activity(type="message", text="msg_20250112_001") - baseline = { - "text": {"assertion_type": "RE_MATCH", "assertion": r"^msg_\d{8}_\d{3}$"} - } - assert_model(activity, baseline) - - def test_assert_model_with_multiple_fields_and_mixed_assertions(self): - """Test multiple fields with different assertion types.""" - activity = Activity( - type="message", text="Hello, World!", channel_id="test-channel" - ) - baseline = { - "type": "message", - "text": ["CONTAINS", "Hello"], - "channel_id": {"assertion_type": "NOT_EQUALS", "assertion": "prod-channel"}, - } - assert_model(activity, baseline) - - def test_assert_model_fails_on_any_field_mismatch(self): - """Test that activity check fails if any field doesn't match.""" - activity = Activity(type="message", text="Hello", channel_id="test-channel") - baseline = {"type": "message", "text": "Hello", "channel_id": "prod-channel"} - assert not check_model(activity, baseline) - - def test_assert_model_with_numeric_fields(self): - """Test with numeric field values.""" - activity = Activity(type="message", locale="en-US") - activity.channel_data = {"timestamp": 1234567890} - baseline = {"type": "message", "channel_data": {"timestamp": 1234567890}} - assert_model(activity, baseline) - - def test_assert_model_with_greater_than_assertion(self): - """Test GREATER_THAN assertion on numeric fields.""" - activity = Activity(type="message") - activity.channel_data = {"count": 100} - baseline = { - "channel_data": { - "count": {"assertion_type": "GREATER_THAN", "assertion": 50} - } - } - - # This test depends on how nested dicts are handled - # If channel_data is compared as a whole dict, this might not work as expected - # Keeping this test to illustrate the concept - assert_model(activity, baseline) - - def test_assert_model_with_complex_nested_structures(self): - """Test with complex nested structures in baseline.""" - activity = Activity( - type="message", conversation={"id": "conv123", "name": "Test Conversation"} - ) - baseline = { - "type": "message", - "conversation": {"id": "conv123", "name": "Test Conversation"}, - } - assert_model(activity, baseline) - - def test_assert_model_with_boolean_fields(self): - """Test with boolean field values.""" - activity = Activity(type="message") - activity.channel_data = {"is_active": True} - baseline = {"channel_data": {"is_active": True}} - assert_model(activity, baseline) - - def test_assert_model_type_mismatch(self): - """Test that different activity types don't match.""" - activity = Activity(type="message", text="Hello") - baseline = {"type": "event", "text": "Hello"} - assert not check_model(activity, baseline) - - def test_assert_model_with_list_fields(self): - """Test with list field values.""" - activity = Activity(type="message") - activity.attachments = [Attachment(content_type="text/plain", content="test")] - baseline = { - "type": "message", - "attachments": [{"content_type": "text/plain", "content": "test"}], - } - assert_model(activity, baseline) - - -class TestAssertModelRealWorldScenarios: - """Tests simulating real-world usage scenarios.""" - - def test_validate_bot_response_message(self): - """Test validating a typical bot response.""" - activity = Activity( - type="message", - text="I found 3 results for your query.", - from_property={"id": "bot123", "name": "HelpBot"}, - ) - baseline = { - "type": "message", - "text": ["RE_MATCH", r"I found \d+ results"], - "from_property": {"id": "bot123"}, - } - assert_model(activity, baseline) - - def test_validate_user_message(self): - """Test validating a user message with flexible text matching.""" - activity = Activity( - type="message", - text="help me with something", - from_property={"id": "user456"}, - ) - baseline = { - "type": "message", - "text": {"assertion_type": "CONTAINS", "assertion": "help"}, - } - assert_model(activity, baseline) - - def test_validate_event_activity(self): - """Test validating an event activity.""" - activity = Activity( - type="event", name="conversationUpdate", value={"action": "add"} - ) - baseline = {"type": "event", "name": "conversationUpdate"} - - assert_model(activity, baseline) - - def test_partial_match_allows_extra_fields(self): - """Test that extra fields in activity don't cause failure.""" - activity = Activity( - type="message", - text="Hello", - channel_id="teams", - conversation={"id": "conv123"}, - from_property={"id": "user123"}, - timestamp="2025-01-12T10:00:00Z", - ) - baseline = {"type": "message", "text": "Hello"} - assert_model(activity, baseline) - - def test_strict_match_with_multiple_fields(self): - """Test strict matching with multiple fields specified.""" - activity = Activity(type="message", text="Hello", channel_id="teams") - baseline = {"type": "message", "text": "Hello", "channel_id": "teams"} - assert_model(activity, baseline) - - def test_flexible_text_matching_with_regex(self): - """Test flexible text matching using regex patterns.""" - activity = Activity(type="message", text="Order #12345 has been confirmed") - baseline = {"type": "message", "text": ["RE_MATCH", r"Order #\d+ has been"]} - assert_model(activity, baseline) - - def test_negative_assertions(self): - """Test using negative assertions to ensure fields don't match.""" - activity = Activity(type="message", text="Success", channel_id="teams") - baseline = { - "type": "message", - "text": {"assertion_type": "NOT_CONTAINS", "assertion": "Error"}, - "channel_id": {"assertion_type": "NOT_EQUALS", "assertion": "slack"}, - } - assert_model(activity, baseline) - - def test_combined_positive_and_negative_assertions(self): - """Test combining positive and negative assertions.""" - activity = Activity( - type="message", text="Operation completed successfully", channel_id="teams" - ) - baseline = { - "type": "message", - "text": ["CONTAINS", "completed"], - "channel_id": ["NOT_EQUALS", "slack"], - } - assert_model(activity, baseline) diff --git a/dev/microsoft-agents-testing/tests/assertions/test_assertion_context.py b/dev/microsoft-agents-testing/tests/assertions/test_assertion_context.py new file mode 100644 index 00000000..676e976a --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/test_assertion_context.py @@ -0,0 +1,477 @@ +import pytest +from unittest.mock import Mock + +from microsoft_agents.testing.assertions.assertion_context import AssertionContext +from microsoft_agents.testing.assertions.types import SafeObject, DynamicObject, Unset +from microsoft_agents.testing.assertions.types.safe_object import resolve, parent + + +class TestAssertionContextInitialization: + """Test AssertionContext initialization.""" + + def test_basic_initialization(self): + """Test basic initialization with actual and baseline sources""" + actual_data = {"key": "value"} + baseline_data = {"key": "baseline"} + + actual_source = SafeObject(actual_data) + context = AssertionContext(actual_source, baseline_data) + + assert context._actual_source == actual_source + assert context._baseline_source == baseline_data + assert context._actual == actual_source + assert context._baseline == baseline_data + assert context._path == "" + assert isinstance(context._context, DynamicObject) + + def test_initialization_with_none_baseline(self): + """Test initialization with None baseline source defaults to empty dict""" + actual_source = SafeObject({"key": "value"}) + context = AssertionContext(actual_source, None) + + assert context._baseline_source == {} + assert context._baseline == {} + + def test_initialization_with_custom_actual_and_baseline(self): + """Test initialization with custom actual and baseline values""" + actual_source = SafeObject({"parent": {"child": "value"}}) + baseline_source = {"parent": {"child": "baseline"}} + + custom_actual = SafeObject("custom_actual") + custom_baseline = "custom_baseline" + + context = AssertionContext( + actual_source, + baseline_source, + actual=custom_actual, + baseline=custom_baseline + ) + + assert context._actual == custom_actual + assert context._baseline == custom_baseline + assert context._actual_source == actual_source + assert context._baseline_source == baseline_source + + def test_initialization_with_custom_context(self): + """Test initialization with custom context object""" + actual_source = SafeObject({"key": "value"}) + baseline_source = {"key": "baseline"} + custom_context = DynamicObject({"custom_key": "custom_value"}) + + context = AssertionContext( + actual_source, + baseline_source, + context=custom_context + ) + + assert context._context == custom_context + assert resolve(context._context)["custom_key"] == "custom_value" + + def test_initialization_with_path(self): + """Test initialization with a custom path""" + actual_source = SafeObject({"key": "value"}) + baseline_source = {"key": "baseline"} + + context = AssertionContext( + actual_source, + baseline_source, + path="parent.child" + ) + + assert context._path == "parent.child" + + def test_initialization_with_all_parameters(self): + """Test initialization with all parameters specified""" + actual_source = SafeObject({"root": "data"}) + baseline_source = {"root": "baseline"} + custom_actual = SafeObject({"nested": "actual"}) + custom_baseline = {"nested": "baseline"} + custom_context = DynamicObject({"test": True}) + custom_path = "root.nested" + + context = AssertionContext( + actual_source, + baseline_source, + actual=custom_actual, + baseline=custom_baseline, + context=custom_context, + path=custom_path + ) + + assert context._actual_source == actual_source + assert context._baseline_source == baseline_source + assert context._actual == custom_actual + assert context._baseline == custom_baseline + assert context._context == custom_context + assert context._path == custom_path + + +class TestAssertionContextNext: + """Test AssertionContext.next() method.""" + + def test_next_creates_new_context_with_key(self): + """Test that next() creates a new context for a nested key""" + actual_data = {"user": {"name": "John", "age": 30}} + baseline_data = {"user": {"name": "Jane", "age": 25}} + + actual_source = SafeObject(actual_data) + context = AssertionContext(actual_source, baseline_data) + + next_context = context.next("user") + + assert next_context._path == "user" + assert next_context._actual_source == actual_source + assert next_context._baseline_source == baseline_data + + def test_next_updates_path_correctly(self): + """Test that next() updates the path correctly""" + actual_source = SafeObject({"a": {"b": {"c": "value"}}}) + baseline_source = {"a": {"b": {"c": "baseline"}}} + + context = AssertionContext(actual_source, baseline_source) + + context_a = context.next("a") + assert context_a._path == "a" + + context_b = context_a.next("b") + assert context_b._path == "a.b" + + context_c = context_b.next("c") + assert context_c._path == "a.b.c" + + def test_next_preserves_context_object(self): + """Test that next() preserves the context object""" + actual_source = SafeObject({"parent": {"child": "value"}}) + baseline_source = {"parent": {"child": "baseline"}} + custom_context = DynamicObject({"preserved": True}) + + context = AssertionContext( + actual_source, + baseline_source, + context=custom_context + ) + + next_context = context.next("parent") + + assert next_context._context == custom_context + assert resolve(next_context._context)["preserved"] is True + + def test_next_with_empty_path(self): + """Test next() when starting path is empty""" + actual_source = SafeObject({"key": "value"}) + baseline_source = {"key": "baseline"} + + context = AssertionContext(actual_source, baseline_source, path="") + next_context = context.next("key") + + assert next_context._path == "key" + + def test_next_with_existing_path(self): + """Test next() when starting path is not empty""" + actual_source = SafeObject({"a": {"b": "value"}}) + baseline_source = {"a": {"b": "baseline"}} + + context = AssertionContext(actual_source, baseline_source, path="existing") + next_context = context.next("a") + + assert next_context._path == "existing.a" + + def test_next_assertion_error_when_baseline_is_none(self): + """Test that next() raises assertion error when baseline is None""" + actual_source = SafeObject({"key": "value"}) + + context = AssertionContext( + actual_source, + baseline_source=None, + baseline=None + ) + + with pytest.raises(Exception): + context.next("key") + + +class TestAssertionContextResolveArgs: + """Test AssertionContext.resolve_args() method.""" + + def test_resolve_args_with_actual_parameter(self): + """Test resolve_args with a function that takes 'actual' parameter""" + actual_data = {"key": "value"} + actual_source = SafeObject(actual_data) + baseline_data = {"key": "baseline"} + + context = AssertionContext(actual_source, baseline_data) + + def query_func(actual): + return lambda: resolve(actual) + + resolved_func = context.resolve_args(query_func) + result = resolved_func() + + assert result == actual_data + + def test_resolve_args_with_path_parameter(self): + """Test resolve_args with a function that takes 'path' parameter""" + actual_source = SafeObject({"key": "value"}) + baseline_data = {"key": "baseline"} + + context = AssertionContext(actual_source, baseline_data, path="parent.child") + + def query_func(path): + return lambda: path + + resolved_func = context.resolve_args(query_func) + result = resolved_func() + + assert result == "parent.child" + + def test_resolve_args_with_value_parameter(self): + """Test resolve_args with a function that takes 'value' parameter""" + actual_source = SafeObject({"parent": {"child": "nested_value"}}) + baseline_data = {"parent": {"child": "baseline"}} + child_value = SafeObject("specific_value") + + context = AssertionContext( + actual_source, + baseline_data, + actual=child_value + ) + + def query_func(value): + return lambda: resolve(value) + + resolved_func = context.resolve_args(query_func) + result = resolved_func() + + assert result == "specific_value" + + def test_resolve_args_with_parent_parameter(self): + """Test resolve_args with a function that takes 'parent' parameter""" + parent_obj = SafeObject({"child": "value"}) + child_obj = SafeObject("child_value", parent_object=parent_obj) + baseline_data = {"key": "baseline"} + + context = AssertionContext( + parent_obj, + baseline_data, + actual=child_obj + ) + + def query_func(parent): + return lambda: parent + + resolved_func = context.resolve_args(query_func) + result = resolved_func() + + assert result == parent_obj + + def test_resolve_args_with_context_parameter(self): + """Test resolve_args with a function that takes 'context' parameter""" + actual_source = SafeObject({"key": "value"}) + baseline_data = {"key": "baseline"} + custom_context = DynamicObject({"test_key": "test_value"}) + + context = AssertionContext( + actual_source, + baseline_data, + context=custom_context + ) + + def query_func(context): + return lambda: resolve(context)["test_key"] + + resolved_func = context.resolve_args(query_func) + result = resolved_func() + + assert result == "test_value" + + def test_resolve_args_with_multiple_parameters(self): + """Test resolve_args with a function that takes multiple parameters""" + actual_data = {"key": "value"} + actual_source = SafeObject(actual_data) + baseline_data = {"key": "baseline"} + custom_context = DynamicObject({"ctx": "context_value"}) + + context = AssertionContext( + actual_source, + baseline_data, + context=custom_context, + path="test.path" + ) + + def query_func(actual, path, context): + return lambda: { + "actual": resolve(actual), + "path": path, + "context": resolve(context)["ctx"] + } + + resolved_func = context.resolve_args(query_func) + result = resolved_func() + + assert result["actual"] == actual_data + assert result["path"] == "test.path" + assert result["context"] == "context_value" + + def test_resolve_args_preserves_function_name(self): + """Test that resolve_args preserves the original function name""" + actual_source = SafeObject({"key": "value"}) + baseline_data = {"key": "baseline"} + + context = AssertionContext(actual_source, baseline_data) + + def my_custom_query_function(path): + return lambda: path + + resolved_func = context.resolve_args(my_custom_query_function) + + assert resolved_func.__name__ == "my_custom_query_function" + + def test_resolve_args_with_unknown_parameter_raises_error(self): + """Test that resolve_args raises RuntimeError for unknown parameters""" + actual_source = SafeObject({"key": "value"}) + baseline_data = {"key": "baseline"} + + context = AssertionContext(actual_source, baseline_data) + + def query_func(unknown_param): + return lambda: None + + with pytest.raises(RuntimeError, match="Unknown argument 'unknown_param'"): + context.resolve_args(query_func) + + def test_resolve_args_with_all_available_parameters(self): + """Test resolve_args with all available parameters""" + actual_data = {"parent": {"child": "value"}} + actual_source = SafeObject(actual_data) + baseline_data = {"parent": {"child": "baseline"}} + custom_context = DynamicObject({"flag": True}) + + parent_obj = SafeObject({"child": "child_val"}) + child_obj = SafeObject("child", parent_object=parent_obj) + + context = AssertionContext( + actual_source, + baseline_data, + actual=child_obj, + context=custom_context, + path="root.parent.child" + ) + + def query_func(actual, path, value, parent, context): + return lambda: { + "actual": isinstance(actual, DynamicObject), + "path": path, + "value": resolve(value), + "parent": parent, + "context": resolve(context)["flag"] + } + + resolved_func = context.resolve_args(query_func) + result = resolved_func() + + assert result["actual"] is True + assert result["path"] == "root.parent.child" + assert result["value"] == "child" + assert result["parent"] == parent_obj + assert result["context"] is True + + def test_resolve_args_actual_is_dynamic_object(self): + """Test that 'actual' parameter is wrapped in DynamicObject""" + actual_data = {"key": "value"} + actual_source = SafeObject(actual_data) + baseline_data = {"key": "baseline"} + + context = AssertionContext(actual_source, baseline_data) + + def query_func(actual): + return lambda: actual + + resolved_func = context.resolve_args(query_func) + result = resolved_func() + + assert isinstance(result, DynamicObject) + assert resolve(result) == actual_data + + +class TestAssertionContextIntegration: + """Integration tests for AssertionContext.""" + + def test_nested_context_navigation(self): + """Test navigating through nested contexts""" + actual_data = { + "user": { + "profile": { + "name": "John", + "address": { + "city": "New York" + } + } + } + } + baseline_data = { + "user": { + "profile": { + "name": "Jane", + "address": { + "city": "Boston" + } + } + } + } + + actual_source = SafeObject(actual_data) + context = AssertionContext(actual_source, baseline_data) + + user_ctx = context.next("user") + profile_ctx = user_ctx.next("profile") + address_ctx = profile_ctx.next("address") + + assert address_ctx._path == "user.profile.address" + + def test_resolve_args_in_nested_context(self): + """Test resolve_args works correctly in nested contexts""" + actual_data = {"level1": {"level2": {"value": "nested"}}} + baseline_data = {"level1": {"level2": {"value": "baseline"}}} + + actual_source = SafeObject(actual_data) + context = AssertionContext(actual_source, baseline_data) + + nested_ctx = context.next("level1").next("level2") + + def query_func(path, value): + return lambda: (path, resolve(value)) + + resolved_func = nested_ctx.resolve_args(query_func) + path_result, value_result = resolved_func() + + assert path_result == "level1.level2" + + def test_context_with_empty_dicts(self): + """Test context with empty dictionaries""" + actual_source = SafeObject({}) + baseline_data = {} + + context = AssertionContext(actual_source, baseline_data) + + assert context._actual_source == actual_source + assert context._baseline_source == {} + assert context._path == "" + + def test_context_with_complex_data_types(self): + """Test context with lists and mixed data types""" + actual_data = { + "items": [1, 2, 3], + "config": {"enabled": True, "count": 42}, + "name": "test" + } + baseline_data = { + "items": [4, 5, 6], + "config": {"enabled": False, "count": 10}, + "name": "baseline" + } + + actual_source = SafeObject(actual_data) + context = AssertionContext(actual_source, baseline_data) + + assert resolve(context._actual_source) == actual_data + assert context._baseline_source == baseline_data \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/assertions/test_assertions.py b/dev/microsoft-agents-testing/tests/assertions/test_assertions.py new file mode 100644 index 00000000..8af1157b --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/test_assertions.py @@ -0,0 +1,570 @@ +import pytest +from unittest.mock import Mock + +from microsoft_agents.testing.assertions.assertions import Assertions +from microsoft_agents.testing.assertions.assertion_context import AssertionContext +from microsoft_agents.testing.assertions.types import SafeObject, DynamicObject, Unset + + +class TestAssertionsExpand: + """Test the Assertions.expand method for flattening and expanding dictionaries.""" + + def test_expand_non_dict_returns_as_is(self): + """Test that non-dict values are returned unchanged""" + assert Assertions.expand("string") == "string" + assert Assertions.expand(123) == 123 + assert Assertions.expand([1, 2, 3]) == [1, 2, 3] + assert Assertions.expand(None) is None + + def test_expand_flat_dict_no_dots(self): + """Test expansion of a flat dictionary without dots in keys""" + data = {"key1": "value1", "key2": "value2"} + result = Assertions.expand(data) + assert result == {"key1": "value1", "key2": "value2"} + + def test_expand_simple_nested_keys(self): + """Test expansion of simple dotted keys""" + data = {"parent.child": "value"} + result = Assertions.expand(data) + assert result == {"parent": {"child": "value"}} + + def test_expand_multiple_levels(self): + """Test expansion of multiple nested levels""" + data = {"root.level1.level2": "value"} + result = Assertions.expand(data) + assert result == {"root": {"level1": {"level2": "value"}}} + + def test_expand_multiple_keys_same_root(self): + """Test expansion with multiple keys sharing the same root""" + data = { + "parent.child1": "value1", + "parent.child2": "value2" + } + result = Assertions.expand(data) + assert result == { + "parent": { + "child1": "value1", + "child2": "value2" + } + } + + def test_expand_mixed_flat_and_nested(self): + """Test expansion with mixed flat and nested keys""" + data = { + "flat_key": "flat_value", + "nested.key": "nested_value" + } + result = Assertions.expand(data) + assert result == { + "flat_key": "flat_value", + "nested": {"key": "nested_value"} + } + + def test_expand_complex_nested_structure(self): + """Test expansion with complex nested structure""" + data = { + "root.child1": "value1", + "root.child2.grandchild": "value2", + "other": "value3" + } + result = Assertions.expand(data) + assert result == { + "root": { + "child1": "value1", + "child2": {"grandchild": "value2"} + }, + "other": "value3" + } + + def test_expand_recursive_expansion(self): + """Test that expansion is applied recursively""" + data = { + "level1.level2": {"level3.level4": "value"} + } + result = Assertions.expand(data) + assert result == { + "level1": { + "level2": { + "level3": {"level4": "value"} + } + } + } + + def test_expand_duplicate_root_raises_error(self): + """Test that duplicate root keys raise RuntimeError""" + data = { + "root": "value1", + "root.child": "value2" + } + with pytest.raises(RuntimeError): + Assertions.expand(data) + + def test_expand_conflicting_structure_raises_error(self): + """Test that conflicting structures raise RuntimeError""" + data = { + "parent": "value", + "parent.child": "child_value" + } + with pytest.raises(RuntimeError): + Assertions.expand(data) + + def test_expand_empty_dict(self): + """Test expansion of an empty dictionary""" + result = Assertions.expand({}) + assert result == {} + + +class TestAssertionsInvoke: + """Test the Assertions.invoke method for invoking query functions.""" + + def test_invoke_returns_tuple_with_bool_and_message(self): + """Test that invoke properly handles functions returning (bool, str) tuple""" + actual = SafeObject({"value": 42}) + + def query_func(value): + return lambda: (True, "Success message") + + context = AssertionContext(actual, {}) + result, message = Assertions.invoke(actual, query_func, context) + + assert result is True + assert message == "Success message" + + def test_invoke_returns_false_tuple(self): + """Test that invoke handles false results correctly""" + actual = SafeObject({"value": 42}) + + def query_func(value): + return lambda: (False, "Failure message") + + context = AssertionContext(actual, {}) + result, message = Assertions.invoke(actual, query_func, context) + + assert result is False + assert message == "Failure message" + + def test_invoke_returns_bool_only(self): + """Test that invoke converts single bool return to tuple""" + actual = SafeObject({"value": 42}) + + def query_func(value): + return lambda: True + + context = AssertionContext(actual, {}) + result, message = Assertions.invoke(actual, query_func, context) + + assert result is True + assert "query_func" in message + + def test_invoke_returns_falsy_value(self): + """Test that invoke handles falsy non-tuple values""" + actual = SafeObject({"value": 42}) + + def query_func(value): + return lambda: 0 + + context = AssertionContext(actual, {}) + result, message = Assertions.invoke(actual, query_func, context) + + assert result is False + assert "query_func" in message + + def test_invoke_returns_truthy_value(self): + """Test that invoke handles truthy non-tuple values""" + actual = SafeObject({"value": 42}) + + def query_func(value): + return lambda: 1 + + context = AssertionContext(actual, {}) + result, message = Assertions.invoke(actual, query_func, context) + + assert result is True + assert "query_func" in message + + +class TestAssertionsCheckVerbose: + """Test the Assertions._check_verbose and check_verbose methods.""" + + def test_check_verbose_equal_primitives(self): + """Test checking equal primitive values""" + result, message = Assertions.check_verbose(42, 42) + assert result is True + assert message == "" + + def test_check_verbose_unequal_primitives(self): + """Test checking unequal primitive values""" + result, message = Assertions.check_verbose(42, 43) + assert result is False + assert "42" in message + assert "43" in message + + def test_check_verbose_equal_strings(self): + """Test checking equal strings""" + result, message = Assertions.check_verbose("hello", "hello") + assert result is True + assert message == "" + + def test_check_verbose_unequal_strings(self): + """Test checking unequal strings""" + result, message = Assertions.check_verbose("hello", "world") + assert result is False + assert "hello" in message + assert "world" in message + + def test_check_verbose_equal_dicts(self): + """Test checking equal dictionaries""" + actual = {"key": "value", "number": 42} + baseline = {"key": "value", "number": 42} + result, message = Assertions.check_verbose(actual, baseline) + assert result is True + assert message == "" + + def test_check_verbose_unequal_dicts(self): + """Test checking unequal dictionaries""" + actual = {"key": "value1"} + baseline = {"key": "value2"} + result, message = Assertions.check_verbose(actual, baseline) + assert result is False + assert "value1" in message + assert "value2" in message + + def test_check_verbose_nested_dicts(self): + """Test checking nested dictionaries""" + actual = {"parent": {"child": "value1"}} + baseline = {"parent": {"child": "value2"}} + result, message = Assertions.check_verbose(actual, baseline) + assert result is False + assert "value1" in message + assert "value2" in message + + def test_check_verbose_equal_lists(self): + """Test checking equal lists""" + actual = [1, 2, 3] + baseline = [1, 2, 3] + result, message = Assertions.check_verbose(actual, baseline) + assert result is True + assert message == "" + + def test_check_verbose_unequal_lists(self): + """Test checking unequal lists""" + actual = [1, 2, 3] + baseline = [1, 2, 4] + result, message = Assertions.check_verbose(actual, baseline) + assert result is False + assert "3" in message + assert "4" in message + + def test_check_verbose_nested_lists(self): + """Test checking nested lists""" + actual = [[1, 2], [3, 4]] + baseline = [[1, 2], [3, 5]] + result, message = Assertions.check_verbose(actual, baseline) + assert result is False + assert "4" in message + assert "5" in message + + def test_check_verbose_with_callable_baseline_passing(self): + """Test checking with a callable baseline that passes""" + actual = {"value": 42} + + def baseline_func(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: (resolve(value) == 42, "Value is 42") + + result, message = Assertions.check_verbose(actual, {"value": baseline_func}) + assert result is True + + def test_check_verbose_with_callable_baseline_failing(self): + """Test checking with a callable baseline that fails""" + actual = {"value": 42} + + def baseline_func(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: (resolve(value) == 100, "Value should be 100") + + result, message = Assertions.check_verbose(actual, {"value": baseline_func}) + assert result is False + assert "Value should be 100" in message + + def test_check_verbose_mixed_dict_with_values_and_callables(self): + """Test checking dict with mixed static values and callables""" + actual = {"static": "value", "dynamic": 42} + + def check_dynamic(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: resolve(value) > 40 + + baseline = {"static": "value", "dynamic": check_dynamic} + result, message = Assertions.check_verbose(actual, baseline) + assert result is True + + def test_check_verbose_complex_nested_structure(self): + """Test checking complex nested structures""" + actual = { + "user": { + "name": "John", + "age": 30, + "hobbies": ["reading", "coding"] + } + } + baseline = { + "user": { + "name": "John", + "age": 30, + "hobbies": ["reading", "coding"] + } + } + result, message = Assertions.check_verbose(actual, baseline) + assert result is True + + def test_check_verbose_complex_nested_structure_with_diff(self): + """Test checking complex nested structures with differences""" + actual = { + "user": { + "name": "John", + "age": 30, + "hobbies": ["reading", "coding"] + } + } + baseline = { + "user": { + "name": "Jane", + "age": 30, + "hobbies": ["reading", "gaming"] + } + } + result, message = Assertions.check_verbose(actual, baseline) + assert result is False + assert "John" in message or "Jane" in message + + +class TestAssertionsCheck: + """Test the Assertions.check method.""" + + def test_check_returns_true_for_equal_values(self): + """Test that check returns True for equal values""" + assert Assertions.check(42, 42) is True + assert Assertions.check("test", "test") is True + assert Assertions.check([1, 2, 3], [1, 2, 3]) is True + + def test_check_returns_false_for_unequal_values(self): + """Test that check returns False for unequal values""" + assert Assertions.check(42, 43) is False + assert Assertions.check("test", "other") is False + assert Assertions.check([1, 2, 3], [1, 2, 4]) is False + + def test_check_with_dict(self): + """Test check with dictionary structures""" + actual = {"key": "value"} + baseline = {"key": "value"} + assert Assertions.check(actual, baseline) is True + + baseline = {"key": "other"} + assert Assertions.check(actual, baseline) is False + + def test_check_with_nested_structures(self): + """Test check with nested structures""" + actual = {"outer": {"inner": "value"}} + baseline = {"outer": {"inner": "value"}} + assert Assertions.check(actual, baseline) is True + + baseline = {"outer": {"inner": "other"}} + assert Assertions.check(actual, baseline) is False + + def test_check_with_callable(self): + """Test check with callable baseline""" + actual = {"value": 42} + + def baseline_func(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: resolve(value) == 42 + + baseline = {"value": baseline_func} + assert Assertions.check(actual, baseline) is True + + +class TestAssertionsValidate: + """Test the Assertions.validate method.""" + + def test_validate_passes_for_equal_values(self): + """Test that validate does not raise for equal values""" + Assertions.validate(42, 42) + Assertions.validate("test", "test") + Assertions.validate([1, 2, 3], [1, 2, 3]) + + def test_validate_raises_for_unequal_values(self): + """Test that validate raises AssertionError for unequal values""" + with pytest.raises(AssertionError): + Assertions.validate(42, 43) + + def test_validate_raises_for_unequal_strings(self): + """Test that validate raises AssertionError for unequal strings""" + with pytest.raises(AssertionError) as exc_info: + Assertions.validate("hello", "world") + assert "hello" in str(exc_info.value) or "world" in str(exc_info.value) + + def test_validate_raises_for_unequal_dicts(self): + """Test that validate raises AssertionError for unequal dicts""" + actual = {"key": "value1"} + baseline = {"key": "value2"} + with pytest.raises(AssertionError) as exc_info: + Assertions.validate(actual, baseline) + assert "value1" in str(exc_info.value) or "value2" in str(exc_info.value) + + def test_validate_passes_for_complex_equal_structures(self): + """Test that validate passes for complex equal structures""" + actual = { + "user": { + "name": "John", + "age": 30, + "hobbies": ["reading", "coding"] + } + } + baseline = { + "user": { + "name": "John", + "age": 30, + "hobbies": ["reading", "coding"] + } + } + Assertions.validate(actual, baseline) + + def test_validate_raises_for_complex_unequal_structures(self): + """Test that validate raises for complex unequal structures""" + actual = { + "user": { + "name": "John", + "age": 30 + } + } + baseline = { + "user": { + "name": "Jane", + "age": 30 + } + } + with pytest.raises(AssertionError): + Assertions.validate(actual, baseline) + + def test_validate_with_callable_baseline_passing(self): + """Test validate with callable baseline that passes""" + actual = {"value": 42} + + def baseline_func(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: resolve(value) == 42 + + baseline = {"value": baseline_func} + Assertions.validate(actual, baseline) + + def test_validate_with_callable_baseline_failing(self): + """Test validate with callable baseline that fails""" + actual = {"value": 42} + + def baseline_func(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: (resolve(value) == 100, "Expected value to be 100") + + baseline = {"value": baseline_func} + with pytest.raises(AssertionError) as exc_info: + Assertions.validate(actual, baseline) + assert "Expected value to be 100" in str(exc_info.value) + + +class TestAssertionsIntegration: + """Integration tests for Assertions class covering complex scenarios.""" + + def test_integration_nested_dict_with_lists(self): + """Test checking nested dicts containing lists""" + actual = { + "users": [ + {"name": "Alice", "age": 25}, + {"name": "Bob", "age": 30} + ] + } + baseline = { + "users": [ + {"name": "Alice", "age": 25}, + {"name": "Bob", "age": 30} + ] + } + assert Assertions.check(actual, baseline) is True + + def test_integration_mixed_callables_and_values(self): + """Test mixing callable checks and static values""" + actual = { + "id": 123, + "name": "test", + "score": 95 + } + + def check_id(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: resolve(value) > 0 + + def check_score(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: resolve(value) >= 90 + + baseline = { + "id": check_id, + "name": "test", + "score": check_score + } + assert Assertions.check(actual, baseline) is True + + def test_integration_deep_nesting(self): + """Test deeply nested structures""" + actual = { + "level1": { + "level2": { + "level3": { + "level4": "deep_value" + } + } + } + } + baseline = { + "level1": { + "level2": { + "level3": { + "level4": "deep_value" + } + } + } + } + assert Assertions.check(actual, baseline) is True + + def test_integration_list_of_dicts_with_callables(self): + """Test list of dicts with callable checks""" + actual = [ + {"value": 10}, + {"value": 20}, + {"value": 30} + ] + + def check_value(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: resolve(value) >= 10 + + baseline = [ + {"value": check_value}, + {"value": check_value}, + {"value": check_value} + ] + assert Assertions.check(actual, baseline) is True + + def test_integration_empty_structures(self): + """Test empty structures""" + assert Assertions.check({}, {}) is True + assert Assertions.check([], []) is True + + def test_integration_none_values(self): + """Test handling of None values""" + actual = {"key": None} + baseline = {"key": None} + assert Assertions.check(actual, baseline) is True + + baseline = {"key": "not_none"} + assert Assertions.check(actual, baseline) is False \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/assertions/test_check_field.py b/dev/microsoft-agents-testing/tests/assertions/test_check_field.py deleted file mode 100644 index cafc556d..00000000 --- a/dev/microsoft-agents-testing/tests/assertions/test_check_field.py +++ /dev/null @@ -1,296 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import pytest - -from microsoft_agents.testing.assertions.check_field import ( - check_field, - _parse_assertion, -) -from microsoft_agents.testing.assertions.type_defs import FieldAssertionType - - -class TestParseAssertion: - - @pytest.fixture( - params=[ - FieldAssertionType.EQUALS, - FieldAssertionType.NOT_EQUALS, - FieldAssertionType.GREATER_THAN, - ] - ) - def assertion_type_str(self, request): - return request.param - - @pytest.fixture(params=["simple_value", {"key": "value"}, 42]) - def assertion_value(self, request): - return request.param - - def test_parse_assertion_dict(self, assertion_value, assertion_type_str): - - assertion, assertion_type = _parse_assertion( - {"assertion_type": assertion_type_str, "assertion": assertion_value} - ) - assert assertion == assertion_value - assert assertion_type == FieldAssertionType(assertion_type_str) - - def test_parse_assertion_list(self, assertion_value, assertion_type_str): - assertion, assertion_type = _parse_assertion( - [assertion_type_str, assertion_value] - ) - assert assertion == assertion_value - assert assertion_type.value == assertion_type_str - - @pytest.mark.parametrize( - "field", - ["value", 123, 12.34], - ) - def test_parse_assertion_default(self, field): - assertion, assertion_type = _parse_assertion(field) - assert assertion == field - assert assertion_type == FieldAssertionType.EQUALS - - @pytest.mark.parametrize( - "field", - [ - {"assertion_type": FieldAssertionType.IN}, - {"assertion_type": FieldAssertionType.IN, "key": "value"}, - [FieldAssertionType.RE_MATCH], - [], - {"assertion_type": "invalid", "assertion": "test"}, - ], - ) - def test_parse_assertion_none(self, field): - assertion, assertion_type = _parse_assertion(field) - assert assertion is None - assert assertion_type is None - - -class TestCheckFieldEquals: - """Tests for EQUALS assertion type.""" - - def test_equals_with_matching_strings(self): - assert check_field("hello", "hello", FieldAssertionType.EQUALS) is True - - def test_equals_with_non_matching_strings(self): - assert check_field("hello", "world", FieldAssertionType.EQUALS) is False - - def test_equals_with_matching_integers(self): - assert check_field(42, 42, FieldAssertionType.EQUALS) is True - - def test_equals_with_non_matching_integers(self): - assert check_field(42, 43, FieldAssertionType.EQUALS) is False - - def test_equals_with_none_values(self): - assert check_field(None, None, FieldAssertionType.EQUALS) is True - - def test_equals_with_boolean_values(self): - assert check_field(True, True, FieldAssertionType.EQUALS) is True - assert check_field(False, False, FieldAssertionType.EQUALS) is True - assert check_field(True, False, FieldAssertionType.EQUALS) is False - - -class TestCheckFieldNotEquals: - """Tests for NOT_EQUALS assertion type.""" - - def test_not_equals_with_different_strings(self): - assert check_field("hello", "world", FieldAssertionType.NOT_EQUALS) is True - - def test_not_equals_with_matching_strings(self): - assert check_field("hello", "hello", FieldAssertionType.NOT_EQUALS) is False - - def test_not_equals_with_different_integers(self): - assert check_field(42, 43, FieldAssertionType.NOT_EQUALS) is True - - def test_not_equals_with_matching_integers(self): - assert check_field(42, 42, FieldAssertionType.NOT_EQUALS) is False - - -class TestCheckFieldGreaterThan: - """Tests for GREATER_THAN assertion type.""" - - def test_greater_than_with_larger_value(self): - assert check_field(10, 5, FieldAssertionType.GREATER_THAN) is True - - def test_greater_than_with_smaller_value(self): - assert check_field(5, 10, FieldAssertionType.GREATER_THAN) is False - - def test_greater_than_with_equal_value(self): - assert check_field(10, 10, FieldAssertionType.GREATER_THAN) is False - - def test_greater_than_with_floats(self): - assert check_field(10.5, 10.2, FieldAssertionType.GREATER_THAN) is True - assert check_field(10.2, 10.5, FieldAssertionType.GREATER_THAN) is False - - def test_greater_than_with_negative_numbers(self): - assert check_field(-5, -10, FieldAssertionType.GREATER_THAN) is True - assert check_field(-10, -5, FieldAssertionType.GREATER_THAN) is False - - -class TestCheckFieldLessThan: - """Tests for LESS_THAN assertion type.""" - - def test_less_than_with_smaller_value(self): - assert check_field(5, 10, FieldAssertionType.LESS_THAN) is True - - def test_less_than_with_larger_value(self): - assert check_field(10, 5, FieldAssertionType.LESS_THAN) is False - - def test_less_than_with_equal_value(self): - assert check_field(10, 10, FieldAssertionType.LESS_THAN) is False - - def test_less_than_with_floats(self): - assert check_field(10.2, 10.5, FieldAssertionType.LESS_THAN) is True - assert check_field(10.5, 10.2, FieldAssertionType.LESS_THAN) is False - - -class TestCheckFieldContains: - """Tests for CONTAINS assertion type.""" - - def test_contains_substring_in_string(self): - assert check_field("hello world", "world", FieldAssertionType.CONTAINS) is True - - def test_contains_substring_not_in_string(self): - assert check_field("hello world", "foo", FieldAssertionType.CONTAINS) is False - - def test_contains_element_in_list(self): - assert check_field([1, 2, 3, 4], 3, FieldAssertionType.CONTAINS) is True - - def test_contains_element_not_in_list(self): - assert check_field([1, 2, 3, 4], 5, FieldAssertionType.CONTAINS) is False - - def test_contains_key_in_dict(self): - assert check_field({"a": 1, "b": 2}, "a", FieldAssertionType.CONTAINS) is True - - def test_contains_key_not_in_dict(self): - assert check_field({"a": 1, "b": 2}, "c", FieldAssertionType.CONTAINS) is False - - def test_contains_empty_string(self): - assert check_field("hello", "", FieldAssertionType.CONTAINS) is True - - -class TestCheckFieldNotContains: - """Tests for NOT_CONTAINS assertion type.""" - - def test_not_contains_substring_not_in_string(self): - assert ( - check_field("hello world", "foo", FieldAssertionType.NOT_CONTAINS) is True - ) - - def test_not_contains_substring_in_string(self): - assert ( - check_field("hello world", "world", FieldAssertionType.NOT_CONTAINS) - is False - ) - - def test_not_contains_element_not_in_list(self): - assert check_field([1, 2, 3, 4], 5, FieldAssertionType.NOT_CONTAINS) is True - - def test_not_contains_element_in_list(self): - assert check_field([1, 2, 3, 4], 3, FieldAssertionType.NOT_CONTAINS) is False - - -class TestCheckFieldReMatch: - """Tests for RE_MATCH assertion type.""" - - def test_re_match_simple_pattern(self): - assert check_field("hello123", r"hello\d+", FieldAssertionType.RE_MATCH) is True - - def test_re_match_no_match(self): - assert check_field("hello", r"\d+", FieldAssertionType.RE_MATCH) is False - - def test_re_match_email_pattern(self): - pattern = r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" - assert ( - check_field("test@example.com", pattern, FieldAssertionType.RE_MATCH) - is True - ) - assert ( - check_field("invalid-email", pattern, FieldAssertionType.RE_MATCH) is False - ) - - def test_re_match_anchored_pattern(self): - assert ( - check_field("hello world", r"^hello", FieldAssertionType.RE_MATCH) is True - ) - assert ( - check_field("hello world", r"^world", FieldAssertionType.RE_MATCH) is False - ) - - def test_re_match_full_string(self): - assert check_field("abc", r"^abc$", FieldAssertionType.RE_MATCH) is True - assert check_field("abcd", r"^abc$", FieldAssertionType.RE_MATCH) is False - - def test_re_match_case_sensitive(self): - assert check_field("Hello", r"hello", FieldAssertionType.RE_MATCH) is False - assert check_field("Hello", r"Hello", FieldAssertionType.RE_MATCH) is True - - -class TestCheckFieldEdgeCases: - """Tests for edge cases and error handling.""" - - def test_invalid_assertion_type(self): - # Passing an unsupported assertion type should return False - with pytest.raises(ValueError): - assert check_field("test", "test", "INVALID_TYPE") - - def test_none_actual_value_with_equals(self): - assert check_field(None, "test", FieldAssertionType.EQUALS) is False - assert check_field(None, None, FieldAssertionType.EQUALS) is True - - def test_empty_string_comparisons(self): - assert check_field("", "", FieldAssertionType.EQUALS) is True - assert check_field("", "test", FieldAssertionType.EQUALS) is False - - def test_empty_list_contains(self): - assert check_field([], "item", FieldAssertionType.CONTAINS) is False - - def test_zero_comparisons(self): - assert check_field(0, 0, FieldAssertionType.EQUALS) is True - assert check_field(0, 1, FieldAssertionType.LESS_THAN) is True - assert check_field(0, -1, FieldAssertionType.GREATER_THAN) is True - - def test_type_mismatch_comparisons(self): - # Different types should work with equality checks - assert check_field("42", 42, FieldAssertionType.EQUALS) is False - assert check_field("42", 42, FieldAssertionType.NOT_EQUALS) is True - - def test_complex_data_structures(self): - actual = {"nested": {"value": 123}} - expected = {"nested": {"value": 123}} - assert check_field(actual, expected, FieldAssertionType.EQUALS) is True - - def test_list_equality(self): - assert check_field([1, 2, 3], [1, 2, 3], FieldAssertionType.EQUALS) is True - assert check_field([1, 2, 3], [3, 2, 1], FieldAssertionType.EQUALS) is False - - -class TestCheckFieldWithRealWorldScenarios: - """Tests simulating real-world usage scenarios.""" - - def test_validate_response_status_code(self): - assert check_field(200, 200, FieldAssertionType.EQUALS) is True - assert check_field(404, 200, FieldAssertionType.NOT_EQUALS) is True - - def test_validate_response_contains_keyword(self): - response = "Success: Operation completed successfully" - assert check_field(response, "Success", FieldAssertionType.CONTAINS) is True - assert check_field(response, "Error", FieldAssertionType.NOT_CONTAINS) is True - - def test_validate_numeric_threshold(self): - temperature = 72.5 - assert check_field(temperature, 100, FieldAssertionType.LESS_THAN) is True - assert check_field(temperature, 0, FieldAssertionType.GREATER_THAN) is True - - def test_validate_message_format(self): - message_id = "msg_20250112_001" - pattern = r"^msg_\d{8}_\d{3}$" - assert check_field(message_id, pattern, FieldAssertionType.RE_MATCH) is True - - def test_validate_list_membership(self): - allowed_roles = ["admin", "user", "guest"] - assert check_field(allowed_roles, "admin", FieldAssertionType.CONTAINS) is True - assert ( - check_field(allowed_roles, "superuser", FieldAssertionType.NOT_CONTAINS) - is True - ) diff --git a/dev/microsoft-agents-testing/tests/assertions/test_fixtures.py b/dev/microsoft-agents-testing/tests/assertions/test_fixtures.py new file mode 100644 index 00000000..32cf9478 --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/test_fixtures.py @@ -0,0 +1,15 @@ +from microsoft_agents.testing import ( + Assertions, + Fixtures +) + +def test_actual(): + + Assertions.validate( + { + "key": 42 + }, + { + "key": Fixtures.actual == 42 + } + ) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/assertions/test_model_assertion.py b/dev/microsoft-agents-testing/tests/assertions/test_model_assertion.py deleted file mode 100644 index 61b6b29e..00000000 --- a/dev/microsoft-agents-testing/tests/assertions/test_model_assertion.py +++ /dev/null @@ -1,626 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import pytest - -from microsoft_agents.activity import Activity -from microsoft_agents.testing import ( - ModelAssertion, - Selector, - AssertionQuantifier, - FieldAssertionType, -) - - -class TestModelAssertionCheckWithQuantifierAll: - """Tests for check() method with ALL quantifier.""" - - @pytest.fixture - def activities(self): - """Create a list of test activities.""" - return [ - Activity(type="message", text="Hello"), - Activity(type="message", text="World"), - Activity(type="event", name="test_event"), - Activity(type="message", text="Goodbye"), - ] - - def test_check_all_matching_activities(self, activities): - """Test that all matching activities pass the assertion.""" - assertion = ModelAssertion( - assertion={"type": "message"}, - selector=Selector(selector={"type": "message"}), - quantifier=AssertionQuantifier.ALL, - ) - passes, error = assertion.check(activities) - assert passes is True - assert error is None - - def test_check_all_with_one_failing_activity(self, activities): - """Test that one failing activity causes ALL assertion to fail.""" - assertion = ModelAssertion( - assertion={"text": "Hello"}, - selector=Selector(selector={"type": "message"}), - quantifier=AssertionQuantifier.ALL, - ) - passes, error = assertion.check(activities) - assert passes is False - assert error is not None - assert "Item did not match the assertion" in error - - def test_check_all_with_empty_selector(self, activities): - """Test ALL quantifier with empty selector (matches all activities).""" - assertion = ModelAssertion( - assertion={"type": "message"}, - selector=Selector(selector={}), - quantifier=AssertionQuantifier.ALL, - ) - passes, error = assertion.check(activities) - # Should fail because not all activities are messages - assert passes is False - - def test_check_all_with_empty_activities(self): - """Test ALL quantifier with empty activities list.""" - assertion = ModelAssertion( - assertion={"type": "message"}, quantifier=AssertionQuantifier.ALL - ) - passes, error = assertion.check([]) - assert passes is True - assert error is None - - def test_check_all_with_complex_assertion(self, activities): - """Test ALL quantifier with complex nested assertion.""" - complex_activities = [ - Activity(type="message", text="Hello", channelData={"id": 1}), - Activity(type="message", text="World", channelData={"id": 2}), - ] - assertion = ModelAssertion( - assertion={"type": "message"}, - selector=Selector(selector={"type": "message"}), - quantifier=AssertionQuantifier.ALL, - ) - passes, error = assertion.check(complex_activities) - assert passes is True - - -class TestModelAssertionCheckWithQuantifierNone: - """Tests for check() method with NONE quantifier.""" - - @pytest.fixture - def activities(self): - """Create a list of test activities.""" - return [ - Activity(type="message", text="Hello"), - Activity(type="message", text="World"), - Activity(type="event", name="test_event"), - ] - - def test_check_none_with_no_matches(self, activities): - """Test NONE quantifier when no activities match.""" - assertion = ModelAssertion( - assertion={"text": "Nonexistent"}, - selector=Selector(selector={"type": "message"}), - quantifier=AssertionQuantifier.NONE, - ) - passes, error = assertion.check(activities) - assert passes is True - assert error is None - - def test_check_none_with_one_match(self, activities): - """Test NONE quantifier fails when one activity matches.""" - assertion = ModelAssertion( - assertion={"text": "Hello"}, - selector=Selector(selector={"type": "message"}), - quantifier=AssertionQuantifier.NONE, - ) - passes, error = assertion.check(activities) - assert passes is False - assert error is not None - assert "Item matched the assertion when none were expected" in error - - def test_check_none_with_all_matching(self, activities): - """Test NONE quantifier fails when all activities match.""" - assertion = ModelAssertion( - assertion={"type": "message"}, - selector=Selector(selector={"type": "message"}), - quantifier=AssertionQuantifier.NONE, - ) - passes, error = assertion.check(activities) - assert passes is False - - def test_check_none_with_empty_activities(self): - """Test NONE quantifier with empty activities list.""" - assertion = ModelAssertion( - assertion={"type": "message"}, quantifier=AssertionQuantifier.NONE - ) - passes, error = assertion.check([]) - assert passes is True - assert error is None - - -class TestModelAssertionCheckWithQuantifierOne: - """Tests for check() method with ONE quantifier.""" - - @pytest.fixture - def activities(self): - """Create a list of test activities.""" - return [ - Activity(type="message", text="First"), - Activity(type="message", text="Second"), - Activity(type="event", name="test_event"), - Activity(type="message", text="Third"), - ] - - def test_check_one_with_exactly_one_match(self, activities): - """Test ONE quantifier passes when exactly one activity matches.""" - assertion = ModelAssertion( - assertion={"text": "First"}, - selector=Selector(selector={"type": "message"}), - quantifier=AssertionQuantifier.ONE, - ) - passes, error = assertion.check(activities) - assert passes is True - assert error is None - - def test_check_one_with_no_matches(self, activities): - """Test ONE quantifier fails when no activities match.""" - assertion = ModelAssertion( - assertion={"text": "Nonexistent"}, - selector=Selector(selector={"type": "message"}), - quantifier=AssertionQuantifier.ONE, - ) - passes, error = assertion.check(activities) - assert passes is False - assert error is not None - assert "Expected exactly one item" in error - assert "found 0" in error - - def test_check_one_with_multiple_matches(self, activities): - """Test ONE quantifier fails when multiple activities match.""" - assertion = ModelAssertion( - assertion={"type": "message"}, - selector=Selector(selector={"type": "message"}), - quantifier=AssertionQuantifier.ONE, - ) - passes, error = assertion.check(activities) - assert passes is False - assert error is not None - assert "Expected exactly one item" in error - assert "found 3" in error - - def test_check_one_with_empty_activities(self): - """Test ONE quantifier with empty activities list.""" - assertion = ModelAssertion( - assertion={"type": "message"}, quantifier=AssertionQuantifier.ONE - ) - passes, error = assertion.check([]) - assert passes is False - assert "found 0" in error - - -class TestModelAssertionCheckWithQuantifierAny: - """Tests for check() method with ANY quantifier.""" - - @pytest.fixture - def activities(self): - """Create a list of test activities.""" - return [ - Activity(type="message", text="Hello"), - Activity(type="message", text="World"), - Activity(type="event", name="test_event"), - ] - - def test_check_any_basic_functionality(self, activities): - """Test that ANY quantifier exists and can be used.""" - # ANY quantifier doesn't have special logic in the current implementation - # but should not cause errors - assertion = ModelAssertion( - assertion={"type": "message"}, quantifier=AssertionQuantifier.ANY - ) - passes, error = assertion.check(activities) - # Based on the implementation, ANY behaves like checking if count > 0 - assert passes is True - assert error is None - - -class TestModelAssertionFromConfig: - """Tests for from_config static method.""" - - def test_from_config_minimal(self): - """Test creating assertion from minimal config.""" - config = {} - assertion = ModelAssertion.from_config(config) - assert assertion._assertion == {} - assert assertion._quantifier == AssertionQuantifier.ALL - - def test_from_config_with_assertion(self): - """Test creating assertion from config with assertion field.""" - config = {"assertion": {"type": "message", "text": "Hello"}} - assertion = ModelAssertion.from_config(config) - assert assertion._assertion == config["assertion"] - - def test_from_config_with_selector(self): - """Test creating assertion from config with selector field.""" - config = {"selector": {"selector": {"type": "message"}, "quantifier": "ALL"}} - assertion = ModelAssertion.from_config(config) - assert assertion._selector is not None - - def test_from_config_with_quantifier(self): - """Test creating assertion from config with quantifier field.""" - config = {"quantifier": "one"} - assertion = ModelAssertion.from_config(config) - assert assertion._quantifier == AssertionQuantifier.ONE - - def test_from_config_with_all_fields(self): - """Test creating assertion from config with all fields.""" - config = { - "assertion": {"type": "message"}, - "selector": { - "selector": {"text": "Hello"}, - "quantifier": "ONE", - "index": 0, - }, - "quantifier": "all", - } - assertion = ModelAssertion.from_config(config) - assert assertion._assertion == {"type": "message"} - assert assertion._quantifier == AssertionQuantifier.ALL - - def test_from_config_with_case_insensitive_quantifier(self): - """Test from_config handles case-insensitive quantifier strings.""" - for quantifier_str in ["all", "ALL", "All", "ONE", "one", "NONE", "none"]: - config = {"quantifier": quantifier_str} - assertion = ModelAssertion.from_config(config) - assert isinstance(assertion._quantifier, AssertionQuantifier) - - def test_from_config_with_complex_assertion(self): - """Test creating assertion from config with complex nested assertion.""" - config = { - "assertion": {"type": "message", "channelData": {"nested": {"value": 123}}}, - "quantifier": "all", - } - assertion = ModelAssertion.from_config(config) - assert assertion._assertion["type"] == "message" - assert assertion._assertion["channelData"]["nested"]["value"] == 123 - - -class TestModelAssertionCombineErrors: - """Tests for _combine_assertion_errors static method.""" - - def test_combine_empty_errors(self): - """Test combining empty error list.""" - result = ModelAssertion._combine_assertion_errors([]) - assert result == "" - - def test_combine_single_error(self): - """Test combining single error.""" - from microsoft_agents.testing.assertions.type_defs import ( - AssertionErrorData, - FieldAssertionType, - ) - - error = AssertionErrorData( - field_path="activity.text", - actual_value="Hello", - assertion="World", - assertion_type=FieldAssertionType.EQUALS, - ) - result = ModelAssertion._combine_assertion_errors([error]) - assert "activity.text" in result - assert "Hello" in result - - def test_combine_multiple_errors(self): - """Test combining multiple errors.""" - from microsoft_agents.testing.assertions.type_defs import ( - AssertionErrorData, - FieldAssertionType, - ) - - errors = [ - AssertionErrorData( - field_path="activity.text", - actual_value="Hello", - assertion="World", - assertion_type=FieldAssertionType.EQUALS, - ), - AssertionErrorData( - field_path="activity.type", - actual_value="message", - assertion="event", - assertion_type=FieldAssertionType.EQUALS, - ), - ] - result = ModelAssertion._combine_assertion_errors(errors) - assert "activity.text" in result - assert "activity.type" in result - assert "\n" in result - - -class TestModelAssertionIntegration: - """Integration tests with realistic scenarios.""" - - @pytest.fixture - def conversation_activities(self): - """Create a realistic conversation flow.""" - return [ - Activity(type="conversationUpdate", name="add_member"), - Activity(type="message", text="Hello bot", from_property={"id": "user1"}), - Activity(type="message", text="Hi there!", from_property={"id": "bot"}), - Activity( - type="message", text="How are you?", from_property={"id": "user1"} - ), - Activity( - type="message", text="I'm doing well!", from_property={"id": "bot"} - ), - Activity(type="typing"), - Activity(type="message", text="Goodbye", from_property={"id": "user1"}), - ] - - def test_assert_all_user_messages_have_from_property(self, conversation_activities): - """Test that all user messages have a from_property.""" - assertion = ModelAssertion( - assertion={"from_property": {"id": "user1"}}, - selector=Selector( - selector={"type": "message", "from_property": {"id": "user1"}}, - ), - quantifier=AssertionQuantifier.ALL, - ) - passes, error = assertion.check(conversation_activities) - assert passes is True - - def test_assert_no_error_messages(self, conversation_activities): - """Test that there are no error messages in the conversation.""" - assertion = ModelAssertion( - assertion={"type": "error"}, - selector=Selector(selector={}), - quantifier=AssertionQuantifier.NONE, - ) - passes, error = assertion.check(conversation_activities) - assert passes is True - - def test_assert_exactly_one_conversation_update(self, conversation_activities): - """Test that there's exactly one conversation update.""" - assertion = ModelAssertion( - assertion={"type": "conversationUpdate"}, - selector=Selector(selector={"type": "conversationUpdate"}), - quantifier=AssertionQuantifier.ONE, - ) - passes, error = assertion.check(conversation_activities) - assert passes is True - - def test_assert_first_message_is_greeting(self, conversation_activities): - """Test that the first message contains a greeting.""" - assertion = ModelAssertion( - assertion={"text": {"assertion_type": "CONTAINS", "assertion": "Hello"}}, - selector=Selector(selector={"type": "message"}, index=0), - quantifier=AssertionQuantifier.ALL, - ) - passes, error = assertion.check(conversation_activities) - assert passes is True - - def test_complex_multi_field_assertion(self, conversation_activities): - """Test complex assertion with multiple fields.""" - assertion = ModelAssertion( - assertion={"type": "message", "from_property": {"id": "bot"}}, - selector=Selector( - selector={"type": "message", "from_property": {"id": "bot"}}, - ), - quantifier=AssertionQuantifier.ALL, - ) - passes, error = assertion.check(conversation_activities) - assert passes is True - - -class TestModelAssertionEdgeCases: - """Tests for edge cases and boundary conditions.""" - - def test_empty_assertion_matches_all(self): - """Test that empty assertion matches all activities.""" - activities = [ - Activity(type="message", text="Hello"), - Activity(type="event", name="test"), - ] - assertion = ModelAssertion(assertion={}, quantifier=AssertionQuantifier.ALL) - passes, error = assertion.check(activities) - assert passes is True - - def test_assertion_with_none_values(self): - """Test assertion with None values.""" - activities = [Activity(type="message")] - assertion = ModelAssertion( - assertion={"text": None}, quantifier=AssertionQuantifier.ALL - ) - passes, error = assertion.check(activities) - # This behavior depends on check_activity implementation - assert isinstance(passes, bool) - - def test_selector_filters_before_assertion(self): - """Test that selector filters activities before assertion check.""" - activities = [ - Activity(type="message", text="Hello"), - Activity(type="event", name="test"), - Activity(type="message", text="World"), - ] - # Selector gets only messages, assertion checks for specific text - assertion = ModelAssertion( - assertion={"text": "Hello"}, - selector=Selector(selector={"type": "message"}, index=0), - quantifier=AssertionQuantifier.ALL, - ) - passes, error = assertion.check(activities) - assert passes is True - - def test_assertion_error_message_format(self): - """Test that error messages are properly formatted.""" - activities = [Activity(type="message", text="Wrong")] - assertion = ModelAssertion( - assertion={"text": "Expected"}, quantifier=AssertionQuantifier.ALL - ) - passes, error = assertion.check(activities) - assert passes is False - assert error is not None - assert "Item did not match the assertion" in error - assert "Error:" in error - - def test_multiple_activities_same_content(self): - """Test handling multiple activities with identical content.""" - activities = [ - Activity(type="message", text="Hello"), - Activity(type="message", text="Hello"), - Activity(type="message", text="Hello"), - ] - assertion = ModelAssertion( - assertion={"text": "Hello"}, quantifier=AssertionQuantifier.ALL - ) - passes, error = assertion.check(activities) - assert passes is True - - def test_assertion_with_unset_fields(self): - """Test assertion against activities with unset fields.""" - activities = [ - Activity(type="message"), # No text field set - ] - assertion = ModelAssertion( - assertion={"type": "message"}, quantifier=AssertionQuantifier.ALL - ) - passes, error = assertion.check(activities) - assert passes is True - - -class TestModelAssertionErrorMessages: - """Tests specifically for error message content and formatting.""" - - def test_all_quantifier_error_includes_activity(self): - """Test that ALL quantifier error includes the failing activity.""" - activities = [Activity(type="message", text="Wrong")] - assertion = ModelAssertion( - assertion={"text": "Expected"}, quantifier=AssertionQuantifier.ALL - ) - passes, error = assertion.check(activities) - assert passes is False - assert "Item did not match the assertion" in error - - def test_none_quantifier_error_includes_activity(self): - """Test that NONE quantifier error includes the matching activity.""" - activities = [Activity(type="message", text="Unexpected")] - assertion = ModelAssertion( - assertion={"text": "Unexpected"}, quantifier=AssertionQuantifier.NONE - ) - passes, error = assertion.check(activities) - assert passes is False - assert "Item matched the assertion when none were expected" in error - - def test_one_quantifier_error_includes_count(self): - """Test that ONE quantifier error includes the actual count.""" - activities = [ - Activity(type="message"), - Activity(type="message"), - ] - assertion = ModelAssertion( - assertion={"type": "message"}, quantifier=AssertionQuantifier.ONE - ) - passes, error = assertion.check(activities) - assert passes is False - assert "Expected exactly one item" in error - assert "2" in error - - -class TestModelAssertionRealWorldScenarios: - """Tests simulating real-world bot testing scenarios.""" - - def test_validate_welcome_message_sent(self): - """Test that a welcome message is sent when user joins.""" - activities = [ - Activity(type="conversationUpdate", name="add_member"), - Activity(type="message", text="Welcome to our bot!"), - ] - assertion = ModelAssertion( - assertion={ - "type": "message", - "text": {"assertion_type": "CONTAINS", "assertion": "Welcome"}, - }, - selector=Selector(selector={"type": "message"}), - quantifier=AssertionQuantifier.ALL, - ) - passes, error = assertion.check(activities) - assert passes is True - - def test_validate_no_duplicate_responses(self): - """Test that bot doesn't send duplicate responses.""" - activities = [ - Activity(type="message", text="Response 1"), - Activity(type="message", text="Response 2"), - Activity(type="message", text="Response 3"), - ] - # Check that exactly one of each unique response exists - for response_text in ["Response 1", "Response 2", "Response 3"]: - assertion = ModelAssertion( - assertion={"text": response_text}, - selector=Selector(selector={"type": "message"}), - quantifier=AssertionQuantifier.ONE, - ) - passes, error = assertion.check(activities) - assert passes is True - - def test_validate_error_handling_response(self): - """Test that bot responds appropriately to errors.""" - activities = [ - Activity(type="message", text="invalid command"), - Activity(type="message", text="I'm sorry, I didn't understand that."), - ] - assertion = ModelAssertion( - assertion={ - "text": { - "assertion_type": "RE_MATCH", - "assertion": "sorry|understand|help", - } - }, - selector=Selector(selector={"type": "message"}, index=-1), # Last message - quantifier=AssertionQuantifier.ALL, - ) - passes, error = assertion.check(activities) - assert not passes - assert "sorry" in error and "understand" in error and "help" in error - assert FieldAssertionType.RE_MATCH.name in error - - def test_validate_typing_indicator_before_response(self): - """Test that typing indicator is sent before response.""" - activities = [ - Activity(type="message", text="User question"), - Activity(type="typing"), - Activity(type="message", text="Bot response"), - ] - # Verify typing indicator exists - typing_assertion = ModelAssertion( - assertion={"type": "typing"}, - selector=Selector(selector={"type": "typing"}), - quantifier=AssertionQuantifier.ONE, - ) - passes, error = typing_assertion.check(activities) - assert passes is True - - def test_validate_conversation_flow_order(self): - """Test that conversation follows expected flow.""" - activities = [ - Activity(type="conversationUpdate"), - Activity(type="message", text="User: Hello"), - Activity(type="typing"), - Activity(type="message", text="Bot: Hi!"), - ] - - # Test each step individually - steps = [ - ({"type": "conversationUpdate"}, 0), - ({"type": "message"}, 1), - ({"type": "typing"}, 2), - ({"type": "message"}, 3), - ] - - for assertion_dict, expected_index in steps: - assertion = ModelAssertion( - assertion=assertion_dict, - selector=Selector(selector={}, index=expected_index), - quantifier=AssertionQuantifier.ALL, - ) - passes, error = assertion.check(activities) - assert passes is True, f"Failed at index {expected_index}: {error}" diff --git a/dev/microsoft-agents-testing/tests/assertions/test_model_query.py b/dev/microsoft-agents-testing/tests/assertions/test_model_query.py new file mode 100644 index 00000000..8af1157b --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/test_model_query.py @@ -0,0 +1,570 @@ +import pytest +from unittest.mock import Mock + +from microsoft_agents.testing.assertions.assertions import Assertions +from microsoft_agents.testing.assertions.assertion_context import AssertionContext +from microsoft_agents.testing.assertions.types import SafeObject, DynamicObject, Unset + + +class TestAssertionsExpand: + """Test the Assertions.expand method for flattening and expanding dictionaries.""" + + def test_expand_non_dict_returns_as_is(self): + """Test that non-dict values are returned unchanged""" + assert Assertions.expand("string") == "string" + assert Assertions.expand(123) == 123 + assert Assertions.expand([1, 2, 3]) == [1, 2, 3] + assert Assertions.expand(None) is None + + def test_expand_flat_dict_no_dots(self): + """Test expansion of a flat dictionary without dots in keys""" + data = {"key1": "value1", "key2": "value2"} + result = Assertions.expand(data) + assert result == {"key1": "value1", "key2": "value2"} + + def test_expand_simple_nested_keys(self): + """Test expansion of simple dotted keys""" + data = {"parent.child": "value"} + result = Assertions.expand(data) + assert result == {"parent": {"child": "value"}} + + def test_expand_multiple_levels(self): + """Test expansion of multiple nested levels""" + data = {"root.level1.level2": "value"} + result = Assertions.expand(data) + assert result == {"root": {"level1": {"level2": "value"}}} + + def test_expand_multiple_keys_same_root(self): + """Test expansion with multiple keys sharing the same root""" + data = { + "parent.child1": "value1", + "parent.child2": "value2" + } + result = Assertions.expand(data) + assert result == { + "parent": { + "child1": "value1", + "child2": "value2" + } + } + + def test_expand_mixed_flat_and_nested(self): + """Test expansion with mixed flat and nested keys""" + data = { + "flat_key": "flat_value", + "nested.key": "nested_value" + } + result = Assertions.expand(data) + assert result == { + "flat_key": "flat_value", + "nested": {"key": "nested_value"} + } + + def test_expand_complex_nested_structure(self): + """Test expansion with complex nested structure""" + data = { + "root.child1": "value1", + "root.child2.grandchild": "value2", + "other": "value3" + } + result = Assertions.expand(data) + assert result == { + "root": { + "child1": "value1", + "child2": {"grandchild": "value2"} + }, + "other": "value3" + } + + def test_expand_recursive_expansion(self): + """Test that expansion is applied recursively""" + data = { + "level1.level2": {"level3.level4": "value"} + } + result = Assertions.expand(data) + assert result == { + "level1": { + "level2": { + "level3": {"level4": "value"} + } + } + } + + def test_expand_duplicate_root_raises_error(self): + """Test that duplicate root keys raise RuntimeError""" + data = { + "root": "value1", + "root.child": "value2" + } + with pytest.raises(RuntimeError): + Assertions.expand(data) + + def test_expand_conflicting_structure_raises_error(self): + """Test that conflicting structures raise RuntimeError""" + data = { + "parent": "value", + "parent.child": "child_value" + } + with pytest.raises(RuntimeError): + Assertions.expand(data) + + def test_expand_empty_dict(self): + """Test expansion of an empty dictionary""" + result = Assertions.expand({}) + assert result == {} + + +class TestAssertionsInvoke: + """Test the Assertions.invoke method for invoking query functions.""" + + def test_invoke_returns_tuple_with_bool_and_message(self): + """Test that invoke properly handles functions returning (bool, str) tuple""" + actual = SafeObject({"value": 42}) + + def query_func(value): + return lambda: (True, "Success message") + + context = AssertionContext(actual, {}) + result, message = Assertions.invoke(actual, query_func, context) + + assert result is True + assert message == "Success message" + + def test_invoke_returns_false_tuple(self): + """Test that invoke handles false results correctly""" + actual = SafeObject({"value": 42}) + + def query_func(value): + return lambda: (False, "Failure message") + + context = AssertionContext(actual, {}) + result, message = Assertions.invoke(actual, query_func, context) + + assert result is False + assert message == "Failure message" + + def test_invoke_returns_bool_only(self): + """Test that invoke converts single bool return to tuple""" + actual = SafeObject({"value": 42}) + + def query_func(value): + return lambda: True + + context = AssertionContext(actual, {}) + result, message = Assertions.invoke(actual, query_func, context) + + assert result is True + assert "query_func" in message + + def test_invoke_returns_falsy_value(self): + """Test that invoke handles falsy non-tuple values""" + actual = SafeObject({"value": 42}) + + def query_func(value): + return lambda: 0 + + context = AssertionContext(actual, {}) + result, message = Assertions.invoke(actual, query_func, context) + + assert result is False + assert "query_func" in message + + def test_invoke_returns_truthy_value(self): + """Test that invoke handles truthy non-tuple values""" + actual = SafeObject({"value": 42}) + + def query_func(value): + return lambda: 1 + + context = AssertionContext(actual, {}) + result, message = Assertions.invoke(actual, query_func, context) + + assert result is True + assert "query_func" in message + + +class TestAssertionsCheckVerbose: + """Test the Assertions._check_verbose and check_verbose methods.""" + + def test_check_verbose_equal_primitives(self): + """Test checking equal primitive values""" + result, message = Assertions.check_verbose(42, 42) + assert result is True + assert message == "" + + def test_check_verbose_unequal_primitives(self): + """Test checking unequal primitive values""" + result, message = Assertions.check_verbose(42, 43) + assert result is False + assert "42" in message + assert "43" in message + + def test_check_verbose_equal_strings(self): + """Test checking equal strings""" + result, message = Assertions.check_verbose("hello", "hello") + assert result is True + assert message == "" + + def test_check_verbose_unequal_strings(self): + """Test checking unequal strings""" + result, message = Assertions.check_verbose("hello", "world") + assert result is False + assert "hello" in message + assert "world" in message + + def test_check_verbose_equal_dicts(self): + """Test checking equal dictionaries""" + actual = {"key": "value", "number": 42} + baseline = {"key": "value", "number": 42} + result, message = Assertions.check_verbose(actual, baseline) + assert result is True + assert message == "" + + def test_check_verbose_unequal_dicts(self): + """Test checking unequal dictionaries""" + actual = {"key": "value1"} + baseline = {"key": "value2"} + result, message = Assertions.check_verbose(actual, baseline) + assert result is False + assert "value1" in message + assert "value2" in message + + def test_check_verbose_nested_dicts(self): + """Test checking nested dictionaries""" + actual = {"parent": {"child": "value1"}} + baseline = {"parent": {"child": "value2"}} + result, message = Assertions.check_verbose(actual, baseline) + assert result is False + assert "value1" in message + assert "value2" in message + + def test_check_verbose_equal_lists(self): + """Test checking equal lists""" + actual = [1, 2, 3] + baseline = [1, 2, 3] + result, message = Assertions.check_verbose(actual, baseline) + assert result is True + assert message == "" + + def test_check_verbose_unequal_lists(self): + """Test checking unequal lists""" + actual = [1, 2, 3] + baseline = [1, 2, 4] + result, message = Assertions.check_verbose(actual, baseline) + assert result is False + assert "3" in message + assert "4" in message + + def test_check_verbose_nested_lists(self): + """Test checking nested lists""" + actual = [[1, 2], [3, 4]] + baseline = [[1, 2], [3, 5]] + result, message = Assertions.check_verbose(actual, baseline) + assert result is False + assert "4" in message + assert "5" in message + + def test_check_verbose_with_callable_baseline_passing(self): + """Test checking with a callable baseline that passes""" + actual = {"value": 42} + + def baseline_func(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: (resolve(value) == 42, "Value is 42") + + result, message = Assertions.check_verbose(actual, {"value": baseline_func}) + assert result is True + + def test_check_verbose_with_callable_baseline_failing(self): + """Test checking with a callable baseline that fails""" + actual = {"value": 42} + + def baseline_func(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: (resolve(value) == 100, "Value should be 100") + + result, message = Assertions.check_verbose(actual, {"value": baseline_func}) + assert result is False + assert "Value should be 100" in message + + def test_check_verbose_mixed_dict_with_values_and_callables(self): + """Test checking dict with mixed static values and callables""" + actual = {"static": "value", "dynamic": 42} + + def check_dynamic(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: resolve(value) > 40 + + baseline = {"static": "value", "dynamic": check_dynamic} + result, message = Assertions.check_verbose(actual, baseline) + assert result is True + + def test_check_verbose_complex_nested_structure(self): + """Test checking complex nested structures""" + actual = { + "user": { + "name": "John", + "age": 30, + "hobbies": ["reading", "coding"] + } + } + baseline = { + "user": { + "name": "John", + "age": 30, + "hobbies": ["reading", "coding"] + } + } + result, message = Assertions.check_verbose(actual, baseline) + assert result is True + + def test_check_verbose_complex_nested_structure_with_diff(self): + """Test checking complex nested structures with differences""" + actual = { + "user": { + "name": "John", + "age": 30, + "hobbies": ["reading", "coding"] + } + } + baseline = { + "user": { + "name": "Jane", + "age": 30, + "hobbies": ["reading", "gaming"] + } + } + result, message = Assertions.check_verbose(actual, baseline) + assert result is False + assert "John" in message or "Jane" in message + + +class TestAssertionsCheck: + """Test the Assertions.check method.""" + + def test_check_returns_true_for_equal_values(self): + """Test that check returns True for equal values""" + assert Assertions.check(42, 42) is True + assert Assertions.check("test", "test") is True + assert Assertions.check([1, 2, 3], [1, 2, 3]) is True + + def test_check_returns_false_for_unequal_values(self): + """Test that check returns False for unequal values""" + assert Assertions.check(42, 43) is False + assert Assertions.check("test", "other") is False + assert Assertions.check([1, 2, 3], [1, 2, 4]) is False + + def test_check_with_dict(self): + """Test check with dictionary structures""" + actual = {"key": "value"} + baseline = {"key": "value"} + assert Assertions.check(actual, baseline) is True + + baseline = {"key": "other"} + assert Assertions.check(actual, baseline) is False + + def test_check_with_nested_structures(self): + """Test check with nested structures""" + actual = {"outer": {"inner": "value"}} + baseline = {"outer": {"inner": "value"}} + assert Assertions.check(actual, baseline) is True + + baseline = {"outer": {"inner": "other"}} + assert Assertions.check(actual, baseline) is False + + def test_check_with_callable(self): + """Test check with callable baseline""" + actual = {"value": 42} + + def baseline_func(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: resolve(value) == 42 + + baseline = {"value": baseline_func} + assert Assertions.check(actual, baseline) is True + + +class TestAssertionsValidate: + """Test the Assertions.validate method.""" + + def test_validate_passes_for_equal_values(self): + """Test that validate does not raise for equal values""" + Assertions.validate(42, 42) + Assertions.validate("test", "test") + Assertions.validate([1, 2, 3], [1, 2, 3]) + + def test_validate_raises_for_unequal_values(self): + """Test that validate raises AssertionError for unequal values""" + with pytest.raises(AssertionError): + Assertions.validate(42, 43) + + def test_validate_raises_for_unequal_strings(self): + """Test that validate raises AssertionError for unequal strings""" + with pytest.raises(AssertionError) as exc_info: + Assertions.validate("hello", "world") + assert "hello" in str(exc_info.value) or "world" in str(exc_info.value) + + def test_validate_raises_for_unequal_dicts(self): + """Test that validate raises AssertionError for unequal dicts""" + actual = {"key": "value1"} + baseline = {"key": "value2"} + with pytest.raises(AssertionError) as exc_info: + Assertions.validate(actual, baseline) + assert "value1" in str(exc_info.value) or "value2" in str(exc_info.value) + + def test_validate_passes_for_complex_equal_structures(self): + """Test that validate passes for complex equal structures""" + actual = { + "user": { + "name": "John", + "age": 30, + "hobbies": ["reading", "coding"] + } + } + baseline = { + "user": { + "name": "John", + "age": 30, + "hobbies": ["reading", "coding"] + } + } + Assertions.validate(actual, baseline) + + def test_validate_raises_for_complex_unequal_structures(self): + """Test that validate raises for complex unequal structures""" + actual = { + "user": { + "name": "John", + "age": 30 + } + } + baseline = { + "user": { + "name": "Jane", + "age": 30 + } + } + with pytest.raises(AssertionError): + Assertions.validate(actual, baseline) + + def test_validate_with_callable_baseline_passing(self): + """Test validate with callable baseline that passes""" + actual = {"value": 42} + + def baseline_func(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: resolve(value) == 42 + + baseline = {"value": baseline_func} + Assertions.validate(actual, baseline) + + def test_validate_with_callable_baseline_failing(self): + """Test validate with callable baseline that fails""" + actual = {"value": 42} + + def baseline_func(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: (resolve(value) == 100, "Expected value to be 100") + + baseline = {"value": baseline_func} + with pytest.raises(AssertionError) as exc_info: + Assertions.validate(actual, baseline) + assert "Expected value to be 100" in str(exc_info.value) + + +class TestAssertionsIntegration: + """Integration tests for Assertions class covering complex scenarios.""" + + def test_integration_nested_dict_with_lists(self): + """Test checking nested dicts containing lists""" + actual = { + "users": [ + {"name": "Alice", "age": 25}, + {"name": "Bob", "age": 30} + ] + } + baseline = { + "users": [ + {"name": "Alice", "age": 25}, + {"name": "Bob", "age": 30} + ] + } + assert Assertions.check(actual, baseline) is True + + def test_integration_mixed_callables_and_values(self): + """Test mixing callable checks and static values""" + actual = { + "id": 123, + "name": "test", + "score": 95 + } + + def check_id(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: resolve(value) > 0 + + def check_score(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: resolve(value) >= 90 + + baseline = { + "id": check_id, + "name": "test", + "score": check_score + } + assert Assertions.check(actual, baseline) is True + + def test_integration_deep_nesting(self): + """Test deeply nested structures""" + actual = { + "level1": { + "level2": { + "level3": { + "level4": "deep_value" + } + } + } + } + baseline = { + "level1": { + "level2": { + "level3": { + "level4": "deep_value" + } + } + } + } + assert Assertions.check(actual, baseline) is True + + def test_integration_list_of_dicts_with_callables(self): + """Test list of dicts with callable checks""" + actual = [ + {"value": 10}, + {"value": 20}, + {"value": 30} + ] + + def check_value(value): + from microsoft_agents.testing.assertions.types.safe_object import resolve + return lambda: resolve(value) >= 10 + + baseline = [ + {"value": check_value}, + {"value": check_value}, + {"value": check_value} + ] + assert Assertions.check(actual, baseline) is True + + def test_integration_empty_structures(self): + """Test empty structures""" + assert Assertions.check({}, {}) is True + assert Assertions.check([], []) is True + + def test_integration_none_values(self): + """Test handling of None values""" + actual = {"key": None} + baseline = {"key": None} + assert Assertions.check(actual, baseline) is True + + baseline = {"key": "not_none"} + assert Assertions.check(actual, baseline) is False \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/assertions/test_selector.py b/dev/microsoft-agents-testing/tests/assertions/test_selector.py deleted file mode 100644 index fc676639..00000000 --- a/dev/microsoft-agents-testing/tests/assertions/test_selector.py +++ /dev/null @@ -1,309 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import pytest - -from microsoft_agents.activity import Activity -from microsoft_agents.testing.assertions.model_selector import Selector - - -class TestSelectorSelectWithQuantifierAll: - """Tests for select() method with ALL quantifier.""" - - @pytest.fixture - def activities(self): - """Create a list of test activities.""" - return [ - Activity(type="message", text="Hello"), - Activity(type="message", text="World"), - Activity(type="event", name="test_event"), - Activity(type="message", text="Goodbye"), - ] - - def test_select_all_matching_type(self, activities): - """Test selecting all activities with matching type.""" - selector = Selector(selector={"type": "message"}) - result = selector.select(activities) - assert len(result) == 3 - assert all(a.type == "message" for a in result) - - def test_select_all_matching_multiple_fields(self, activities): - """Test selecting all activities matching multiple fields.""" - selector = Selector( - selector={"type": "message", "text": "Hello"}, - ) - result = selector.select(activities) - assert len(result) == 1 - assert result[0].text == "Hello" - - def test_select_all_no_matches(self, activities): - """Test selecting all with no matches returns empty list.""" - selector = Selector( - selector={"type": "nonexistent"}, - ) - result = selector.select(activities) - assert len(result) == 0 - - def test_select_all_empty_selector(self, activities): - """Test selecting all with empty selector returns all activities.""" - selector = Selector(selector={}) - result = selector.select(activities) - assert len(result) == len(activities) - - def test_select_all_from_empty_list(self): - """Test selecting from empty activity list.""" - selector = Selector(selector={"type": "message"}) - result = selector.select([]) - assert len(result) == 0 - - -class TestSelectorSelectWithQuantifierOne: - """Tests for select() method with ONE quantifier.""" - - @pytest.fixture - def activities(self): - """Create a list of test activities.""" - return [ - Activity(type="message", text="First"), - Activity(type="message", text="Second"), - Activity(type="event", name="test_event"), - Activity(type="message", text="Third"), - ] - - def test_select_one_default_index(self, activities): - """Test selecting one activity with default index (0).""" - selector = Selector(selector={"type": "message"}, index=0) - result = selector.select(activities) - assert len(result) == 1 - assert result[0].text == "First" - - def test_select_one_explicit_index(self, activities): - """Test selecting one activity with explicit index.""" - selector = Selector(selector={"type": "message"}, index=1) - result = selector.select(activities) - assert len(result) == 1 - assert result[0].text == "Second" - - def test_select_one_last_index(self, activities): - """Test selecting one activity with last valid index.""" - selector = Selector(selector={"type": "message"}, index=2) - result = selector.select(activities) - assert len(result) == 1 - assert result[0].text == "Third" - - def test_select_one_negative_index(self, activities): - """Test selecting one activity with negative index.""" - selector = Selector(selector={"type": "message"}, index=-1) - result = selector.select(activities) - assert len(result) == 1 - assert result[0].text == "Third" - - def test_select_one_negative_index_from_start(self, activities): - """Test selecting one activity with negative index from start.""" - selector = Selector(selector={"type": "message"}, index=-2) - result = selector.select(activities) - assert len(result) == 1 - assert result[0].text == "Second" - - def test_select_one_index_out_of_range(self, activities): - """Test selecting with index out of range returns empty list.""" - selector = Selector(selector={"type": "message"}, index=10) - result = selector.select(activities) - assert len(result) == 0 - - def test_select_one_negative_index_out_of_range(self, activities): - """Test selecting with negative index out of range returns empty list.""" - selector = Selector(selector={"type": "message"}, index=-10) - result = selector.select(activities) - assert len(result) == 0 - - def test_select_one_no_matches(self, activities): - """Test selecting one with no matches returns empty list.""" - selector = Selector(selector={"type": "nonexistent"}, index=0) - result = selector.select(activities) - assert len(result) == 0 - - def test_select_one_from_empty_list(self): - """Test selecting one from empty list returns empty list.""" - selector = Selector(selector={"type": "message"}, index=0) - result = selector.select([]) - assert len(result) == 0 - - -class TestSelectorSelectFirst: - """Tests for select_first() method.""" - - @pytest.fixture - def activities(self): - """Create a list of test activities.""" - return [ - Activity(type="message", text="First"), - Activity(type="message", text="Second"), - Activity(type="event", name="test_event"), - ] - - def test_select_first_with_matches(self, activities): - """Test select_first returns first matching activity.""" - selector = Selector(selector={"type": "message"}) - result = selector.select_first(activities) - assert result is not None - assert result.text == "First" - - def test_select_first_no_matches(self, activities): - """Test select_first with no matches returns None.""" - selector = Selector( - selector={"type": "nonexistent"}, - ) - result = selector.select_first(activities) - assert result is None - - def test_select_first_empty_list(self): - """Test select_first on empty list returns None.""" - selector = Selector(selector={"type": "message"}) - result = selector.select_first([]) - assert result is None - - def test_select_first_with_one_quantifier(self, activities): - """Test select_first with ONE quantifier and specific index.""" - selector = Selector(selector={"type": "message"}, index=1) - result = selector.select_first(activities) - assert result is not None - assert result.text == "Second" - - -class TestSelectorCallable: - """Tests for __call__ method.""" - - @pytest.fixture - def activities(self): - """Create a list of test activities.""" - return [ - Activity(type="message", text="Hello"), - Activity(type="event", name="test_event"), - ] - - def test_call_invokes_select(self, activities): - """Test that calling selector instance invokes select().""" - selector = Selector(selector={"type": "message"}) - result = selector(activities) - assert len(result) == 1 - assert result[0].text == "Hello" - - def test_call_returns_same_as_select(self, activities): - """Test that __call__ returns same result as select().""" - selector = Selector(selector={"type": "event"}, index=0) - call_result = selector(activities) - select_result = selector.select(activities) - assert call_result == select_result - - -class TestSelectorIntegration: - """Integration tests with realistic scenarios.""" - - @pytest.fixture - def conversation_activities(self): - """Create a realistic conversation flow.""" - return [ - Activity(type="conversationUpdate", name="add_member"), - Activity(type="message", text="Hello bot", from_property={"id": "user1"}), - Activity(type="message", text="Hi there!", from_property={"id": "bot"}), - Activity( - type="message", text="How are you?", from_property={"id": "user1"} - ), - Activity( - type="message", text="I'm doing well!", from_property={"id": "bot"} - ), - Activity(type="typing"), - Activity(type="message", text="Goodbye", from_property={"id": "user1"}), - ] - - def test_select_all_user_messages(self, conversation_activities): - """Test selecting all messages from a specific user.""" - selector = Selector( - selector={"type": "message", "from_property": {"id": "user1"}}, - ) - result = selector.select(conversation_activities) - assert len(result) == 3 - - def test_select_first_bot_response(self, conversation_activities): - """Test selecting first bot response.""" - selector = Selector( - selector={"type": "message", "from_property": {"id": "bot"}}, index=0 - ) - result = selector.select(conversation_activities) - assert len(result) == 1 - assert result[0].text == "Hi there!" - - def test_select_last_message_negative_index(self, conversation_activities): - """Test selecting last message using negative index.""" - selector = Selector(selector={"type": "message"}, index=-1) - result = selector.select(conversation_activities) - assert len(result) == 1 - assert result[0].text == "Goodbye" - - def test_select_typing_indicator(self, conversation_activities): - """Test selecting typing indicator.""" - selector = Selector( - selector={"type": "typing"}, - ) - result = selector.select(conversation_activities) - assert len(result) == 1 - - def test_select_conversation_update(self, conversation_activities): - """Test selecting conversation update events.""" - selector = Selector( - selector={"type": "conversationUpdate"}, - ) - result = selector.select(conversation_activities) - assert len(result) == 1 - assert result[0].name == "add_member" - - -class TestSelectorEdgeCases: - """Tests for edge cases and boundary conditions.""" - - def test_select_with_partial_match(self): - """Test that partial matches work correctly.""" - activities = [ - Activity(type="message", text="Hello", channelData={"id": 1}), - Activity(type="message", text="World"), - ] - # Only matching on type, not text - selector = Selector(selector={"type": "message"}) - result = selector.select(activities) - assert len(result) == 2 - - def test_select_with_none_values(self): - """Test selecting activities with None values.""" - activities = [ - Activity(type="message"), - Activity(type="message", text="Hello"), - ] - selector = Selector( - selector={"type": "message", "text": None}, - ) - result = selector.select(activities) - # This depends on how check_activity handles None - assert isinstance(result, list) - - def test_select_single_activity_list(self): - """Test selecting from list with single activity.""" - activities = [Activity(type="message", text="Only one")] - selector = Selector(selector={"type": "message"}, index=0) - result = selector.select(activities) - assert len(result) == 1 - assert result[0].text == "Only one" - - def test_select_with_boundary_index_zero(self): - """Test selecting with index 0 on single item.""" - activities = [Activity(type="message", text="Single")] - selector = Selector(selector={"type": "message"}, index=0) - result = selector.select(activities) - assert len(result) == 1 - - def test_select_with_boundary_negative_one(self): - """Test selecting with index -1 on single item.""" - activities = [Activity(type="message", text="Single")] - selector = Selector(selector={"type": "message"}, index=-1) - result = selector.select(activities) - assert len(result) == 1 diff --git a/dev/integration/agents/basic_agent/__init__.py b/dev/microsoft-agents-testing/tests/assertions/types/__init__.py similarity index 100% rename from dev/integration/agents/basic_agent/__init__.py rename to dev/microsoft-agents-testing/tests/assertions/types/__init__.py diff --git a/dev/microsoft-agents-testing/tests/assertions/types/test_dynamic_object.py b/dev/microsoft-agents-testing/tests/assertions/types/test_dynamic_object.py new file mode 100644 index 00000000..bfe7b29b --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/types/test_dynamic_object.py @@ -0,0 +1,573 @@ +import pytest + +from microsoft_agents.testing import Unset, DynamicObject +from microsoft_agents.testing.assertions.types.safe_object import resolve, parent + + +class TestDynamicObjectPrimitives: + """Test DynamicObject with primitive types - should return primitives directly.""" + + def test_int_returns_int(self): + result = DynamicObject(42) + assert result == 42 + assert isinstance(result, int) + assert not isinstance(result, DynamicObject) + + def test_float_returns_float(self): + result = DynamicObject(3.14) + assert result == 3.14 + assert isinstance(result, float) + assert not isinstance(result, DynamicObject) + + def test_str_returns_str(self): + result = DynamicObject("hello") + assert result == "hello" + assert isinstance(result, str) + assert not isinstance(result, DynamicObject) + + def test_bool_returns_bool(self): + result_true = DynamicObject(True) + result_false = DynamicObject(False) + assert result_true is True + assert result_false is False + assert isinstance(result_true, bool) + assert not isinstance(result_true, DynamicObject) + + def test_none_returns_none(self): + result = DynamicObject(None) + assert result is None + assert not isinstance(result, DynamicObject) + + def test_unset_returns_unset(self): + result = DynamicObject(Unset) + assert result is Unset + assert not isinstance(result, DynamicObject) + + def test_zero_returns_zero(self): + result = DynamicObject(0) + assert result == 0 + assert isinstance(result, int) + assert not isinstance(result, DynamicObject) + + def test_empty_string_returns_empty_string(self): + result = DynamicObject("") + assert result == "" + assert isinstance(result, str) + assert not isinstance(result, DynamicObject) + + +class TestDynamicObjectDict: + """Test DynamicObject with dictionary values.""" + + def test_dict_creates_dynamic_object(self): + data = {"name": "John", "age": 30} + obj = DynamicObject(data) + assert isinstance(obj, DynamicObject) + assert resolve(obj) == data + + def test_getattr_on_dict(self): + data = {"name": "John", "age": 30} + obj = DynamicObject(data) + # DynamicObject returns primitives directly + assert obj.name == "John" + assert obj.age == 30 + + def test_getattr_missing_returns_unset(self): + data = {"name": "John"} + obj = DynamicObject(data) + result = obj.missing_field + assert result is Unset + + def test_getitem_on_dict(self): + data = {"name": "John", "age": 30} + obj = DynamicObject(data) + assert obj["name"] == "John" + assert obj["age"] == 30 + + def test_getitem_missing_returns_unset(self): + data = {"name": "John"} + obj = DynamicObject(data) + result = obj["missing_key"] + assert result is Unset + + def test_nested_dict_access(self): + data = { + "user": { + "profile": { + "name": "John", + "age": 30 + } + } + } + obj = DynamicObject(data) + assert obj["user"]["profile"]["name"] == "John" + assert obj.user.profile.age == 30 + + def test_empty_dict(self): + obj = DynamicObject({}) + assert isinstance(obj, DynamicObject) + assert resolve(obj) == {} + + +class TestDynamicObjectList: + """Test DynamicObject with list values.""" + + def test_list_creates_dynamic_object(self): + data = [1, 2, 3] + obj = DynamicObject(data) + assert isinstance(obj, DynamicObject) + assert resolve(obj) == data + + def test_getitem_on_list(self): + data = ["a", "b", "c"] + obj = DynamicObject(data) + # List indexing returns wrapped items + assert obj[0] == "a" + assert obj[1] == "b" + assert obj[2] == "c" + + def test_getitem_on_list_out_of_range(self): + data = ["a", "b", "c"] + obj = DynamicObject(data) + with pytest.raises(IndexError): + obj[10] + + def test_list_with_dicts(self): + data = [ + {"name": "John", "age": 30}, + {"name": "Jane", "age": 25} + ] + obj = DynamicObject(data) + assert obj[0]["name"] == "John" + assert obj[1]["age"] == 25 + + def test_list_with_nested_lists(self): + data = [[1, 2], [3, 4], [5, 6]] + obj = DynamicObject(data) + assert obj[0][0] == 1 + assert obj[1][1] == 4 + assert obj[2][0] == 5 + + def test_empty_list(self): + obj = DynamicObject([]) + assert isinstance(obj, DynamicObject) + assert resolve(obj) == [] + + +class TestDynamicObjectCustomClass: + """Test DynamicObject with custom class instances.""" + + def test_custom_class_creates_dynamic_object(self): + class Person: + def __init__(self): + self.name = "John" + self.age = 30 + + person = Person() + obj = DynamicObject(person) + assert isinstance(obj, DynamicObject) + assert resolve(obj) is person + + def test_getattr_on_custom_class(self): + class Person: + def __init__(self): + self.name = "John" + self.age = 30 + + person = Person() + obj = DynamicObject(person) + assert obj.name == "John" + assert obj.age == 30 + + def test_getattr_missing_on_custom_class(self): + class Person: + def __init__(self): + self.name = "John" + + person = Person() + obj = DynamicObject(person) + result = obj.missing_attr + assert result is Unset + + def test_nested_custom_class(self): + class Address: + def __init__(self): + self.street = "123 Main St" + self.city = "Boston" + + class Person: + def __init__(self): + self.name = "John" + self.address = Address() + + person = Person() + obj = DynamicObject(person) + assert obj.address.street == "123 Main St" + assert obj.address.city == "Boston" + + +class TestDynamicObjectContains: + """Test the __contains__ method.""" + + def test_contains_with_dict(self): + data = {"name": "John", "age": 30} + obj = DynamicObject(data) + assert "name" in obj + assert "age" in obj + assert "missing" not in obj + + def test_contains_with_list(self): + data = [1, 2, 3, 4, 5] + obj = DynamicObject(data) + assert 1 in obj + assert 3 in obj + assert 10 not in obj + + def test_contains_with_string(self): + # Strings are primitives and returned directly + result = DynamicObject("hello") + assert "h" in result + assert "x" not in result + + def test_contains_with_set(self): + data = {1, 2, 3, 4, 5} + obj = DynamicObject(data) + assert 1 in obj + assert 10 not in obj + + def test_contains_with_tuple(self): + data = (1, 2, 3) + obj = DynamicObject(data) + assert 1 in obj + assert 10 not in obj + + def test_contains_with_non_iterable(self): + data = 42 + obj = DynamicObject(data) + # 42 is a primitive, returned directly, so this tests int + with pytest.raises(TypeError): + "test" in obj + + def test_contains_with_custom_class(self): + class CustomContainer: + def __contains__(self, item): + return item == "special" + + obj = DynamicObject(CustomContainer()) + assert "special" in obj + assert "other" not in obj + + +class TestDynamicObjectEquality: + """Test the __eq__ method.""" + + def test_equality_with_same_dict(self): + data1 = {"name": "John"} + data2 = {"name": "John"} + obj1 = DynamicObject(data1) + obj2 = DynamicObject(data2) + assert obj1 == obj2 + + def test_equality_with_different_dict(self): + data1 = {"name": "John"} + data2 = {"name": "Jane"} + obj1 = DynamicObject(data1) + obj2 = DynamicObject(data2) + assert obj1 != obj2 + + def test_equality_with_raw_value(self): + data = {"name": "John"} + obj = DynamicObject(data) + assert obj == data + + def test_equality_with_primitives(self): + # Primitives are returned as-is + obj1 = DynamicObject(42) + obj2 = DynamicObject(42) + assert obj1 == obj2 + assert obj1 == 42 + + def test_equality_with_list(self): + data1 = [1, 2, 3] + data2 = [1, 2, 3] + obj1 = DynamicObject(data1) + obj2 = DynamicObject(data2) + assert obj1 == obj2 + + def test_equality_with_custom_class(self): + class Person: + def __init__(self, name): + self.name = name + + def __eq__(self, other): + return isinstance(other, Person) and self.name == other.name + + person1 = Person("John") + person2 = Person("John") + obj1 = DynamicObject(person1) + obj2 = DynamicObject(person2) + assert obj1 == obj2 + + +class TestDynamicObjectBool: + """Test the __bool__ method.""" + + def test_bool_with_non_empty_dict(self): + obj = DynamicObject({"key": "value"}) + assert bool(obj) is True + + def test_bool_with_empty_dict(self): + obj = DynamicObject({}) + assert bool(obj) is False + + def test_bool_with_non_empty_list(self): + obj = DynamicObject([1, 2, 3]) + assert bool(obj) is True + + def test_bool_with_empty_list(self): + obj = DynamicObject([]) + assert bool(obj) is False + + def test_bool_with_custom_class_true(self): + class AlwaysTrue: + def __bool__(self): + return True + + obj = DynamicObject(AlwaysTrue()) + assert bool(obj) is True + + def test_bool_with_custom_class_false(self): + class AlwaysFalse: + def __bool__(self): + return False + + obj = DynamicObject(AlwaysFalse()) + assert bool(obj) is False + + def test_bool_with_unset(self): + # Unset is returned directly as primitive + result = DynamicObject(Unset) + assert bool(result) is False + + def test_bool_with_none(self): + # None is returned directly as primitive + result = DynamicObject(None) + assert bool(result) is False + + +class TestDynamicObjectLen: + """Test the __len__ method.""" + + def test_len_with_dict(self): + obj = DynamicObject({"a": 1, "b": 2, "c": 3}) + assert len(obj) == 3 + + def test_len_with_list(self): + obj = DynamicObject([1, 2, 3, 4, 5]) + assert len(obj) == 5 + + def test_len_with_empty_dict(self): + obj = DynamicObject({}) + assert len(obj) == 0 + + def test_len_with_empty_list(self): + obj = DynamicObject([]) + assert len(obj) == 0 + + def test_len_with_string(self): + # Strings are primitives, returned directly + result = DynamicObject("hello") + assert len(result) == 5 + + def test_len_with_tuple(self): + obj = DynamicObject((1, 2, 3)) + assert len(obj) == 3 + + def test_len_with_set(self): + obj = DynamicObject({1, 2, 3, 4}) + assert len(obj) == 4 + + def test_len_with_non_sized_object(self): + obj = DynamicObject(42) + # 42 is primitive, returned directly + with pytest.raises(TypeError): + len(obj) + + def test_len_with_custom_class(self): + class CustomSized: + def __len__(self): + return 42 + + obj = DynamicObject(CustomSized()) + assert len(obj) == 42 + + +class TestDynamicObjectChaining: + """Test chaining of attribute/item access.""" + + def test_chaining_all_exist(self): + data = { + "level1": { + "level2": { + "level3": "value" + } + } + } + obj = DynamicObject(data) + result = obj["level1"]["level2"]["level3"] + assert result == "value" + + def test_chaining_with_missing(self): + data = { + "level1": { + "level2": {} + } + } + obj = DynamicObject(data) + result = obj["level1"]["level2"]["missing"]["nested"] + assert result is Unset + + def test_mixed_getattr_getitem(self): + class Container: + def __init__(self): + self.data = {"key": "value"} + + container = Container() + obj = DynamicObject(container) + result = obj.data["key"] + assert result == "value" + + def test_deeply_nested_structure(self): + data = { + "a": { + "b": { + "c": { + "d": { + "e": "deep_value" + } + } + } + } + } + obj = DynamicObject(data) + assert obj.a.b.c.d.e == "deep_value" + + +class TestDynamicObjectParent: + """Test parent tracking.""" + + def test_parent_is_none_for_root(self): + obj = DynamicObject({"key": "value"}) + assert parent(obj) is None + + def test_parent_is_set_for_child(self): + data = {"child": {"key": "value"}} + obj = DynamicObject(data) + child = obj["child"] + assert isinstance(child, DynamicObject) + assert parent(child) is obj + + def test_parent_chain(self): + data = {"level1": {"level2": {"level3": "value"}}} + obj = DynamicObject(data) + level1 = obj["level1"] + level2 = level1["level2"] + assert parent(level2) is level1 + assert parent(level1) is obj + + +class TestDynamicObjectEdgeCases: + """Test edge cases and special scenarios.""" + + def test_nested_dynamic_objects(self): + inner = {"value": 42} + outer = {"inner": inner} + obj = DynamicObject(outer) + inner_obj = obj["inner"] + assert isinstance(inner_obj, DynamicObject) + assert resolve(inner_obj) == inner + + def test_complex_nested_structure(self): + data = { + "users": [ + {"name": "John", "age": 30}, + {"name": "Jane", "age": 25} + ], + "count": 2 + } + obj = DynamicObject(data) + assert obj["count"] == 2 + assert obj["users"][0]["name"] == "John" + assert obj["users"][1]["age"] == 25 + + def test_mixed_types_in_list(self): + data = [1, "two", 3.0, True, None, {"key": "value"}] + obj = DynamicObject(data) + assert obj[0] == 1 + assert obj[1] == "two" + assert obj[2] == 3.0 + assert obj[3] is True + assert obj[4] is None + assert obj[5]["key"] == "value" + + def test_wrapping_already_wrapped_object(self): + data = {"key": "value"} + obj1 = DynamicObject(data) + obj2 = DynamicObject(obj1) + # SafeObject.__new__ returns existing SafeObject + assert obj2 is obj1 + + def test_negative_list_indexing(self): + data = [1, 2, 3, 4, 5] + obj = DynamicObject(data) + assert obj[-1] == 5 + assert obj[-2] == 4 + + def test_slice_on_list(self): + data = [1, 2, 3, 4, 5] + obj = DynamicObject(data) + sliced = obj[1:3] + # Slicing returns a DynamicObject wrapping a list + assert isinstance(sliced, DynamicObject) + assert resolve(sliced) == [2, 3] + + +class TestDynamicObjectStringRepresentation: + """Test string representations.""" + + def test_str_with_dict(self): + data = {"name": "John"} + obj = DynamicObject(data) + assert str(obj) == str(data) + + def test_str_with_list(self): + data = [1, 2, 3] + obj = DynamicObject(data) + assert str(obj) == str(data) + + def test_str_with_custom_class(self): + class Person: + def __init__(self): + self.name = "John" + + def __str__(self): + return f"Person({self.name})" + + person = Person() + obj = DynamicObject(person) + assert str(obj) == "Person(John)" + + +class TestDynamicObjectWithParentParameter: + """Test explicit parent parameter.""" + + def test_with_explicit_parent(self): + parent_data = {"parent": "value"} + parent_obj = DynamicObject(parent_data) + child_data = {"child": "value"} + child_obj = DynamicObject(child_data, parent_obj) + assert parent(child_obj) is parent_obj + + def test_with_none_parent(self): + obj = DynamicObject({"key": "value"}, None) + assert parent(obj) is None \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/assertions/types/test_safe_object.py b/dev/microsoft-agents-testing/tests/assertions/types/test_safe_object.py new file mode 100644 index 00000000..8f96390c --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/types/test_safe_object.py @@ -0,0 +1,557 @@ +import pytest + +from microsoft_agents.testing import Unset, SafeObject +from microsoft_agents.testing.assertions.types.safe_object import resolve, parent + + +class TestSafeObjectPrimitives: + """Test SafeObject with primitive types.""" + + def test_int_wrapping(self): + obj = SafeObject(42) + assert isinstance(obj, SafeObject) + assert resolve(obj) == 42 + + def test_float_wrapping(self): + obj = SafeObject(3.14) + assert isinstance(obj, SafeObject) + assert resolve(obj) == 3.14 + + def test_str_wrapping(self): + obj = SafeObject("hello") + assert isinstance(obj, SafeObject) + assert resolve(obj) == "hello" + + def test_bool_wrapping(self): + obj_true = SafeObject(True) + obj_false = SafeObject(False) + assert isinstance(obj_true, SafeObject) + assert isinstance(obj_false, SafeObject) + assert resolve(obj_true) is True + assert resolve(obj_false) is False + + def test_none_wrapping(self): + obj = SafeObject(None) + assert isinstance(obj, SafeObject) + assert resolve(obj) is None + + def test_unset_wrapping(self): + obj = SafeObject(Unset) + assert isinstance(obj, SafeObject) + assert resolve(obj) is Unset + + +class TestSafeObjectDict: + """Test SafeObject with dictionary values.""" + + def test_dict_creates_safe_object(self): + data = {"name": "John", "age": 30} + obj = SafeObject(data) + assert isinstance(obj, SafeObject) + assert resolve(obj) == data + + def test_getattr_on_dict(self): + data = {"name": "John", "age": 30} + obj = SafeObject(data) + name = obj.name + age = obj.age + assert isinstance(name, SafeObject) + assert isinstance(age, SafeObject) + assert resolve(name) == "John" + assert resolve(age) == 30 + + def test_getattr_missing_returns_unset(self): + data = {"name": "John"} + obj = SafeObject(data) + result = obj.missing_field + assert isinstance(result, SafeObject) + assert resolve(result) is Unset + + def test_getitem_on_dict(self): + data = {"name": "John", "age": 30} + obj = SafeObject(data) + name = obj["name"] + age = obj["age"] + assert isinstance(name, SafeObject) + assert isinstance(age, SafeObject) + assert resolve(name) == "John" + assert resolve(age) == 30 + + def test_getitem_missing_returns_unset(self): + data = {"name": "John"} + obj = SafeObject(data) + result = obj["missing_key"] + assert isinstance(result, SafeObject) + assert resolve(result) is Unset + + def test_nested_dict_access(self): + data = { + "user": { + "profile": { + "name": "John", + "age": 30 + } + } + } + obj = SafeObject(data) + name = obj["user"]["profile"]["name"] + age = obj["user"]["profile"]["age"] + assert isinstance(name, SafeObject) + assert isinstance(age, SafeObject) + assert resolve(name) == "John" + assert resolve(age) == 30 + + +class TestSafeObjectCustomClass: + """Test SafeObject with custom class instances.""" + + def test_custom_class_creates_safe_object(self): + class Person: + def __init__(self): + self.name = "John" + self.age = 30 + + person = Person() + obj = SafeObject(person) + assert isinstance(obj, SafeObject) + assert resolve(obj) is person + + def test_getattr_on_custom_class(self): + class Person: + def __init__(self): + self.name = "John" + self.age = 30 + + person = Person() + obj = SafeObject(person) + name = obj.name + age = obj.age + assert isinstance(name, SafeObject) + assert isinstance(age, SafeObject) + assert resolve(name) == "John" + assert resolve(age) == 30 + + def test_getattr_missing_on_custom_class(self): + class Person: + def __init__(self): + self.name = "John" + + person = Person() + obj = SafeObject(person) + result = obj.missing_attr + assert isinstance(result, SafeObject) + assert resolve(result) is Unset + + +class TestSafeObjectList: + """Test SafeObject with list values.""" + + def test_list_creates_safe_object(self): + data = [1, 2, 3] + obj = SafeObject(data) + assert isinstance(obj, SafeObject) + assert resolve(obj) == data + + def test_getitem_on_list(self): + data = ["a", "b", "c"] + obj = SafeObject(data) + item0 = obj[0] + item1 = obj[1] + item2 = obj[2] + assert isinstance(item0, SafeObject) + assert isinstance(item1, SafeObject) + assert isinstance(item2, SafeObject) + assert resolve(item0) == "a" + assert resolve(item1) == "b" + assert resolve(item2) == "c" + + def test_getitem_negative_index(self): + data = ["a", "b", "c"] + obj = SafeObject(data) + last = obj[-1] + assert isinstance(last, SafeObject) + assert resolve(last) == "c" + + def test_getitem_out_of_bounds(self): + data = ["a", "b", "c"] + obj = SafeObject(data) + with pytest.raises(IndexError): + obj[10] + + def test_list_of_dicts(self): + data = [ + {"name": "John", "age": 30}, + {"name": "Jane", "age": 25} + ] + obj = SafeObject(data) + first = obj[0] + assert isinstance(first, SafeObject) + name = first["name"] + assert resolve(name) == "John" + + +class TestSafeObjectResolveFunction: + """Test the resolve function.""" + + def test_resolve_safe_object(self): + data = {"name": "John"} + obj = SafeObject(data) + assert resolve(obj) == data + assert resolve(obj) is data + + def test_resolve_non_safe_object(self): + value = 42 + assert resolve(value) == 42 + assert resolve(value) is value + + def test_resolve_string(self): + value = "hello" + assert resolve(value) == "hello" + + def test_resolve_none(self): + assert resolve(None) is None + + def test_resolve_nested_safe_object(self): + data = {"user": {"name": "John"}} + obj = SafeObject(data) + user_obj = obj["user"] + assert resolve(user_obj) == {"name": "John"} + + +class TestSafeObjectParentTracking: + """Test parent tracking functionality.""" + + def test_root_has_no_parent(self): + data = {"name": "John"} + obj = SafeObject(data) + assert parent(obj) is None + + def test_child_has_parent(self): + data = {"user": {"name": "John"}} + obj = SafeObject(data) + user_obj = obj["user"] + assert parent(user_obj) is obj + + def test_grandchild_has_parent(self): + data = { + "level1": { + "level2": { + "level3": "value" + } + } + } + obj = SafeObject(data) + level2_obj = obj["level1"]["level2"] + level3_obj = level2_obj["level3"] + assert parent(level3_obj) is level2_obj + assert parent(level2_obj) is not obj # level2 parent is level1, not root + + def test_parent_chain(self): + data = {"a": {"b": {"c": "value"}}} + obj = SafeObject(data) + a_obj = obj["a"] + b_obj = a_obj["b"] + c_obj = b_obj["c"] + + assert parent(c_obj) is b_obj + assert parent(b_obj) is a_obj + assert parent(a_obj) is obj + assert parent(obj) is None + + def test_parent_not_set_when_parent_value_is_none(self): + parent_obj = SafeObject(None) + child_obj = SafeObject("child", parent_obj) + assert parent(child_obj) is None + + def test_parent_not_set_when_parent_value_is_unset(self): + parent_obj = SafeObject(Unset) + child_obj = SafeObject("child", parent_obj) + assert parent(child_obj) is None + + +class TestSafeObjectNew: + """Test __new__ behavior.""" + + def test_wrapping_safe_object_returns_same(self): + obj1 = SafeObject(42) + obj2 = SafeObject(obj1) + assert obj2 is obj1 + + def test_wrapping_safe_object_ignores_parent(self): + parent_obj = SafeObject({"key": "value"}) + obj1 = SafeObject(42) + obj2 = SafeObject(obj1, parent_obj) + assert obj2 is obj1 + assert parent(obj2) is None # Original parent is preserved + + +class TestSafeObjectStringRepresentation: + """Test string representations.""" + + def test_str_with_dict(self): + data = {"name": "John"} + obj = SafeObject(data) + assert str(obj) == str(data) + + def test_str_with_primitive(self): + obj = SafeObject(42) + assert str(obj) == "42" + + def test_str_with_unset(self): + obj = SafeObject(Unset) + assert str(obj) == "Unset" + + def test_repr(self): + data = {"name": "John"} + obj = SafeObject(data) + assert repr(obj) == f"SafeObject({data!r})" + + def test_repr_with_primitive(self): + obj = SafeObject(42) + assert repr(obj) == "SafeObject(42)" + + def test_str_with_custom_class(self): + class Person: + def __init__(self): + self.name = "John" + + def __str__(self): + return f"Person({self.name})" + + person = Person() + obj = SafeObject(person) + assert str(obj) == "Person(John)" + + +class TestSafeObjectReadonly: + """Test that SafeObject inherits readonly behavior.""" + + def test_cannot_set_attribute(self): + data = {"name": "John"} + obj = SafeObject(data) + with pytest.raises(AttributeError, match="Cannot set attribute"): + obj.new_attr = "value" + + def test_cannot_delete_attribute(self): + data = {"name": "John"} + obj = SafeObject(data) + with pytest.raises(AttributeError, match="Cannot delete attribute"): + del obj.name + + def test_cannot_set_item(self): + data = {"name": "John"} + obj = SafeObject(data) + with pytest.raises(AttributeError, match="Cannot set item"): + obj["new_key"] = "value" + + def test_cannot_delete_item(self): + data = {"name": "John"} + obj = SafeObject(data) + with pytest.raises(AttributeError, match="Cannot delete item"): + del obj["name"] + + def test_cannot_modify_internal_value(self): + data = {"name": "John"} + obj = SafeObject(data) + with pytest.raises(AttributeError): + obj.__value__ = "new_value" + + def test_cannot_modify_internal_parent(self): + data = {"name": "John"} + obj = SafeObject(data) + with pytest.raises(AttributeError): + obj.__parent__ = None + + +class TestSafeObjectChaining: + """Test chaining of attribute/item access.""" + + def test_chaining_all_exist(self): + data = { + "level1": { + "level2": { + "level3": "value" + } + } + } + obj = SafeObject(data) + result = obj["level1"]["level2"]["level3"] + assert isinstance(result, SafeObject) + assert resolve(result) == "value" + + def test_chaining_with_missing(self): + data = { + "level1": { + "level2": {} + } + } + obj = SafeObject(data) + result = obj["level1"]["level2"]["missing"]["nested"] + # SafeObject should handle missing gracefully + assert isinstance(result, SafeObject) + assert resolve(result) is Unset + + def test_mixed_getattr_getitem(self): + class Container: + def __init__(self): + self.data = {"key": "value"} + + container = Container() + obj = SafeObject(container) + result = obj.data["key"] + assert isinstance(result, SafeObject) + assert resolve(result) == "value" + + def test_chaining_through_unset(self): + data = {"level1": {}} + obj = SafeObject(data) + result = obj["level1"]["missing"]["deep"]["nested"] + # Should chain through Unset values + assert isinstance(result, SafeObject) + assert resolve(result) is Unset + + +class TestSafeObjectEdgeCases: + """Test edge cases and special scenarios.""" + + def test_empty_dict(self): + obj = SafeObject({}) + assert isinstance(obj, SafeObject) + assert resolve(obj) == {} + + def test_empty_string(self): + obj = SafeObject("") + assert isinstance(obj, SafeObject) + assert resolve(obj) == "" + + def test_zero(self): + obj = SafeObject(0) + assert isinstance(obj, SafeObject) + assert resolve(obj) == 0 + + def test_empty_list(self): + obj = SafeObject([]) + assert isinstance(obj, SafeObject) + assert resolve(obj) == [] + + def test_nested_safe_objects_with_parents(self): + data = {"outer": {"inner": {"value": 42}}} + obj = SafeObject(data) + outer_obj = obj["outer"] + inner_obj = outer_obj["inner"] + value_obj = inner_obj["value"] + + assert parent(outer_obj) is obj + assert parent(inner_obj) is outer_obj + assert parent(value_obj) is inner_obj + + def test_complex_nested_structure(self): + data = { + "users": [ + {"name": "John", "age": 30}, + {"name": "Jane", "age": 25} + ], + "count": 2, + "metadata": { + "version": "1.0", + "author": "test" + } + } + obj = SafeObject(data) + + count = obj["count"] + assert resolve(count) == 2 + + users = obj["users"] + first_user = users[0] + first_name = first_user["name"] + assert resolve(first_name) == "John" + + version = obj["metadata"]["version"] + assert resolve(version) == "1.0" + + def test_dict_with_none_values(self): + data = {"key": None} + obj = SafeObject(data) + result = obj["key"] + assert isinstance(result, SafeObject) + assert resolve(result) is None + + def test_accessing_method_on_dict(self): + data = {"name": "John"} + obj = SafeObject(data) + # Accessing a dict method through SafeObject + result = obj.get + assert isinstance(result, SafeObject) + # get is a method of dict, so it should exist + assert resolve(result) is Unset # But accessed as attribute, returns Unset + + +class TestSafeObjectTypeAnnotations: + """Test type-related behavior.""" + + def test_generic_type_preservation(self): + data = {"key": "value"} + obj: SafeObject[dict] = SafeObject(data) + assert isinstance(obj, SafeObject) + + def test_resolve_overload_with_safe_object(self): + obj = SafeObject(42) + result = resolve(obj) + assert result == 42 + + def test_resolve_overload_with_non_safe_object(self): + value = "hello" + result = resolve(value) + assert result == "hello" + + +class TestSafeObjectWithCallables: + """Test SafeObject with callable objects.""" + + def test_wrapping_function(self): + def func(): + return "result" + + obj = SafeObject(func) + assert isinstance(obj, SafeObject) + assert resolve(obj) is func + + def test_wrapping_lambda(self): + lamb = lambda x: x * 2 + obj = SafeObject(lamb) + assert isinstance(obj, SafeObject) + assert resolve(obj) is lamb + + def test_wrapping_class_method(self): + class MyClass: + def method(self): + return "result" + + instance = MyClass() + obj = SafeObject(instance) + method_obj = obj.method + assert isinstance(method_obj, SafeObject) + # The method should be accessible + assert callable(resolve(method_obj)) + + +class TestSafeObjectComparison: + """Test comparison behavior through SafeObject.""" + + def test_str_representation_equality(self): + data1 = {"name": "John"} + data2 = {"name": "John"} + obj1 = SafeObject(data1) + obj2 = SafeObject(data2) + + # String representations should be equal + assert str(obj1) == str(obj2) + + def test_repr_representation_equality(self): + data = {"name": "John"} + obj1 = SafeObject(data) + obj2 = SafeObject(data) + + # repr should show the wrapped value + assert repr(obj1) == repr(obj2) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/assertions/types/test_unset.py b/dev/microsoft-agents-testing/tests/assertions/types/test_unset.py new file mode 100644 index 00000000..53f1a2d2 --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/types/test_unset.py @@ -0,0 +1,36 @@ +import pytest + +from microsoft_agents.testing import Unset + +def test_unset_init_error(): + with pytest.raises(Exception): + Unset() + +def test_unset_ops(): + val = Unset + assert val is Unset + assert val == Unset + assert not val + assert bool(val) is False + assert str(val) == "Unset" + +def test_unset_set(): + with pytest.raises(AttributeError): + Unset.value = 1 + with pytest.raises(AttributeError): + del Unset.value + with pytest.raises(AttributeError): + setattr(Unset, 'value', 1) + with pytest.raises(AttributeError): + delattr(Unset, "value") + with pytest.raises(AttributeError): + Unset["key"] = 1 + with pytest.raises(AttributeError): + del Unset["key"] + +def test_unset_get(): + val = Unset + assert Unset.get("key", None) is Unset + assert val.get("key", None) is Unset + assert getattr(Unset, "key", 42) is Unset + assert val["key"] is Unset \ No newline at end of file diff --git a/dev/integration/agents/basic_agent/python/__init__.py b/dev/microsoft-agents-testing/tests/cli/__init__.py similarity index 100% rename from dev/integration/agents/basic_agent/python/__init__.py rename to dev/microsoft-agents-testing/tests/cli/__init__.py diff --git a/dev/integration/agents/basic_agent/python/src/__init__.py b/dev/microsoft-agents-testing/tests/cli/commands/__init__.py similarity index 100% rename from dev/integration/agents/basic_agent/python/src/__init__.py rename to dev/microsoft-agents-testing/tests/cli/commands/__init__.py diff --git a/dev/integration/agents/basic_agent/python/src/weather/__init__.py b/dev/microsoft-agents-testing/tests/cli/commands/benchmark/__init__.py similarity index 100% rename from dev/integration/agents/basic_agent/python/src/weather/__init__.py rename to dev/microsoft-agents-testing/tests/cli/commands/benchmark/__init__.py diff --git a/dev/integration/agents/basic_agent/python/src/weather/agents/__init__.py b/dev/microsoft-agents-testing/tests/cli/commands/ddt/__init__.py similarity index 100% rename from dev/integration/agents/basic_agent/python/src/weather/agents/__init__.py rename to dev/microsoft-agents-testing/tests/cli/commands/ddt/__init__.py diff --git a/dev/integration/tests/__init__.py b/dev/microsoft-agents-testing/tests/cli/commands/post/__init__.py similarity index 100% rename from dev/integration/tests/__init__.py rename to dev/microsoft-agents-testing/tests/cli/commands/post/__init__.py diff --git a/dev/integration/tests/basic_agent/__init__.py b/dev/microsoft-agents-testing/tests/cli/common/__init__.py similarity index 100% rename from dev/integration/tests/basic_agent/__init__.py rename to dev/microsoft-agents-testing/tests/cli/common/__init__.py diff --git a/dev/integration/tests/quickstart/__init__.py b/dev/microsoft-agents-testing/tests/cli/common/executor/__init__.py similarity index 100% rename from dev/integration/tests/quickstart/__init__.py rename to dev/microsoft-agents-testing/tests/cli/common/executor/__init__.py diff --git a/dev/microsoft-agents-testing/tests/cli/common/executor/test_coroutine_executor.py b/dev/microsoft-agents-testing/tests/cli/common/executor/test_coroutine_executor.py new file mode 100644 index 00000000..7861b20a --- /dev/null +++ b/dev/microsoft-agents-testing/tests/cli/common/executor/test_coroutine_executor.py @@ -0,0 +1,219 @@ +# test_coroutine_executor.py +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest +import asyncio +from unittest.mock import patch, AsyncMock + +from microsoft_agents.testing.cli.common.executor import ( + CoroutineExecutor, + ExecutionResult, +) + + +class TestCoroutineExecutor: + """Tests for CoroutineExecutor class.""" + + @pytest.fixture + def executor(self): + """Create a CoroutineExecutor instance for testing.""" + return CoroutineExecutor() + + def test_initialization(self, executor): + """Test that CoroutineExecutor can be instantiated.""" + assert executor is not None + assert isinstance(executor, CoroutineExecutor) + + def test_run_with_single_worker(self, executor): + """Test run method with a single worker.""" + call_count = 0 + + async def test_func(): + nonlocal call_count + call_count += 1 + return f"result_{call_count}" + + results = executor.run(test_func, num_workers=1) + + assert len(results) == 1 + assert all(isinstance(r, ExecutionResult) for r in results) + assert results[0].result == "result_1" + assert results[0].success is True + assert results[0].exe_id == 0 + + def test_run_with_multiple_workers(self, executor): + """Test run method with multiple workers.""" + call_count = 0 + + async def test_func(): + nonlocal call_count + call_count += 1 + return f"result_{call_count}" + + num_workers = 5 + results = executor.run(test_func, num_workers=num_workers) + + assert len(results) == num_workers + assert all(isinstance(r, ExecutionResult) for r in results) + assert all(r.success for r in results) + # Check that exe_ids are sequential + assert [r.exe_id for r in results] == list(range(num_workers)) + + def test_run_with_failing_function(self, executor): + """Test run method when the function raises an exception.""" + test_error = ValueError("test error") + + async def failing_func(): + raise test_error + + results = executor.run(failing_func, num_workers=1) + + assert len(results) == 1 + assert results[0].success is False + assert results[0].error == test_error + assert results[0].result is None + + def test_run_with_mixed_success_and_failure(self, executor): + """Test run with some workers succeeding and some failing.""" + call_count = 0 + + async def mixed_func(): + nonlocal call_count + call_count += 1 + if call_count % 2 == 0: + raise ValueError(f"Error on call {call_count}") + return f"success_{call_count}" + + results = executor.run(mixed_func, num_workers=4) + + assert len(results) == 4 + success_count = sum(1 for r in results if r.success) + failure_count = sum(1 for r in results if not r.success) + # Due to async nature, we just check that we have both successes and failures + assert success_count > 0 + assert failure_count > 0 + + def test_run_with_async_delay(self, executor): + """Test run with async functions that have delays.""" + async def delayed_func(): + await asyncio.sleep(0.01) + return "completed" + + results = executor.run(delayed_func, num_workers=3) + + assert len(results) == 3 + assert all(r.success for r in results) + assert all(r.result == "completed" for r in results) + assert all(r.duration >= 0.01 for r in results) + + def test_run_records_execution_times(self, executor): + """Test that run properly records execution times.""" + async def test_func(): + await asyncio.sleep(0.05) + return "done" + + results = executor.run(test_func, num_workers=2) + + for result in results: + assert result.start_time > 0 + assert result.end_time > result.start_time + assert result.duration >= 0.05 + + def test_run_with_zero_workers(self, executor): + """Test run with zero workers.""" + async def test_func(): + return "result" + + results = executor.run(test_func, num_workers=0) + + assert len(results) == 0 + assert results == [] + + def test_run_with_large_number_of_workers(self, executor): + """Test run with a large number of concurrent workers.""" + counter = 0 + + async def counting_func(): + nonlocal counter + counter += 1 + await asyncio.sleep(0.001) + return counter + + num_workers = 50 + results = executor.run(counting_func, num_workers=num_workers) + + assert len(results) == num_workers + assert all(r.success for r in results) + assert all(isinstance(r.result, int) for r in results) + + def test_run_with_function_returning_none(self, executor): + """Test run when the function returns None.""" + async def none_func(): + return None + + results = executor.run(none_func, num_workers=2) + + assert len(results) == 2 + assert all(r.success for r in results) + assert all(r.result is None for r in results) + + def test_run_with_function_returning_complex_objects(self, executor): + """Test run with functions returning complex objects.""" + async def complex_func(): + return { + "list": [1, 2, 3], + "dict": {"nested": "value"}, + "tuple": (4, 5, 6), + } + + results = executor.run(complex_func, num_workers=3) + + assert len(results) == 3 + assert all(r.success for r in results) + assert all(isinstance(r.result, dict) for r in results) + assert all("list" in r.result for r in results) + + def test_run_preserves_exception_details(self, executor): + """Test that run preserves exception details.""" + class CustomException(Exception): + def __init__(self, message, code): + super().__init__(message) + self.code = code + + async def custom_error_func(): + raise CustomException("Custom error message", 42) + + results = executor.run(custom_error_func, num_workers=1) + + assert len(results) == 1 + assert results[0].success is False + assert isinstance(results[0].error, CustomException) + assert str(results[0].error) == "Custom error message" + assert results[0].error.code == 42 + + def test_run_execution_ids_are_sequential(self, executor): + """Test that execution IDs are assigned sequentially.""" + async def test_func(): + return "result" + + results = executor.run(test_func, num_workers=10) + + exe_ids = [r.exe_id for r in results] + assert exe_ids == list(range(10)) + + def test_concurrent_execution_performance(self, executor): + """Test that concurrent execution is actually concurrent.""" + import time + + async def sleep_func(): + await asyncio.sleep(0.1) + return "done" + + start = time.time() + results = executor.run(sleep_func, num_workers=5) + duration = time.time() - start + + # If truly concurrent, 5 tasks sleeping 0.1s should take ~0.1s, not 0.5s + assert duration < 0.3 # Give some margin for overhead + assert all(r.success for r in results) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/cli/common/executor/test_execution_result.py b/dev/microsoft-agents-testing/tests/cli/common/executor/test_execution_result.py new file mode 100644 index 00000000..c9411c52 --- /dev/null +++ b/dev/microsoft-agents-testing/tests/cli/common/executor/test_execution_result.py @@ -0,0 +1,127 @@ +# test_execution_result.py +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest + +from microsoft_agents.testing.cli.common.executor import ExecutionResult + + +class TestExecutionResult: + """Tests for ExecutionResult dataclass.""" + + def test_initialization_with_success(self): + """Test creating an ExecutionResult with a successful result.""" + result = ExecutionResult( + exe_id=1, + start_time=100.0, + end_time=105.0, + result="success_value", + ) + assert result.exe_id == 1 + assert result.start_time == 100.0 + assert result.end_time == 105.0 + assert result.result == "success_value" + assert result.error is None + + def test_initialization_with_error(self): + """Test creating an ExecutionResult with an error.""" + test_error = ValueError("test error") + result = ExecutionResult( + exe_id=2, + start_time=100.0, + end_time=105.0, + error=test_error, + ) + assert result.exe_id == 2 + assert result.start_time == 100.0 + assert result.end_time == 105.0 + assert result.result is None + assert result.error == test_error + + def test_success_property_returns_true_when_no_error(self): + """Test that success property returns True when error is None.""" + result = ExecutionResult( + exe_id=1, + start_time=100.0, + end_time=105.0, + result="success", + ) + assert result.success is True + + def test_success_property_returns_false_when_error_exists(self): + """Test that success property returns False when error exists.""" + result = ExecutionResult( + exe_id=1, + start_time=100.0, + end_time=105.0, + error=Exception("error"), + ) + assert result.success is False + + def test_duration_property_calculates_correctly(self): + """Test that duration property calculates the correct time difference.""" + result = ExecutionResult( + exe_id=1, + start_time=100.5, + end_time=105.7, + result="test", + ) + assert result.duration == pytest.approx(5.2) + + def test_duration_property_with_zero_duration(self): + """Test duration property when start and end times are equal.""" + result = ExecutionResult( + exe_id=1, + start_time=100.0, + end_time=100.0, + result="test", + ) + assert result.duration == 0.0 + + def test_duration_property_with_fractional_seconds(self): + """Test duration with fractional seconds.""" + result = ExecutionResult( + exe_id=1, + start_time=100.123456, + end_time=100.987654, + result="test", + ) + expected_duration = 100.987654 - 100.123456 + assert result.duration == pytest.approx(expected_duration) + + def test_result_with_none_values(self): + """Test ExecutionResult with None as result value.""" + result = ExecutionResult( + exe_id=1, + start_time=100.0, + end_time=105.0, + result=None, + ) + assert result.result is None + assert result.success is True # No error means success + + def test_result_with_complex_object(self): + """Test ExecutionResult with complex object as result.""" + complex_result = {"key": "value", "nested": {"data": [1, 2, 3]}} + result = ExecutionResult( + exe_id=1, + start_time=100.0, + end_time=105.0, + result=complex_result, + ) + assert result.result == complex_result + assert result.success is True + + def test_multiple_execution_results_different_ids(self): + """Test creating multiple ExecutionResults with different IDs.""" + result1 = ExecutionResult(exe_id=1, start_time=100.0, end_time=105.0) + result2 = ExecutionResult(exe_id=2, start_time=200.0, end_time=210.0) + result3 = ExecutionResult(exe_id=3, start_time=300.0, end_time=315.0) + + assert result1.exe_id == 1 + assert result2.exe_id == 2 + assert result3.exe_id == 3 + assert result1.duration == 5.0 + assert result2.duration == 10.0 + assert result3.duration == 15.0 \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/cli/common/executor/test_executor.py b/dev/microsoft-agents-testing/tests/cli/common/executor/test_executor.py new file mode 100644 index 00000000..09475d33 --- /dev/null +++ b/dev/microsoft-agents-testing/tests/cli/common/executor/test_executor.py @@ -0,0 +1,180 @@ +# test_executor.py +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest +from unittest.mock import AsyncMock, patch, Mock +from datetime import datetime, timezone + +from microsoft_agents.testing.cli.common.executor import Executor, ExecutionResult + + +class ConcreteExecutor(Executor): + """Concrete implementation of Executor for testing.""" + + def run(self, func, num_workers=1): + """Minimal implementation for testing.""" + return [] + + +class TestExecutor: + """Tests for the Executor base class.""" + + @pytest.fixture + def executor(self): + """Create a concrete executor instance for testing.""" + return ConcreteExecutor() + + @pytest.mark.asyncio + async def test_run_func_with_successful_execution(self, executor): + """Test run_func with a successful async function.""" + async def success_func(): + return "success" + + result = await executor.run_func(1, success_func) + + assert isinstance(result, ExecutionResult) + assert result.exe_id == 1 + assert result.result == "success" + assert result.error is None + assert result.success is True + assert result.start_time > 0 + assert result.end_time > result.start_time + + @pytest.mark.asyncio + async def test_run_func_with_exception(self, executor): + """Test run_func when the async function raises an exception.""" + test_exception = ValueError("test error") + + async def failing_func(): + raise test_exception + + result = await executor.run_func(2, failing_func) + + assert isinstance(result, ExecutionResult) + assert result.exe_id == 2 + assert result.result is None + assert result.error == test_exception + assert result.success is False + assert result.start_time > 0 + assert result.end_time > result.start_time + + @pytest.mark.asyncio + async def test_run_func_execution_time_tracking(self, executor): + """Test that run_func correctly tracks execution time.""" + import asyncio + + async def slow_func(): + await asyncio.sleep(0.1) + return "done" + + result = await executor.run_func(3, slow_func) + + assert result.duration >= 0.1 + assert result.success is True + assert result.result == "done" + + @pytest.mark.asyncio + async def test_run_func_with_different_exe_ids(self, executor): + """Test run_func with different execution IDs.""" + async def test_func(): + return "result" + + result1 = await executor.run_func(1, test_func) + result2 = await executor.run_func(42, test_func) + result3 = await executor.run_func(999, test_func) + + assert result1.exe_id == 1 + assert result2.exe_id == 42 + assert result3.exe_id == 999 + + @pytest.mark.asyncio + async def test_run_func_with_none_return_value(self, executor): + """Test run_func when the function returns None.""" + async def none_func(): + return None + + result = await executor.run_func(1, none_func) + + assert result.result is None + assert result.success is True + assert result.error is None + + @pytest.mark.asyncio + async def test_run_func_with_complex_return_value(self, executor): + """Test run_func with complex return values.""" + complex_value = {"data": [1, 2, 3], "nested": {"key": "value"}} + + async def complex_func(): + return complex_value + + result = await executor.run_func(1, complex_func) + + assert result.result == complex_value + assert result.success is True + + @pytest.mark.asyncio + async def test_run_func_timestamps_use_utc(self, executor): + """Test that run_func uses UTC timezone for timestamps.""" + async def test_func(): + return "result" + + with patch('microsoft_agents.testing.cli.common.executor.executor.datetime') as mock_datetime: + mock_now = Mock() + mock_now.timestamp.return_value = 1234567890.0 + mock_datetime.now.return_value = mock_now + + result = await executor.run_func(1, test_func) + + # Verify that datetime.now was called with timezone.utc + assert mock_datetime.now.call_count == 2 + mock_datetime.now.assert_any_call(timezone.utc) + + @pytest.mark.asyncio + async def test_run_func_catches_all_exception_types(self, executor): + """Test that run_func catches different types of exceptions.""" + exceptions = [ + ValueError("value error"), + TypeError("type error"), + RuntimeError("runtime error"), + Exception("generic exception"), + ] + + for exc in exceptions: + async def failing_func(): + raise exc + + result = await executor.run_func(1, failing_func) + assert result.error == exc + assert result.success is False + + def test_run_method_not_implemented(self, executor): + """Test that the run method raises NotImplementedError on base class.""" + # Create an instance of the abstract class without implementing run + class IncompleteExecutor(Executor): + pass + + # This should work because Python doesn't enforce abstract methods at instantiation + # unless we use ABCMeta, but the method should still raise NotImplementedError + with pytest.raises(NotImplementedError): + # Call the base class run method directly + Executor.run(executor, lambda: None, 1) + + def test_executor_is_abstract_base_class(self): + """Test that Executor is defined as an abstract base class.""" + from abc import ABC + assert issubclass(Executor, ABC) + + @pytest.mark.asyncio + async def test_run_func_with_async_generator(self, executor): + """Test run_func with an async function that yields values.""" + async def generator_func(): + values = [] + for i in range(3): + values.append(i) + return values + + result = await executor.run_func(1, generator_func) + + assert result.result == [0, 1, 2] + assert result.success is True \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/cli/common/executor/test_thread_executor.py b/dev/microsoft-agents-testing/tests/cli/common/executor/test_thread_executor.py new file mode 100644 index 00000000..eb5fed8b --- /dev/null +++ b/dev/microsoft-agents-testing/tests/cli/common/executor/test_thread_executor.py @@ -0,0 +1,299 @@ +# test_thread_executor.py +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest +import asyncio +import time +import threading +from unittest.mock import patch, MagicMock + +from microsoft_agents.testing.cli.common.executor import ( + ThreadExecutor, + ExecutionResult, +) + + +class TestThreadExecutor: + """Tests for ThreadExecutor class.""" + + @pytest.fixture + def executor(self): + """Create a ThreadExecutor instance for testing.""" + return ThreadExecutor() + + def test_initialization(self, executor): + """Test that ThreadExecutor can be instantiated.""" + assert executor is not None + assert isinstance(executor, ThreadExecutor) + + def test_run_with_single_worker(self, executor): + """Test run method with a single worker thread.""" + call_count = 0 + + async def test_func(): + nonlocal call_count + call_count += 1 + return f"result_{call_count}" + + results = executor.run(test_func, num_workers=1) + + assert len(results) == 1 + assert all(isinstance(r, ExecutionResult) for r in results) + assert results[0].result == "result_1" + assert results[0].success is True + assert results[0].exe_id == 0 + + def test_run_with_multiple_workers(self, executor): + """Test run method with multiple worker threads.""" + call_count = 0 + lock = threading.Lock() + + async def test_func(): + nonlocal call_count + with lock: + call_count += 1 + current = call_count + return f"result_{current}" + + num_workers = 5 + results = executor.run(test_func, num_workers=num_workers) + + assert len(results) == num_workers + assert all(isinstance(r, ExecutionResult) for r in results) + assert all(r.success for r in results) + # Check that exe_ids are present + exe_ids = [r.exe_id for r in results] + assert len(exe_ids) == num_workers + assert all(0 <= id < num_workers for id in exe_ids) + + def test_run_with_failing_function(self, executor): + """Test run method when the function raises an exception.""" + test_error = ValueError("test error") + + async def failing_func(): + raise test_error + + results = executor.run(failing_func, num_workers=1) + + assert len(results) == 1 + assert results[0].success is False + assert isinstance(results[0].error, ValueError) + assert str(results[0].error) == "test error" + assert results[0].result is None + + def test_run_with_mixed_success_and_failure(self, executor): + """Test run with some workers succeeding and some failing.""" + call_count = 0 + lock = threading.Lock() + + async def mixed_func(): + nonlocal call_count + with lock: + call_count += 1 + current = call_count + if current % 2 == 0: + raise ValueError(f"Error on call {current}") + return f"success_{current}" + + results = executor.run(mixed_func, num_workers=6) + + assert len(results) == 6 + success_count = sum(1 for r in results if r.success) + failure_count = sum(1 for r in results if not r.success) + # Should have both successes and failures + assert success_count == 3 + assert failure_count == 3 + + def test_run_with_async_delay(self, executor): + """Test run with async functions that have delays.""" + async def delayed_func(): + await asyncio.sleep(0.01) + return "completed" + + results = executor.run(delayed_func, num_workers=3) + + assert len(results) == 3 + assert all(r.success for r in results) + assert all(r.result == "completed" for r in results) + assert all(r.duration >= 0.01 for r in results) + + def test_run_records_execution_times(self, executor): + """Test that run properly records execution times.""" + async def test_func(): + await asyncio.sleep(0.05) + return "done" + + results = executor.run(test_func, num_workers=2) + + for result in results: + assert result.start_time > 0 + assert result.end_time > result.start_time + assert result.duration >= 0.05 + + def test_run_with_zero_workers(self, executor): + """Test run with zero workers.""" + async def test_func(): + return "result" + + with pytest.raises(ValueError): + executor.run(test_func, num_workers=0) + + def test_run_with_large_number_of_workers(self, executor): + """Test run with a large number of concurrent worker threads.""" + counter = 0 + lock = threading.Lock() + + async def counting_func(): + nonlocal counter + with lock: + counter += 1 + current = counter + await asyncio.sleep(0.001) + return current + + num_workers = 50 + results = executor.run(counting_func, num_workers=num_workers) + + assert len(results) == num_workers + assert all(r.success for r in results) + assert all(isinstance(r.result, int) for r in results) + + def test_run_with_function_returning_none(self, executor): + """Test run when the function returns None.""" + async def none_func(): + return None + + results = executor.run(none_func, num_workers=2) + + assert len(results) == 2 + assert all(r.success for r in results) + assert all(r.result is None for r in results) + + def test_run_with_function_returning_complex_objects(self, executor): + """Test run with functions returning complex objects.""" + async def complex_func(): + return { + "list": [1, 2, 3], + "dict": {"nested": "value"}, + "tuple": (4, 5, 6), + } + + results = executor.run(complex_func, num_workers=3) + + assert len(results) == 3 + assert all(r.success for r in results) + assert all(isinstance(r.result, dict) for r in results) + assert all("list" in r.result for r in results) + + def test_run_preserves_exception_details(self, executor): + """Test that run preserves exception details.""" + class CustomException(Exception): + def __init__(self, message, code): + super().__init__(message) + self.code = code + + async def custom_error_func(): + raise CustomException("Custom error message", 42) + + results = executor.run(custom_error_func, num_workers=1) + + assert len(results) == 1 + assert results[0].success is False + assert isinstance(results[0].error, CustomException) + assert str(results[0].error) == "Custom error message" + assert results[0].error.code == 42 + + def test_run_uses_thread_pool_executor(self, executor): + """Test that run uses ThreadPoolExecutor.""" + async def test_func(): + return threading.current_thread().name + + results = executor.run(test_func, num_workers=3) + + assert len(results) == 3 + # Thread names should indicate they're from ThreadPoolExecutor + thread_names = [r.result for r in results] + assert all(isinstance(name, str) for name in thread_names) + + def test_run_execution_ids_are_assigned(self, executor): + """Test that execution IDs are assigned correctly.""" + async def test_func(): + return "result" + + results = executor.run(test_func, num_workers=10) + + exe_ids = sorted([r.exe_id for r in results]) + assert exe_ids == list(range(10)) + + def test_concurrent_execution_performance(self, executor): + """Test that concurrent execution using threads is actually concurrent.""" + async def sleep_func(): + await asyncio.sleep(0.1) + return "done" + + start = time.time() + results = executor.run(sleep_func, num_workers=5) + duration = time.time() - start + + # If truly concurrent, 5 tasks sleeping 0.1s should take ~0.1s, not 0.5s + assert duration < 0.5 # Give margin for thread overhead + assert all(r.success for r in results) + + def test_thread_safety_with_shared_state(self, executor): + """Test thread safety when accessing shared state.""" + shared_list = [] + lock = threading.Lock() + + async def append_func(): + with lock: + current_len = len(shared_list) + await asyncio.sleep(0.001) # Simulate some async work + shared_list.append(current_len) + return current_len + + results = executor.run(append_func, num_workers=10) + + assert len(results) == 10 + assert all(r.success for r in results) + assert len(shared_list) == 10 + # All appended values should be unique if thread-safe + assert len(set(shared_list)) == 10 + + def test_run_with_asyncio_operations(self, executor): + """Test that async operations work correctly in threads.""" + async def async_operations(): + # Test various asyncio operations + await asyncio.sleep(0.01) + result = await asyncio.gather( + asyncio.sleep(0.01, result="a"), + asyncio.sleep(0.01, result="b"), + ) + return result + + results = executor.run(async_operations, num_workers=3) + + assert len(results) == 3 + assert all(r.success for r in results) + assert all(r.result == ["a", "b"] for r in results) + + def test_run_with_different_return_types(self, executor): + """Test run with functions returning different types.""" + test_cases = [ + (lambda: 42, int), + (lambda: "string", str), + (lambda: [1, 2, 3], list), + (lambda: {"key": "value"}, dict), + (lambda: (1, 2), tuple), + (lambda: True, bool), + (lambda: 3.14, float), + ] + + for func_body, expected_type in test_cases: + async def async_wrapper(): + return func_body() + + results = executor.run(async_wrapper, num_workers=1) + assert len(results) == 1 + assert results[0].success is True + assert isinstance(results[0].result, expected_type) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/cli/common/test_create_payload_sender.py b/dev/microsoft-agents-testing/tests/cli/common/test_create_payload_sender.py new file mode 100644 index 00000000..3b1907b6 --- /dev/null +++ b/dev/microsoft-agents-testing/tests/cli/common/test_create_payload_sender.py @@ -0,0 +1,310 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import asyncio +import pytest +from unittest.mock import AsyncMock, Mock, patch, MagicMock +from typing import Any + +from microsoft_agents.testing.cli.common.create_payload_sender import ( + create_payload_sender, +) + + +class TestCreatePayloadSender: + """Test suite for create_payload_sender function.""" + + @pytest.fixture + def mock_token(self): + """Fixture for mocked token.""" + return "mock_bearer_token_12345" + + @pytest.fixture + def mock_config(self): + """Fixture for mocked CLI config.""" + with patch( + "microsoft_agents.testing.cli.common.create_payload_sender.cli_config" + ) as mock: + mock.app_id = "test-app-id" + mock.app_secret = "test-app-secret" + mock.tenant_id = "test-tenant-id" + mock.agent_endpoint = "http://localhost:3978/api/messages/" + yield mock + + @pytest.fixture + def sample_payload(self): + """Fixture for sample payload.""" + return { + "type": "message", + "text": "Hello, world!", + "from": {"id": "user1", "name": "Test User"}, + } + + @pytest.mark.asyncio + async def test_create_payload_sender_returns_callable( + self, sample_payload, mock_config, mock_token + ): + """Test that create_payload_sender returns a callable.""" + with patch( + "microsoft_agents.testing.cli.common.create_payload_sender.generate_token", + return_value=mock_token, + ): + sender = create_payload_sender(sample_payload) + assert callable(sender) + + @pytest.mark.asyncio + async def test_payload_sender_generates_token_with_correct_params( + self, sample_payload, mock_config, mock_token + ): + """Test that token generation is called with correct parameters.""" + with patch( + "microsoft_agents.testing.cli.common.create_payload_sender.generate_token", + return_value=mock_token, + ) as mock_generate_token: + create_payload_sender(sample_payload) + + mock_generate_token.assert_called_once_with( + mock_config.app_id, + mock_config.app_secret, + mock_config.tenant_id, + ) + + @pytest.mark.asyncio + async def test_payload_sender_makes_post_request( + self, sample_payload, mock_config, mock_token + ): + """Test that payload sender makes a POST request with correct parameters.""" + mock_response = Mock() + mock_response.content = b"Response content" + + with patch( + "microsoft_agents.testing.cli.common.create_payload_sender.generate_token", + return_value=mock_token, + ), patch( + "microsoft_agents.testing.cli.common.create_payload_sender.requests.post", + return_value=mock_response, + ) as mock_post: + sender = create_payload_sender(sample_payload) + result = await sender() + + mock_post.assert_called_once_with( + mock_config.agent_endpoint, + headers={ + "Authorization": f"Bearer {mock_token}", + "Content-Type": "application/json", + }, + json=sample_payload, + timeout=60, + ) + assert result == b"Response content" + + @pytest.mark.asyncio + async def test_payload_sender_with_custom_timeout( + self, sample_payload, mock_config, mock_token + ): + """Test that custom timeout is respected.""" + mock_response = Mock() + mock_response.content = b"Response content" + custom_timeout = 120 + + with patch( + "microsoft_agents.testing.cli.common.create_payload_sender.generate_token", + return_value=mock_token, + ), patch( + "microsoft_agents.testing.cli.common.create_payload_sender.requests.post", + return_value=mock_response, + ) as mock_post: + sender = create_payload_sender(sample_payload, timeout=custom_timeout) + await sender() + + assert mock_post.call_args[1]["timeout"] == custom_timeout + + @pytest.mark.asyncio + async def test_payload_sender_returns_response_content( + self, sample_payload, mock_config, mock_token + ): + """Test that payload sender returns response content.""" + expected_content = b'{"status": "success", "id": "123"}' + mock_response = Mock() + mock_response.content = expected_content + + with patch( + "microsoft_agents.testing.cli.common.create_payload_sender.generate_token", + return_value=mock_token, + ), patch( + "microsoft_agents.testing.cli.common.create_payload_sender.requests.post", + return_value=mock_response, + ): + sender = create_payload_sender(sample_payload) + result = await sender() + + assert result == expected_content + + @pytest.mark.asyncio + async def test_payload_sender_uses_asyncio_to_thread( + self, sample_payload, mock_config, mock_token + ): + """Test that the sender uses asyncio.to_thread for the blocking call.""" + mock_response = Mock() + mock_response.content = b"Response content" + + with patch( + "microsoft_agents.testing.cli.common.create_payload_sender.generate_token", + return_value=mock_token, + ), patch( + "microsoft_agents.testing.cli.common.create_payload_sender.requests.post", + return_value=mock_response, + ), patch( + "microsoft_agents.testing.cli.common.create_payload_sender.asyncio.to_thread", + new_callable=AsyncMock, + ) as mock_to_thread: + mock_to_thread.return_value = mock_response + + sender = create_payload_sender(sample_payload) + await sender() + + mock_to_thread.assert_called_once() + + @pytest.mark.asyncio + async def test_payload_sender_with_empty_payload( + self, mock_config, mock_token + ): + """Test payload sender with an empty payload.""" + empty_payload: dict[str, Any] = {} + mock_response = Mock() + mock_response.content = b"" + + with patch( + "microsoft_agents.testing.cli.common.create_payload_sender.generate_token", + return_value=mock_token, + ), patch( + "microsoft_agents.testing.cli.common.create_payload_sender.requests.post", + return_value=mock_response, + ) as mock_post: + sender = create_payload_sender(empty_payload) + result = await sender() + + mock_post.assert_called_once() + assert mock_post.call_args[1]["json"] == empty_payload + assert result == b"" + + @pytest.mark.asyncio + async def test_payload_sender_with_complex_payload( + self, mock_config, mock_token + ): + """Test payload sender with a complex nested payload.""" + complex_payload = { + "type": "message", + "text": "Complex message", + "attachments": [ + {"contentType": "application/json", "content": {"key": "value"}}, + {"contentType": "text/plain", "content": "Plain text"}, + ], + "channelData": {"custom": {"nested": {"data": [1, 2, 3]}}}, + } + mock_response = Mock() + mock_response.content = b"Response" + + with patch( + "microsoft_agents.testing.cli.common.create_payload_sender.generate_token", + return_value=mock_token, + ), patch( + "microsoft_agents.testing.cli.common.create_payload_sender.requests.post", + return_value=mock_response, + ) as mock_post: + sender = create_payload_sender(complex_payload) + await sender() + + assert mock_post.call_args[1]["json"] == complex_payload + + @pytest.mark.asyncio + async def test_multiple_invocations_of_same_sender( + self, sample_payload, mock_config, mock_token + ): + """Test that the same sender can be invoked multiple times.""" + mock_response = Mock() + mock_response.content = b"Response" + + with patch( + "microsoft_agents.testing.cli.common.create_payload_sender.generate_token", + return_value=mock_token, + ), patch( + "microsoft_agents.testing.cli.common.create_payload_sender.requests.post", + return_value=mock_response, + ) as mock_post: + sender = create_payload_sender(sample_payload) + + # Call sender multiple times + await sender() + await sender() + await sender() + + assert mock_post.call_count == 3 + + @pytest.mark.asyncio + async def test_authorization_header_format( + self, sample_payload, mock_config, mock_token + ): + """Test that the Authorization header is correctly formatted.""" + mock_response = Mock() + mock_response.content = b"Response" + + with patch( + "microsoft_agents.testing.cli.common.create_payload_sender.generate_token", + return_value=mock_token, + ), patch( + "microsoft_agents.testing.cli.common.create_payload_sender.requests.post", + return_value=mock_response, + ) as mock_post: + sender = create_payload_sender(sample_payload) + await sender() + + headers = mock_post.call_args[1]["headers"] + assert headers["Authorization"] == f"Bearer {mock_token}" + assert headers["Content-Type"] == "application/json" + + @pytest.mark.asyncio + async def test_payload_sender_request_exception_propagates( + self, sample_payload, mock_config, mock_token + ): + """Test that exceptions from requests.post are propagated.""" + with patch( + "microsoft_agents.testing.cli.common.create_payload_sender.generate_token", + return_value=mock_token, + ), patch( + "microsoft_agents.testing.cli.common.create_payload_sender.requests.post", + side_effect=Exception("Network error"), + ): + sender = create_payload_sender(sample_payload) + + with pytest.raises(Exception, match="Network error"): + await sender() + + @pytest.mark.asyncio + async def test_different_payloads_create_independent_senders( + self, mock_config, mock_token + ): + """Test that different payloads create independent sender functions.""" + payload1 = {"type": "message", "text": "Message 1"} + payload2 = {"type": "message", "text": "Message 2"} + + mock_response = Mock() + mock_response.content = b"Response" + + with patch( + "microsoft_agents.testing.cli.common.create_payload_sender.generate_token", + return_value=mock_token, + ), patch( + "microsoft_agents.testing.cli.common.create_payload_sender.requests.post", + return_value=mock_response, + ) as mock_post: + sender1 = create_payload_sender(payload1) + sender2 = create_payload_sender(payload2) + + await sender1() + await sender2() + + assert mock_post.call_count == 2 + # Verify different payloads were sent + assert mock_post.call_args_list[0][1]["json"] == payload1 + assert mock_post.call_args_list[1][1]["json"] == payload2 \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/assertions/test_integration_assertion.py b/dev/microsoft-agents-testing/tests/cli/test_cli.py similarity index 100% rename from dev/microsoft-agents-testing/tests/assertions/test_integration_assertion.py rename to dev/microsoft-agents-testing/tests/cli/test_cli.py diff --git a/dev/microsoft-agents-testing/tests/cli/test_cli_config.py b/dev/microsoft-agents-testing/tests/cli/test_cli_config.py new file mode 100644 index 00000000..88e10395 --- /dev/null +++ b/dev/microsoft-agents-testing/tests/cli/test_cli_config.py @@ -0,0 +1,225 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os +import pytest +from unittest.mock import patch + +from microsoft_agents.testing.cli.cli_config import _CLIConfig, cli_config + + +class TestCLIConfig: + + def test_default_values(self): + """Test that default values are set correctly.""" + config = _CLIConfig() + assert config.tenant_id == "" + assert config.app_id == "" + assert config.app_secret == "" + assert config.agent_url == "http://localhost:3978/" + assert config.service_url == "http://localhost:8001/" + + def test_agent_endpoint_property(self): + """Test that agent_endpoint returns the correct URL.""" + config = _CLIConfig() + assert config.agent_endpoint == "http://localhost:3978/api/messages/" + + def test_agent_endpoint_with_custom_url(self): + """Test agent_endpoint with a custom agent_url.""" + config = _CLIConfig() + config.agent_url = "http://example.com:5000/" + assert config.agent_endpoint == "http://example.com:5000/api/messages/" + + def test_agent_endpoint_without_trailing_slash(self): + """Test agent_endpoint when agent_url doesn't have trailing slash.""" + config = _CLIConfig() + config.agent_url = "http://example.com:5000" + assert config.agent_endpoint == "http://example.com:5000/api/messages/" + + def test_load_from_config_empty_dict(self): + """Test load_from_config with an empty dictionary.""" + config = _CLIConfig() + config.load_from_config({}) + + assert config.tenant_id == "" + assert config.app_id == "" + assert config.app_secret == "" + assert config.agent_url == "http://localhost:3978/" + + def test_load_from_config_partial_dict(self): + """Test load_from_config with partial configuration.""" + config = _CLIConfig() + config.load_from_config({ + "tenant_id": "test-tenant", + "app_id": "test-app" + }) + + assert config.tenant_id == "test-tenant" + assert config.app_id == "test-app" + assert config.app_secret == "" + assert config.agent_url == "http://localhost:3978/" + + def test_load_from_config_full_dict(self): + """Test load_from_config with full configuration.""" + config = _CLIConfig() + config.load_from_config({ + "tenant_id": "test-tenant", + "app_id": "test-app", + "app_secret": "test-secret", + "agent_url": "http://example.com/" + }) + + assert config.tenant_id == "test-tenant" + assert config.app_id == "test-app" + assert config.app_secret == "test-secret" + assert config.agent_url == "http://example.com/" + + def test_load_from_config_none_uses_env(self): + """Test load_from_config with None uses environment variables.""" + with patch.dict(os.environ, { + "tenant_id": "env-tenant", + "app_id": "env-app", + "app_secret": "env-secret", + "agent_url": "http://env.example.com/" + }, clear=False): + config = _CLIConfig() + config.load_from_config(None) + + assert config.tenant_id == "env-tenant" + assert config.app_id == "env-app" + assert config.app_secret == "env-secret" + assert config.agent_url == "http://env.example.com/" + + def test_load_from_config_updates_existing_values(self): + """Test that load_from_config updates existing values.""" + config = _CLIConfig() + config.tenant_id = "old-tenant" + config.app_id = "old-app" + + config.load_from_config({ + "tenant_id": "new-tenant", + "app_secret": "new-secret" + }) + + assert config.tenant_id == "new-tenant" + assert config.app_id == "old-app" + assert config.app_secret == "new-secret" + + def test_load_from_connection_default_connection_name(self): + """Test load_from_connection with default connection name.""" + with patch.dict(os.environ, { + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID": "connection-app-id", + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET": "connection-secret", + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID": "connection-tenant" + }, clear=False): + config = _CLIConfig() + config.load_from_connection() + + assert config.app_id == "connection-app-id" + assert config.app_secret == "connection-secret" + assert config.tenant_id == "connection-tenant" + + def test_load_from_connection_custom_connection_name(self): + """Test load_from_connection with custom connection name.""" + with patch.dict(os.environ, { + "CONNECTIONS__CUSTOM_CONNECTION__SETTINGS__CLIENTID": "custom-app-id", + "CONNECTIONS__CUSTOM_CONNECTION__SETTINGS__CLIENTSECRET": "custom-secret", + "CONNECTIONS__CUSTOM_CONNECTION__SETTINGS__TENANTID": "custom-tenant" + }, clear=False): + config = _CLIConfig() + config.load_from_connection("CUSTOM_CONNECTION") + + assert config.app_id == "custom-app-id" + assert config.app_secret == "custom-secret" + assert config.tenant_id == "custom-tenant" + + def test_load_from_connection_partial_env_vars(self): + """Test load_from_connection with only some environment variables set.""" + with patch.dict(os.environ, { + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID": "partial-app-id" + }, clear=False): + config = _CLIConfig() + config.app_secret = "existing-secret" + config.load_from_connection() + + assert config.app_id == "partial-app-id" + assert config.app_secret == "existing-secret" + assert config.tenant_id == "" + + def test_load_from_connection_no_env_vars(self): + """Test load_from_connection with no matching environment variables.""" + config = _CLIConfig() + config.tenant_id = "existing-tenant" + config.app_id = "existing-app" + + # Ensure no connection env vars exist + env_clean = {k: v for k, v in os.environ.items() + if not k.startswith("CONNECTIONS__TEST_CONNECTION__")} + + with patch.dict(os.environ, env_clean, clear=True): + config.load_from_connection("TEST_CONNECTION") + + # Should retain existing values + assert config.tenant_id == "existing-tenant" + assert config.app_id == "existing-app" + + def test_load_from_connection_config_param_unused(self): + """Test that config parameter in load_from_connection is unused.""" + with patch.dict(os.environ, { + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID": "env-app-id" + }, clear=False): + config = _CLIConfig() + # The config parameter is defined but not used in the implementation + config.load_from_connection("SERVICE_CONNECTION", {"app_id": "ignored"}) + + assert config.app_id == "env-app-id" + + def test_cli_config_singleton_loaded_from_env(self): + """Test that the module-level cli_config instance is initialized.""" + # The cli_config singleton should be loaded from config on import + assert isinstance(cli_config, _CLIConfig) + + +class TestCLIConfigIntegration: + """Integration tests for _CLIConfig with various scenarios.""" + + def test_full_workflow_load_config_then_connection(self): + """Test loading from config first, then from connection.""" + config = _CLIConfig() + + # First load from config + config.load_from_config({ + "tenant_id": "config-tenant", + "app_id": "config-app", + "agent_url": "http://config.example.com/" + }) + + assert config.tenant_id == "config-tenant" + assert config.app_id == "config-app" + + # Then load from connection (should override) + with patch.dict(os.environ, { + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID": "connection-app", + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID": "connection-tenant" + }, clear=False): + config.load_from_connection() + + assert config.tenant_id == "connection-tenant" + assert config.app_id == "connection-app" + # agent_url should remain from config load + assert config.agent_url == "http://config.example.com/" + + def test_environment_variable_override(self): + """Test that environment variables work as expected.""" + prev_env = os.environ.copy() + os.environ["tenant_id"] = "env-tenant" + os.environ["app_id"] = "env-app" + + config = _CLIConfig() + config.load_from_config() + + assert config.tenant_id == "env-tenant" + assert config.app_id == "env-app" + + for key in prev_env: + os.environ[key] = prev_env[key] diff --git a/dev/microsoft-agents-testing/tests/integration/core/_common.py b/dev/microsoft-agents-testing/tests/integration/_common.py similarity index 100% rename from dev/microsoft-agents-testing/tests/integration/core/_common.py rename to dev/microsoft-agents-testing/tests/integration/_common.py diff --git a/dev/microsoft-agents-testing/tests/integration/core/__init__.py b/dev/microsoft-agents-testing/tests/integration/client/__init__.py similarity index 100% rename from dev/microsoft-agents-testing/tests/integration/core/__init__.py rename to dev/microsoft-agents-testing/tests/integration/client/__init__.py diff --git a/dev/microsoft-agents-testing/tests/integration/core/client/_common.py b/dev/microsoft-agents-testing/tests/integration/client/_common.py similarity index 100% rename from dev/microsoft-agents-testing/tests/integration/core/client/_common.py rename to dev/microsoft-agents-testing/tests/integration/client/_common.py diff --git a/dev/microsoft-agents-testing/tests/integration/core/client/test_agent_client.py b/dev/microsoft-agents-testing/tests/integration/client/test_agent_client.py similarity index 69% rename from dev/microsoft-agents-testing/tests/integration/core/client/test_agent_client.py rename to dev/microsoft-agents-testing/tests/integration/client/test_agent_client.py index 3bc59452..cdd597ab 100644 --- a/dev/microsoft-agents-testing/tests/integration/core/client/test_agent_client.py +++ b/dev/microsoft-agents-testing/tests/integration/client/test_agent_client.py @@ -4,7 +4,7 @@ from aioresponses import aioresponses from msal import ConfidentialClientApplication -from microsoft_agents.activity import Activity +from microsoft_agents.activity import Activity, InvokeResponse from microsoft_agents.testing import AgentClient from ._common import DEFAULTS @@ -82,3 +82,34 @@ async def test_send_expect_replies(self, mocker, agent_client, aioresponses_mock assert replies[0].text == "Response from service" assert replies[1].text == "Another response" assert replies[0].type == replies[1].type == "message" + + @pytest.mark.asyncio + async def test_send_invoke_activity(self, mocker, agent_client, aioresponses_mock): + mocker.patch.object( + AgentClient, "get_access_token", return_value="mocked_token" + ) + mocker.patch.object( + ConfidentialClientApplication, + "__new__", + return_value=mocker.Mock(spec=ConfidentialClientApplication), + ) + + res = InvokeResponse(status=200, body={"status": "Invoke processed"}) + + assert agent_client.agent_url + aioresponses_mock.post( + f"{agent_client.agent_url}api/messages", + payload=res.model_dump(by_alias=True, exclude_none=True), + ) + + response = await agent_client.send_invoke_activity( + Activity( + type="invoke", + name="test_invoke", + value={"key": "value"}, + ) + ) + assert response is not None + assert isinstance(response, InvokeResponse) + assert response.status == 200 + assert response.body == {"status": "Invoke processed"} diff --git a/dev/microsoft-agents-testing/tests/integration/core/client/test_response_client.py b/dev/microsoft-agents-testing/tests/integration/client/test_response_client.py similarity index 100% rename from dev/microsoft-agents-testing/tests/integration/core/client/test_response_client.py rename to dev/microsoft-agents-testing/tests/integration/client/test_response_client.py diff --git a/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py b/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py deleted file mode 100644 index 729148fc..00000000 --- a/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py +++ /dev/null @@ -1,825 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import pytest -import asyncio -from unittest.mock import AsyncMock, MagicMock, patch, call -from copy import deepcopy - -from microsoft_agents.activity import Activity -from microsoft_agents.testing.assertions import ModelAssertion -from microsoft_agents.testing.integration.core import AgentClient, ResponseClient -from microsoft_agents.testing.integration.data_driven import DataDrivenTest - - -class TestDataDrivenTestInit: - """Tests for DataDrivenTest initialization.""" - - def test_init_minimal(self): - """Test initialization with minimal required fields.""" - test_flow = {"name": "test1"} - ddt = DataDrivenTest(test_flow) - - assert ddt._name == "test1" - assert ddt._description == "" - assert ddt._input_defaults == {} - assert ddt._assertion_defaults == {} - assert ddt._sleep_defaults == {} - assert ddt._test == [] - - def test_init_with_description(self): - """Test initialization with description.""" - test_flow = {"name": "test1", "description": "Test description"} - ddt = DataDrivenTest(test_flow) - - assert ddt._name == "test1" - assert ddt._description == "Test description" - - def test_init_with_defaults(self): - """Test initialization with defaults.""" - test_flow = { - "name": "test1", - "defaults": { - "input": {"activity": {"type": "message", "locale": "en-US"}}, - "assertion": {"quantifier": "all"}, - "sleep": {"duration": 1.0}, - }, - } - ddt = DataDrivenTest(test_flow) - - assert ddt._input_defaults == { - "activity": {"type": "message", "locale": "en-US"} - } - assert ddt._assertion_defaults == {"quantifier": "all"} - assert ddt._sleep_defaults == {"duration": 1.0} - - def test_init_with_test_steps(self): - """Test initialization with test steps.""" - test_flow = { - "name": "test1", - "test": [ - {"type": "input", "activity": {"text": "Hello"}}, - {"type": "assertion", "activity": {"text": "Hi"}}, - ], - } - ddt = DataDrivenTest(test_flow) - - assert len(ddt._test) == 2 - assert ddt._test[0]["type"] == "input" - assert ddt._test[1]["type"] == "assertion" - - def test_init_with_parent_defaults(self): - """Test initialization with parent defaults.""" - parent = { - "defaults": { - "input": {"activity": {"type": "message"}}, - "assertion": {"quantifier": "one"}, - "sleep": {"duration": 0.5}, - } - } - test_flow = { - "name": "test1", - "parent": parent, - "defaults": { - "input": {"activity": {"locale": "en-US"}}, - "assertion": {"quantifier": "all"}, - }, - } - ddt = DataDrivenTest(test_flow) - - # Child defaults should override parent - assert ddt._input_defaults == { - "activity": {"type": "message", "locale": "en-US"} - } - assert ddt._assertion_defaults == {"quantifier": "all"} - assert ddt._sleep_defaults == {"duration": 0.5} - - def test_init_without_name_raises_error(self): - """Test that missing name field raises ValueError.""" - test_flow = {"description": "Test without name"} - - with pytest.raises(ValueError, match="Test flow must have a 'name' field"): - DataDrivenTest(test_flow) - - def test_init_parent_defaults_dont_mutate_original(self): - """Test that merging parent defaults doesn't mutate original dictionaries.""" - parent = { - "defaults": { - "input": {"activity": {"type": "message"}}, - } - } - test_flow = { - "name": "test1", - "parent": parent, - "defaults": { - "input": {"activity": {"locale": "en-US"}}, - }, - } - - original_parent_defaults = deepcopy(parent["defaults"]["input"]) - ddt = DataDrivenTest(test_flow) - - # Verify parent defaults weren't modified - assert parent["defaults"]["input"] == original_parent_defaults - - -class TestDataDrivenTestLoadInput: - """Tests for _load_input method.""" - - def test_load_input_basic(self): - """Test loading a basic input activity.""" - test_flow = {"name": "test1"} - ddt = DataDrivenTest(test_flow) - - input_data = {"activity": {"type": "message", "text": "Hello"}} - activity = ddt._load_input(input_data) - - assert isinstance(activity, Activity) - assert activity.type == "message" - assert activity.text == "Hello" - - def test_load_input_with_defaults(self): - """Test loading input with defaults applied.""" - test_flow = { - "name": "test1", - "defaults": {"input": {"activity": {"type": "message", "locale": "en-US"}}}, - } - ddt = DataDrivenTest(test_flow) - - input_data = {"activity": {"text": "Hello"}} - activity = ddt._load_input(input_data) - - assert activity.type == "message" - assert activity.text == "Hello" - assert activity.locale == "en-US" - - def test_load_input_override_defaults(self): - """Test that explicit input values override defaults.""" - test_flow = { - "name": "test1", - "defaults": {"input": {"activity": {"type": "message", "locale": "en-US"}}}, - } - ddt = DataDrivenTest(test_flow) - - input_data = {"activity": {"type": "event", "locale": "fr-FR"}} - activity = ddt._load_input(input_data) - - assert activity.type == "event" - assert activity.locale == "fr-FR" - - def test_load_input_empty_activity_fails(self): - """Test loading input with empty activity.""" - test_flow = {"name": "test1"} - ddt = DataDrivenTest(test_flow) - - input_data = {"activity": {}} - - with pytest.raises(Exception): - ddt._load_input(input_data) - - def test_load_input_nested_defaults(self): - """Test loading input with nested default values.""" - test_flow = { - "name": "test1", - "defaults": { - "input": {"activity": {"channelData": {"nested": {"value": 123}}}} - }, - } - ddt = DataDrivenTest(test_flow) - - input_data = {"activity": {"type": "message", "text": "Hello"}} - activity = ddt._load_input(input_data) - - assert activity.text == "Hello" - assert activity.channel_data == {"nested": {"value": 123}} - - def test_load_input_no_activity_key_raises(self): - """Test loading input when activity key is missing.""" - test_flow = {"name": "test1"} - ddt = DataDrivenTest(test_flow) - - input_data = {} - - with pytest.raises(Exception): - ddt._load_input(input_data) - - -class TestDataDrivenTestLoadAssertion: - """Tests for _load_assertion method.""" - - def test_load_assertion_basic(self): - """Test loading a basic assertion.""" - test_flow = {"name": "test1"} - ddt = DataDrivenTest(test_flow) - - assertion_data = {"activity": {"type": "message", "text": "Hello"}} - assertion = ddt._load_assertion(assertion_data) - - assert isinstance(assertion, ModelAssertion) - - def test_load_assertion_with_defaults(self): - """Test loading assertion with defaults applied.""" - test_flow = {"name": "test1", "defaults": {"assertion": {"quantifier": "one"}}} - ddt = DataDrivenTest(test_flow) - - assertion_data = {"activity": {"text": "Hello"}} - assertion = ddt._load_assertion(assertion_data) - - assert isinstance(assertion, ModelAssertion) - - def test_load_assertion_override_defaults(self): - """Test that explicit assertion values override defaults.""" - test_flow = {"name": "test1", "defaults": {"assertion": {"quantifier": "one"}}} - ddt = DataDrivenTest(test_flow) - - assertion_data = {"quantifier": "all", "activity": {"text": "Hello"}} - assertion = ddt._load_assertion(assertion_data) - - assert isinstance(assertion, ModelAssertion) - - def test_load_assertion_with_selector(self): - """Test loading assertion with selector.""" - test_flow = {"name": "test1"} - ddt = DataDrivenTest(test_flow) - - assertion_data = { - "activity": {"type": "message"}, - "selector": {"selector": {"type": "message"}}, - } - assertion = ddt._load_assertion(assertion_data) - - assert isinstance(assertion, ModelAssertion) - - def test_load_assertion_empty(self): - """Test loading empty assertion.""" - test_flow = {"name": "test1"} - ddt = DataDrivenTest(test_flow) - - assertion_data = {} - assertion = ddt._load_assertion(assertion_data) - - assert isinstance(assertion, ModelAssertion) - - -class TestDataDrivenTestSleep: - """Tests for _sleep method.""" - - @pytest.mark.asyncio - async def test_sleep_with_explicit_duration(self): - """Test sleep with explicit duration.""" - test_flow = {"name": "test1"} - ddt = DataDrivenTest(test_flow) - - sleep_data = {"duration": 0.1} - start_time = asyncio.get_event_loop().time() - await ddt._sleep(sleep_data) - elapsed = asyncio.get_event_loop().time() - start_time - - assert elapsed >= 0.1 - assert elapsed < 0.2 # Allow some margin - - @pytest.mark.asyncio - async def test_sleep_with_default_duration(self): - """Test sleep using default duration.""" - test_flow = {"name": "test1", "defaults": {"sleep": {"duration": 0.1}}} - ddt = DataDrivenTest(test_flow) - - sleep_data = {} - start_time = asyncio.get_event_loop().time() - await ddt._sleep(sleep_data) - elapsed = asyncio.get_event_loop().time() - start_time - - assert elapsed >= 0.1 - - @pytest.mark.asyncio - async def test_sleep_zero_duration(self): - """Test sleep with zero duration.""" - test_flow = {"name": "test1"} - ddt = DataDrivenTest(test_flow) - - sleep_data = {"duration": 0} - start_time = asyncio.get_event_loop().time() - await ddt._sleep(sleep_data) - elapsed = asyncio.get_event_loop().time() - start_time - - assert elapsed < 0.1 - - @pytest.mark.asyncio - async def test_sleep_no_duration_no_default(self): - """Test sleep with no duration and no default.""" - test_flow = {"name": "test1"} - ddt = DataDrivenTest(test_flow) - - sleep_data = {} - start_time = asyncio.get_event_loop().time() - await ddt._sleep(sleep_data) - elapsed = asyncio.get_event_loop().time() - start_time - - # Should default to 0 - assert elapsed < 0.1 - - @pytest.mark.asyncio - async def test_sleep_override_default(self): - """Test that explicit duration overrides default.""" - test_flow = {"name": "test1", "defaults": {"sleep": {"duration": 1.0}}} - ddt = DataDrivenTest(test_flow) - - sleep_data = {"duration": 0.05} - start_time = asyncio.get_event_loop().time() - await ddt._sleep(sleep_data) - elapsed = asyncio.get_event_loop().time() - start_time - - assert elapsed >= 0.05 - assert elapsed < 0.2 # Should not use default 1.0 - - -class TestDataDrivenTestRun: - """Tests for run method.""" - - @pytest.mark.asyncio - async def test_run_empty_test(self): - """Test running empty test.""" - test_flow = {"name": "test1", "test": []} - ddt = DataDrivenTest(test_flow) - - agent_client = AsyncMock(spec=AgentClient) - response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock(return_value=[]) - - await ddt.run(agent_client, response_client) - - agent_client.send_activity.assert_not_called() - - @pytest.mark.asyncio - async def test_run_single_input(self): - """Test running test with single input.""" - test_flow = { - "name": "test1", - "test": [ - {"type": "input", "activity": {"type": "message", "text": "Hello"}} - ], - } - ddt = DataDrivenTest(test_flow) - - agent_client = AsyncMock(spec=AgentClient) - response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock(return_value=[]) - - await ddt.run(agent_client, response_client) - - agent_client.send_activity.assert_called_once() - call_args = agent_client.send_activity.call_args[0][0] - assert isinstance(call_args, Activity) - assert call_args.text == "Hello" - - @pytest.mark.asyncio - async def test_run_input_and_assertion(self): - """Test running test with input and assertion.""" - test_flow = { - "name": "test1", - "test": [ - {"type": "input", "activity": {"type": "message", "text": "Hello"}}, - {"type": "assertion", "activity": {"type": "message"}}, - ], - } - ddt = DataDrivenTest(test_flow) - - agent_client = AsyncMock(spec=AgentClient) - response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock( - return_value=[Activity(type="message", text="Hi")] - ) - - await ddt.run(agent_client, response_client) - - agent_client.send_activity.assert_called_once() - response_client.pop.assert_called_once() - - @pytest.mark.asyncio - async def test_run_with_sleep(self): - """Test running test with sleep step.""" - test_flow = {"name": "test1", "test": [{"type": "sleep", "duration": 0.05}]} - ddt = DataDrivenTest(test_flow) - - agent_client = AsyncMock(spec=AgentClient) - response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock(return_value=[]) - - start_time = asyncio.get_event_loop().time() - await ddt.run(agent_client, response_client) - elapsed = asyncio.get_event_loop().time() - start_time - - assert elapsed >= 0.05 - - @pytest.mark.asyncio - async def test_run_missing_step_type_raises_error(self): - """Test that missing step type raises ValueError.""" - test_flow = {"name": "test1", "test": [{"activity": {"text": "Hello"}}]} - ddt = DataDrivenTest(test_flow) - - agent_client = AsyncMock(spec=AgentClient) - response_client = AsyncMock(spec=ResponseClient) - - with pytest.raises(ValueError, match="Each step must have a 'type' field"): - await ddt.run(agent_client, response_client) - - @pytest.mark.asyncio - async def test_run_multiple_steps(self): - """Test running test with multiple steps.""" - test_flow = { - "name": "test1", - "test": [ - {"type": "input", "activity": {"type": "message", "text": "Hello"}}, - {"type": "sleep", "duration": 0.01}, - {"type": "assertion", "activity": {"type": "message"}}, - {"type": "input", "activity": {"type": "message", "text": "Goodbye"}}, - ], - } - ddt = DataDrivenTest(test_flow) - - agent_client = AsyncMock(spec=AgentClient) - response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock( - return_value=[Activity(type="message", text="Hi")] - ) - - await ddt.run(agent_client, response_client) - - assert agent_client.send_activity.call_count == 2 - - @pytest.mark.asyncio - async def test_run_assertion_accumulates_responses(self): - """Test that assertion accumulates responses from previous steps.""" - test_flow = { - "name": "test1", - "test": [ - {"type": "input", "activity": {"type": "message", "text": "Hello"}}, - { - "type": "assertion", - "activity": {"type": "message"}, - "quantifier": "all", - }, - ], - } - ddt = DataDrivenTest(test_flow) - - agent_client = AsyncMock(spec=AgentClient) - response_client = AsyncMock(spec=ResponseClient) - - # Mock multiple responses - responses = [ - Activity(type="message", text="Response 1"), - Activity(type="message", text="Response 2"), - ] - response_client.pop = AsyncMock(return_value=responses) - - await ddt.run(agent_client, response_client) - - response_client.pop.assert_called_once() - - @pytest.mark.asyncio - async def test_run_assertion_fails_raises_assertion_error(self): - """Test that failing assertion raises AssertionError.""" - test_flow = { - "name": "test1", - "test": [ - {"type": "input", "activity": {"type": "message", "text": "Hello"}}, - {"type": "assertion", "activity": {"text": "Expected text"}}, - ], - } - ddt = DataDrivenTest(test_flow) - - agent_client = AsyncMock(spec=AgentClient) - response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock( - return_value=[Activity(type="message", text="Different text")] - ) - - with pytest.raises(AssertionError): - await ddt.run(agent_client, response_client) - - @pytest.mark.asyncio - async def test_run_with_defaults_applied(self): - """Test that defaults are applied during run.""" - test_flow = { - "name": "test1", - "defaults": {"input": {"activity": {"type": "message", "locale": "en-US"}}}, - "test": [{"type": "input", "activity": {"text": "Hello"}}], - } - ddt = DataDrivenTest(test_flow) - - agent_client = AsyncMock(spec=AgentClient) - response_client = AsyncMock(spec=ResponseClient) - - await ddt.run(agent_client, response_client) - - call_args = agent_client.send_activity.call_args[0][0] - assert call_args.type == "message" - assert call_args.text == "Hello" - assert call_args.locale == "en-US" - - @pytest.mark.asyncio - async def test_run_multiple_assertions_extend_responses(self): - """Test that multiple assertions extend the responses list.""" - test_flow = { - "name": "test1", - "test": [ - {"type": "input", "activity": {"type": "message", "text": "Hello"}}, - {"type": "assertion", "activity": {"type": "message"}}, - {"type": "input", "activity": {"type": "message", "text": "World"}}, - {"type": "assertion", "activity": {"type": "message"}}, - ], - } - ddt = DataDrivenTest(test_flow) - - agent_client = AsyncMock(spec=AgentClient) - response_client = AsyncMock(spec=ResponseClient) - - # First pop returns one activity, second pop returns another - response_client.pop = AsyncMock( - side_effect=[ - [Activity(type="message", text="Response 1")], - [Activity(type="message", text="Response 2")], - ] - ) - - await ddt.run(agent_client, response_client) - - assert response_client.pop.call_count == 2 - - -class TestDataDrivenTestIntegration: - """Integration tests with realistic scenarios.""" - - @pytest.mark.asyncio - async def test_full_conversation_flow(self): - """Test a complete conversation flow.""" - test_flow = { - "name": "greeting_test", - "description": "Test greeting conversation", - "defaults": { - "input": {"activity": {"type": "message", "locale": "en-US"}}, - "assertion": {"quantifier": "all"}, - }, - "test": [ - {"type": "input", "activity": {"text": "Hello"}}, - {"type": "sleep", "duration": 0.05}, - { - "type": "assertion", - "activity": {"type": "message"}, - "selector": {"selector": {"type": "message"}}, - }, - ], - } - ddt = DataDrivenTest(test_flow) - - agent_client = AsyncMock(spec=AgentClient) - response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock( - return_value=[Activity(type="message", text="Hi! How can I help you?")] - ) - - await ddt.run(agent_client, response_client) - - # Verify input was sent - assert agent_client.send_activity.call_count == 1 - - # Verify assertion was checked - assert response_client.pop.call_count == 1 - - @pytest.mark.asyncio - async def test_complex_multi_turn_conversation(self): - """Test multi-turn conversation with multiple inputs and assertions.""" - test_flow = { - "name": "multi_turn_test", - "test": [ - { - "type": "input", - "activity": {"type": "message", "text": "What's the weather?"}, - }, - {"type": "assertion", "activity": {"type": "message"}}, - {"type": "sleep", "duration": 0.01}, - {"type": "input", "activity": {"type": "message", "text": "Thank you"}}, - { - "type": "assertion", - "activity": {"type": "message"}, - "quantifier": "any", - }, - ], - } - ddt = DataDrivenTest(test_flow) - - agent_client = AsyncMock(spec=AgentClient) - response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock( - side_effect=[ - [Activity(type="message", text="It's sunny today")], - [Activity(type="message", text="You're welcome!")], - ] - ) - - await ddt.run(agent_client, response_client) - - assert agent_client.send_activity.call_count == 2 - assert response_client.pop.call_count == 2 - - @pytest.mark.asyncio - async def test_with_parent_inheritance(self): - """Test data driven test with parent defaults inheritance.""" - parent = { - "defaults": { - "input": {"activity": {"type": "message", "locale": "en-US"}}, - "sleep": {"duration": 0.01}, - } - } - - test_flow = { - "name": "child_test", - "parent": parent, - "defaults": {"input": {"activity": {"channel_id": "test-channel"}}}, - "test": [ - {"type": "input", "activity": {"text": "Hello"}}, - {"type": "sleep"}, - {"type": "assertion", "activity": {"type": "message"}}, - ], - } - ddt = DataDrivenTest(test_flow) - - agent_client = AsyncMock(spec=AgentClient) - response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock( - return_value=[Activity(type="message", text="Hi")] - ) - - start_time = asyncio.get_event_loop().time() - await ddt.run(agent_client, response_client) - elapsed = asyncio.get_event_loop().time() - start_time - - # Verify inherited sleep duration was used - assert elapsed >= 0.01 - - # Verify merged defaults were applied - call_args = agent_client.send_activity.call_args[0][0] - assert call_args.type == "message" - assert call_args.locale == "en-US" - assert call_args.channel_id == "test-channel" - - -class TestDataDrivenTestEdgeCases: - """Tests for edge cases and error conditions.""" - - def test_empty_name_string_raises_error(self): - """Test that empty name string raises ValueError.""" - test_flow = {"name": ""} - - with pytest.raises(ValueError, match="Test flow must have a 'name' field"): - DataDrivenTest(test_flow) - - def test_none_name_raises_error(self): - """Test that None name raises ValueError.""" - test_flow = {"name": None} - - with pytest.raises(ValueError, match="Test flow must have a 'name' field"): - DataDrivenTest(test_flow) - - @pytest.mark.asyncio - async def test_run_unknown_step_type(self): - """Test that unknown step type is ignored (no error in current implementation).""" - test_flow = { - "name": "test1", - "test": [{"type": "unknown_type", "data": "something"}], - } - ddt = DataDrivenTest(test_flow) - - agent_client = AsyncMock(spec=AgentClient) - response_client = AsyncMock(spec=ResponseClient) - - # Should complete without error (unknown types are simply skipped) - await ddt.run(agent_client, response_client) - - @pytest.mark.asyncio - async def test_run_assertion_with_no_prior_responses(self): - """Test assertion when no responses have been collected.""" - test_flow = { - "name": "test1", - "test": [{"type": "assertion", "activity": {"type": "message"}}], - } - ddt = DataDrivenTest(test_flow) - - agent_client = AsyncMock(spec=AgentClient) - response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock(return_value=[]) - - # Should pass because empty list matches ALL quantifier with no failures - await ddt.run(agent_client, response_client) - - def test_deep_nested_defaults(self): - """Test deeply nested default values.""" - test_flow = { - "name": "test1", - "defaults": { - "input": { - "activity": { - "channel_data": {"level1": {"level2": {"level3": "value"}}} - } - } - }, - } - ddt = DataDrivenTest(test_flow) - - assert ( - ddt._input_defaults["activity"]["channel_data"]["level1"]["level2"][ - "level3" - ] - == "value" - ) - - @pytest.mark.asyncio - async def test_load_input_preserves_original_data(self): - """Test that _load_input doesn't mutate original input data.""" - test_flow = { - "name": "test1", - "defaults": {"input": {"activity": {"type": "message"}}}, - } - ddt = DataDrivenTest(test_flow) - - original_input = {"activity": {"text": "Hello"}} - original_copy = deepcopy(original_input) - - ddt._load_input(original_input) - - # Original should be modified (update_with_defaults modifies in place) - # But let's verify the activity is still loadable - assert original_input is not None - - @pytest.mark.asyncio - async def test_run_with_special_activity_types(self): - """Test running with non-message activity types.""" - test_flow = { - "name": "test1", - "test": [ - { - "type": "input", - "activity": {"type": "event", "name": "custom_event"}, - }, - {"type": "assertion", "activity": {"type": "event"}}, - ], - } - ddt = DataDrivenTest(test_flow) - - agent_client = AsyncMock(spec=AgentClient) - response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock( - return_value=[Activity(type="event", name="response_event")] - ) - - await ddt.run(agent_client, response_client) - - call_args = agent_client.send_activity.call_args[0][0] - assert call_args.type == "event" - assert call_args.name == "custom_event" - - -class TestDataDrivenTestProperties: - """Tests for accessing test properties.""" - - def test_name_property(self): - """Test accessing the name property.""" - test_flow = {"name": "my_test"} - ddt = DataDrivenTest(test_flow) - - assert ddt._name == "my_test" - - def test_description_property(self): - """Test accessing the description property.""" - test_flow = {"name": "test1", "description": "This is a test"} - ddt = DataDrivenTest(test_flow) - - assert ddt._description == "This is a test" - - def test_defaults_properties(self): - """Test accessing defaults properties.""" - test_flow = { - "name": "test1", - "defaults": { - "input": {"activity": {"type": "message"}}, - "assertion": {"quantifier": "all"}, - "sleep": {"duration": 1.0}, - }, - } - ddt = DataDrivenTest(test_flow) - - assert ddt._input_defaults == {"activity": {"type": "message"}} - assert ddt._assertion_defaults == {"quantifier": "all"} - assert ddt._sleep_defaults == {"duration": 1.0} - - def test_test_steps_property(self): - """Test accessing test steps property.""" - test_flow = { - "name": "test1", - "test": [{"type": "input"}, {"type": "assertion"}], - } - ddt = DataDrivenTest(test_flow) - - assert len(ddt._test) == 2 - assert ddt._test[0]["type"] == "input" diff --git a/dev/microsoft-agents-testing/tests/integration/data_driven/test_ddt.py b/dev/microsoft-agents-testing/tests/integration/data_driven/test_ddt.py deleted file mode 100644 index fe7eec0f..00000000 --- a/dev/microsoft-agents-testing/tests/integration/data_driven/test_ddt.py +++ /dev/null @@ -1,657 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import pytest -import tempfile -import json -from pathlib import Path -from unittest.mock import Mock, AsyncMock, patch, MagicMock - -from microsoft_agents.activity import Activity -from microsoft_agents.testing.integration.core import ( - Integration, - AgentClient, - ResponseClient, -) -from microsoft_agents.testing.integration.data_driven import DataDrivenTest, ddt -from microsoft_agents.testing.integration.data_driven.ddt import _add_test_method - - -class TestAddTestMethod: - """Tests for _add_test_method function.""" - - def test_add_test_method_creates_method(self): - """Test that _add_test_method creates a new test method on the class.""" - - class TestClass(Integration): - pass - - mock_ddt = Mock(spec=DataDrivenTest) - mock_ddt.name = "test_case_1" - mock_ddt.run = AsyncMock() - - _add_test_method(TestClass, mock_ddt) - - assert hasattr(TestClass, "test_data_driven__test_case_1") - method = getattr(TestClass, "test_data_driven__test_case_1") - assert callable(method) - - def test_add_test_method_replaces_slashes_in_name(self): - """Test that slashes in test name are replaced with underscores.""" - - class TestClass(Integration): - pass - - mock_ddt = Mock(spec=DataDrivenTest) - mock_ddt.name = "folder/subfolder/test_case" - mock_ddt.run = AsyncMock() - - _add_test_method(TestClass, mock_ddt) - - assert hasattr(TestClass, "test_data_driven__folder_subfolder_test_case") - assert not hasattr(TestClass, "test_data_driven__folder/subfolder/test_case") - - def test_add_test_method_replaces_dots_in_name(self): - """Test that dots in test name are replaced with underscores.""" - - class TestClass(Integration): - pass - - mock_ddt = Mock(spec=DataDrivenTest) - mock_ddt.name = "test.case.with.dots" - mock_ddt.run = AsyncMock() - - _add_test_method(TestClass, mock_ddt) - - assert hasattr(TestClass, "test_data_driven__test_case_with_dots") - - def test_add_test_method_replaces_multiple_special_chars(self): - """Test that multiple special characters are replaced.""" - - class TestClass(Integration): - pass - - mock_ddt = Mock(spec=DataDrivenTest) - mock_ddt.name = "path/to/test.case.name" - mock_ddt.run = AsyncMock() - - _add_test_method(TestClass, mock_ddt) - - assert hasattr(TestClass, "test_data_driven__path_to_test_case_name") - - @pytest.mark.asyncio - async def test_add_test_method_runs_data_driven_test(self): - """Test that the added method runs the data driven test.""" - - class TestClass(Integration): - pass - - mock_ddt = Mock(spec=DataDrivenTest) - mock_ddt.name = "test_case" - mock_ddt.run = AsyncMock() - - _add_test_method(TestClass, mock_ddt) - - test_instance = TestClass() - mock_agent_client = AsyncMock(spec=AgentClient) - mock_response_client = AsyncMock(spec=ResponseClient) - - await test_instance.test_data_driven__test_case( - mock_agent_client, mock_response_client - ) - - mock_ddt.run.assert_called_once_with(mock_agent_client, mock_response_client) - - @pytest.mark.asyncio - async def test_add_test_method_has_pytest_asyncio_mark(self): - """Test that the added method has pytest.mark.asyncio decorator.""" - - class TestClass(Integration): - pass - - mock_ddt = Mock(spec=DataDrivenTest) - mock_ddt.name = "test_case" - mock_ddt.run = AsyncMock() - - _add_test_method(TestClass, mock_ddt) - - method = getattr(TestClass, "test_data_driven__test_case") - assert hasattr(method, "pytestmark") - assert any(mark.name == "asyncio" for mark in method.pytestmark) - - def test_add_test_method_multiple_tests(self): - """Test adding multiple test methods to the same class.""" - - class TestClass(Integration): - pass - - mock_ddt1 = Mock(spec=DataDrivenTest) - mock_ddt1.name = "test_case_1" - mock_ddt1.run = AsyncMock() - - mock_ddt2 = Mock(spec=DataDrivenTest) - mock_ddt2.name = "test_case_2" - mock_ddt2.run = AsyncMock() - - _add_test_method(TestClass, mock_ddt1) - _add_test_method(TestClass, mock_ddt2) - - assert hasattr(TestClass, "test_data_driven__test_case_1") - assert hasattr(TestClass, "test_data_driven__test_case_2") - - @pytest.mark.asyncio - async def test_add_test_method_preserves_test_scope(self): - """Test that each added method maintains its own test scope.""" - - class TestClass(Integration): - pass - - mock_ddt1 = Mock(spec=DataDrivenTest) - mock_ddt1.name = "test_1" - mock_ddt1.run = AsyncMock() - - mock_ddt2 = Mock(spec=DataDrivenTest) - mock_ddt2.name = "test_2" - mock_ddt2.run = AsyncMock() - - _add_test_method(TestClass, mock_ddt1) - _add_test_method(TestClass, mock_ddt2) - - test_instance = TestClass() - mock_agent_client = AsyncMock(spec=AgentClient) - mock_response_client = AsyncMock(spec=ResponseClient) - - await test_instance.test_data_driven__test_1( - mock_agent_client, mock_response_client - ) - await test_instance.test_data_driven__test_2( - mock_agent_client, mock_response_client - ) - - # Each test should call its own run method - mock_ddt1.run.assert_called_once() - mock_ddt2.run.assert_called_once() - - def test_add_test_method_empty_name(self): - """Test adding method with empty test name.""" - - class TestClass(Integration): - pass - - mock_ddt = Mock(spec=DataDrivenTest) - mock_ddt.name = "" - mock_ddt.run = AsyncMock() - - _add_test_method(TestClass, mock_ddt) - - assert hasattr(TestClass, "test_data_driven__") - - def test_add_test_method_name_with_spaces(self): - """Test that spaces in names are preserved (converted to underscores by replace).""" - - class TestClass(Integration): - pass - - mock_ddt = Mock(spec=DataDrivenTest) - mock_ddt.name = "test with spaces" - mock_ddt.run = AsyncMock() - - _add_test_method(TestClass, mock_ddt) - - # Spaces are not replaced by the current implementation - assert hasattr(TestClass, "test_data_driven__test with spaces") - - -class TestDdtDecorator: - """Tests for ddt decorator function.""" - - def test_ddt_decorator_raises_if_no_tests(self): - """Test that ddt raises if not tests are found.""" - with pytest.raises(RuntimeError): - ddt("test_path") - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_decorator_recursive_false(self, mock_load_ddts): - """Test that ddt decorator respects recursive parameter.""" - mock_load_ddts.return_value = [DataDrivenTest({"name": "test1"})] - - @ddt("test_path", recursive=False) - class TestClass(Integration): - pass - - mock_load_ddts.assert_called_once_with("test_path", recursive=False) - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_decorator_adds_test_methods(self, mock_load_ddts): - """Test that ddt decorator adds test methods for each loaded test.""" - mock_ddt1 = Mock(spec=DataDrivenTest) - mock_ddt1.name = "test_1" - mock_ddt1.run = AsyncMock() - - mock_ddt2 = Mock(spec=DataDrivenTest) - mock_ddt2.name = "test_2" - mock_ddt2.run = AsyncMock() - - mock_load_ddts.return_value = [mock_ddt1, mock_ddt2] - - @ddt("test_path") - class TestClass(Integration): - pass - - assert hasattr(TestClass, "test_data_driven__test_1") - assert hasattr(TestClass, "test_data_driven__test_2") - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_decorator_returns_same_class(self, mock_load_ddts): - """Test that ddt decorator returns the same class (modified).""" - mock_load_ddts.return_value = [DataDrivenTest({"name": "test_case"})] - - class TestClass(Integration): - pass - - decorated = ddt("test_path")(TestClass) - - assert decorated is TestClass - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_decorator_preserves_existing_methods(self, mock_load_ddts): - """Test that ddt decorator preserves existing test methods.""" - mock_load_ddts.return_value = [DataDrivenTest({"name": "new_test"})] - - @ddt("test_path") - class TestClass(Integration): - def test_existing_method(self): - pass - - assert hasattr(TestClass, "test_existing_method") - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_decorator_with_path_as_pathlib_path(self, mock_load_ddts): - """Test ddt decorator with pathlib.Path object.""" - mock_load_ddts.return_value = [DataDrivenTest({"name": "test1"})] - test_path = Path("test_path") - - @ddt(str(test_path)) - class TestClass(Integration): - pass - - mock_load_ddts.assert_called_once_with(str(test_path), recursive=True) - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_decorator_multiple_classes(self, mock_load_ddts): - """Test that ddt decorator can be applied to multiple classes.""" - mock_ddt = Mock(spec=DataDrivenTest) - mock_ddt.name = "test_case" - mock_ddt.run = AsyncMock() - mock_load_ddts.return_value = [mock_ddt] - - @ddt("test_path") - class TestClass1(Integration): - pass - - @ddt("test_path") - class TestClass2(Integration): - pass - - assert hasattr(TestClass1, "test_data_driven__test_case") - assert hasattr(TestClass2, "test_data_driven__test_case") - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_decorator_with_relative_path(self, mock_load_ddts): - """Test ddt decorator with relative path.""" - mock_load_ddts.return_value = [DataDrivenTest({"name": "test1"})] - - @ddt("./tests/data") - class TestClass(Integration): - pass - - mock_load_ddts.assert_called_once_with("./tests/data", recursive=True) - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_decorator_with_absolute_path(self, mock_load_ddts): - """Test ddt decorator with absolute path.""" - mock_load_ddts.return_value = [DataDrivenTest({"name": "test1"})] - abs_path = Path("/absolute/path/to/tests").as_posix() - - @ddt(abs_path) - class TestClass(Integration): - pass - - mock_load_ddts.assert_called_once_with(abs_path, recursive=True) - - -class TestDdtDecoratorIntegration: - """Integration tests for ddt decorator with actual file loading.""" - - def test_ddt_decorator_loads_real_json_files(self): - """Test ddt decorator with actual JSON files.""" - with tempfile.TemporaryDirectory() as temp_dir: - # Create test file - test_data = { - "name": "real_test", - "test": [ - {"type": "input", "activity": {"type": "message", "text": "Hello"}} - ], - } - test_file = Path(temp_dir) / "test.json" - with open(test_file, "w", encoding="utf-8") as f: - json.dump(test_data, f) - - @ddt(temp_dir, recursive=False) - class TestClass(Integration): - pass - - assert hasattr(TestClass, "test_data_driven__real_test") - - def test_ddt_decorator_loads_real_yaml_files(self): - """Test ddt decorator with actual YAML files.""" - with tempfile.TemporaryDirectory() as temp_dir: - # Create test file - yaml_content = """name: yaml_test -test: - - type: input - activity: - type: message - text: Hello -""" - test_file = Path(temp_dir) / "test.yaml" - with open(test_file, "w", encoding="utf-8") as f: - f.write(yaml_content) - - @ddt(temp_dir, recursive=False) - class TestClass(Integration): - pass - - assert hasattr(TestClass, "test_data_driven__yaml_test") - - def test_ddt_decorator_loads_multiple_files(self): - """Test ddt decorator loading multiple test files.""" - with tempfile.TemporaryDirectory() as temp_dir: - # Create multiple test files - for i in range(3): - test_data = {"name": f"test_{i}", "test": []} - test_file = Path(temp_dir) / f"test_{i}.json" - with open(test_file, "w", encoding="utf-8") as f: - json.dump(test_data, f) - - @ddt(temp_dir, recursive=False) - class TestClass(Integration): - pass - - assert hasattr(TestClass, "test_data_driven__test_0") - assert hasattr(TestClass, "test_data_driven__test_1") - assert hasattr(TestClass, "test_data_driven__test_2") - - def test_ddt_decorator_recursive_loading(self): - """Test ddt decorator with recursive directory loading.""" - with tempfile.TemporaryDirectory() as temp_dir: - # Create subdirectory - sub_dir = Path(temp_dir) / "subdir" - sub_dir.mkdir() - - # Create test in root - root_data = {"name": "root_test", "test": []} - root_file = Path(temp_dir) / "root.json" - with open(root_file, "w", encoding="utf-8") as f: - json.dump(root_data, f) - - # Create test in subdirectory - sub_data = {"name": "sub_test", "test": []} - sub_file = sub_dir / "sub.json" - with open(sub_file, "w", encoding="utf-8") as f: - json.dump(sub_data, f) - - @ddt(temp_dir, recursive=True) - class TestClass(Integration): - pass - - assert hasattr(TestClass, "test_data_driven__root_test") - assert hasattr(TestClass, "test_data_driven__sub_test") - - def test_ddt_decorator_non_recursive_skips_subdirs(self): - """Test that non-recursive mode skips subdirectories.""" - with tempfile.TemporaryDirectory() as temp_dir: - # Create subdirectory - sub_dir = Path(temp_dir) / "subdir" - sub_dir.mkdir() - - # Create test in subdirectory - sub_data = {"name": "sub_test", "test": []} - sub_file = sub_dir / "sub.json" - with open(sub_file, "w", encoding="utf-8") as f: - json.dump(sub_data, f) - - with pytest.raises(Exception): - - @ddt(temp_dir, recursive=False) - class TestClass(Integration): - pass - - @pytest.mark.asyncio - async def test_ddt_decorated_class_runs_tests(self): - """Test that decorated class can actually run the generated tests.""" - with tempfile.TemporaryDirectory() as temp_dir: - # Create test file - test_data = { - "name": "runnable_test", - "test": [ - {"type": "input", "activity": {"type": "message", "text": "Hello"}} - ], - } - test_file = Path(temp_dir) / "test.json" - with open(test_file, "w", encoding="utf-8") as f: - json.dump(test_data, f) - - @ddt(temp_dir, recursive=False) - class TestClass(Integration): - pass - - test_instance = TestClass() - mock_agent_client = AsyncMock(spec=AgentClient) - mock_response_client = AsyncMock(spec=ResponseClient) - - await test_instance.test_data_driven__runnable_test( - mock_agent_client, mock_response_client - ) - - # Verify the test ran - mock_agent_client.send_activity.assert_called_once() - - -class TestDdtDecoratorEdgeCases: - """Tests for edge cases in ddt decorator.""" - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_decorator_with_load_error(self, mock_load_ddts): - """Test ddt decorator behavior when load_ddts raises an error.""" - mock_load_ddts.side_effect = FileNotFoundError("Test files not found") - - with pytest.raises(FileNotFoundError): - - @ddt("nonexistent_path") - class TestClass(Integration): - pass - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_decorator_with_duplicate_test_names(self, mock_load_ddts): - """Test that duplicate test names overwrite previous methods.""" - mock_ddt1 = Mock(spec=DataDrivenTest) - mock_ddt1.name = "test_duplicate" - mock_ddt1.run = AsyncMock(return_value="first") - - mock_ddt2 = Mock(spec=DataDrivenTest) - mock_ddt2.name = "test_duplicate" - mock_ddt2.run = AsyncMock(return_value="second") - - mock_load_ddts.return_value = [mock_ddt1, mock_ddt2] - - @ddt("test_path") - class TestClass(Integration): - pass - - # Second test should overwrite the first - assert hasattr(TestClass, "test_data_driven__test_duplicate") - # Only one method with this name should exist - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_decorator_preserves_class_attributes(self, mock_load_ddts): - """Test that ddt decorator preserves class attributes.""" - mock_load_ddts.return_value = [DataDrivenTest({"name": "test1"})] - - @ddt("test_path") - class TestClass(Integration): - class_attr = "test_value" - _service_url = "http://example.com" - - assert TestClass.class_attr == "test_value" - assert TestClass._service_url == "http://example.com" - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_decorator_preserves_class_docstring(self, mock_load_ddts): - """Test that ddt decorator preserves class docstring.""" - mock_load_ddts.return_value = [DataDrivenTest({"name": "test1"})] - - @ddt("test_path") - class TestClass(Integration): - """This is a test class docstring.""" - - pass - - assert TestClass.__doc__ == "This is a test class docstring." - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_decorator_with_special_characters_in_path(self, mock_load_ddts): - """Test ddt decorator with special characters in path.""" - mock_load_ddts.return_value = [DataDrivenTest({"name": "test1"})] - special_path = "test path/with spaces/and-dashes" - - @ddt(special_path) - class TestClass(Integration): - pass - - mock_load_ddts.assert_called_once_with(special_path, recursive=True) - - def test_ddt_decorator_with_test_name_collision(self): - """Test that generated test names don't collide with existing methods.""" - with tempfile.TemporaryDirectory() as temp_dir: - test_data = {"name": "existing_test", "test": []} - test_file = Path(temp_dir) / "test.json" - with open(test_file, "w", encoding="utf-8") as f: - json.dump(test_data, f) - - @ddt(temp_dir, recursive=False) - class TestClass(Integration): - def test_data_driven__existing_test(self): - """Existing method with same name.""" - return "original" - - # The decorator will overwrite the existing method - assert hasattr(TestClass, "test_data_driven__existing_test") - - -class TestDdtDecoratorWithRealIntegrationClass: - """Tests using actual Integration class features.""" - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_on_integration_subclass(self, mock_load_ddts): - """Test ddt decorator on a proper Integration subclass.""" - mock_ddt = Mock(spec=DataDrivenTest) - mock_ddt.name = "integration_test" - mock_ddt.run = AsyncMock() - mock_load_ddts.return_value = [mock_ddt] - - @ddt("test_path") - class MyIntegrationTest(Integration): - _service_url = "http://localhost:3978" - _agent_url = "http://localhost:8000" - - assert hasattr(MyIntegrationTest, "test_data_driven__integration_test") - assert MyIntegrationTest._service_url == "http://localhost:3978" - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_with_integration_fixtures(self, mock_load_ddts): - """Test that ddt-generated tests can work with Integration fixtures.""" - mock_ddt = Mock(spec=DataDrivenTest) - mock_ddt.name = "fixture_test" - mock_ddt.run = AsyncMock() - mock_load_ddts.return_value = [mock_ddt] - - @ddt("test_path") - class MyIntegrationTest(Integration): - _service_url = "http://localhost:3978" - _agent_url = "http://localhost:8000" - - # The generated method should accept agent_client and response_client parameters - import inspect - - method = getattr(MyIntegrationTest, "test_data_driven__fixture_test") - sig = inspect.signature(method) - params = list(sig.parameters.keys()) - - assert "self" in params - assert "agent_client" in params - assert "response_client" in params - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_multiple_decorators_on_same_class(self, mock_load_ddts): - """Test applying multiple ddt decorators to the same class.""" - mock_ddt1 = Mock(spec=DataDrivenTest) - mock_ddt1.name = "test_1" - mock_ddt1.run = AsyncMock() - - mock_ddt2 = Mock(spec=DataDrivenTest) - mock_ddt2.name = "test_2" - mock_ddt2.run = AsyncMock() - - mock_load_ddts.side_effect = [[mock_ddt1], [mock_ddt2]] - - @ddt("path2") - @ddt("path1") - class TestClass(Integration): - pass - - assert hasattr(TestClass, "test_data_driven__test_1") - assert hasattr(TestClass, "test_data_driven__test_2") - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_ddt_decorator_return_type(self, mock_load_ddts): - """Test that ddt decorator returns the correct type.""" - mock_load_ddts.return_value = [DataDrivenTest({"name": "test1"})] - - class TestClass(Integration): - pass - - decorated = ddt("test_path")(TestClass) - - assert isinstance(decorated, type) - assert issubclass(decorated, Integration) - - -class TestDdtDecoratorDocumentation: - """Tests related to documentation and metadata.""" - - def test_ddt_function_has_docstring(self): - """Test that ddt function has proper documentation.""" - assert ddt.__doc__ is not None - assert "data driven tests" in ddt.__doc__.lower() - - def test_add_test_method_has_docstring(self): - """Test that _add_test_method has proper documentation.""" - assert _add_test_method.__doc__ is not None - - @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") - def test_generated_test_methods_are_discoverable(self, mock_load_ddts): - """Test that generated test methods are discoverable by pytest.""" - mock_ddt = Mock(spec=DataDrivenTest) - mock_ddt.name = "discoverable_test" - mock_ddt.run = AsyncMock() - mock_load_ddts.return_value = [mock_ddt] - - @ddt("test_path") - class TestClass(Integration): - pass - - # Check that the method name starts with 'test_' so pytest can discover it - method_name = "test_data_driven__discoverable_test" - assert hasattr(TestClass, method_name) - assert method_name.startswith("test_") diff --git a/dev/microsoft-agents-testing/tests/integration/data_driven/test_load_ddts.py b/dev/microsoft-agents-testing/tests/integration/data_driven/test_load_ddts.py deleted file mode 100644 index 75c28686..00000000 --- a/dev/microsoft-agents-testing/tests/integration/data_driven/test_load_ddts.py +++ /dev/null @@ -1,362 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import json -import pytest -import tempfile -import os -from pathlib import Path - -from microsoft_agents.testing.integration.data_driven import DataDrivenTest -from microsoft_agents.testing.integration.data_driven.load_ddts import load_ddts - - -class TestLoadDdts: - """Tests for load_ddts function.""" - - def test_load_ddts_from_empty_directory(self): - """Test loading from an empty directory returns empty list.""" - with tempfile.TemporaryDirectory() as temp_dir: - result = load_ddts(temp_dir, recursive=False) - assert result == [] - - def test_load_single_json_file(self): - """Test loading a single JSON test file.""" - with tempfile.TemporaryDirectory() as temp_dir: - test_data = { - "name": "test1", - "description": "Test 1", - "test": [{"type": "input", "activity": {"text": "Hello"}}], - } - - json_file = Path(temp_dir) / "test1.json" - with open(json_file, "w", encoding="utf-8") as f: - json.dump(test_data, f) - - result = load_ddts(temp_dir, recursive=False) - - assert len(result) == 1 - assert isinstance(result[0], DataDrivenTest) - assert result[0]._name == "test1" - - def test_load_single_yaml_file(self): - """Test loading a single YAML test file.""" - with tempfile.TemporaryDirectory() as temp_dir: - yaml_content = """name: test1 -description: Test 1 -test: - - type: input - activity: - text: Hello -""" - - yaml_file = Path(temp_dir) / "test1.yaml" - with open(yaml_file, "w", encoding="utf-8") as f: - f.write(yaml_content) - - result = load_ddts(temp_dir, recursive=False) - - assert len(result) == 1 - assert isinstance(result[0], DataDrivenTest) - assert result[0]._name == "test1" - - def test_load_multiple_files(self): - """Test loading multiple test files.""" - with tempfile.TemporaryDirectory() as temp_dir: - # Create JSON file - json_data = { - "name": "json_test", - "test": [{"type": "input", "activity": {"text": "Hello"}}], - } - json_file = Path(temp_dir) / "test1.json" - with open(json_file, "w", encoding="utf-8") as f: - json.dump(json_data, f) - - # Create YAML file - yaml_content = """name: yaml_test -test: - - type: input - activity: - text: World -""" - yaml_file = Path(temp_dir) / "test2.yaml" - with open(yaml_file, "w", encoding="utf-8") as f: - f.write(yaml_content) - - result = load_ddts(temp_dir, recursive=False) - - assert len(result) == 2 - names = {test._name for test in result} - assert "json_test" in names - assert "yaml_test" in names - - def test_load_recursive(self): - """Test loading files recursively from subdirectories.""" - with tempfile.TemporaryDirectory() as temp_dir: - # Create subdirectory - sub_dir = Path(temp_dir) / "subdir" - sub_dir.mkdir() - - # Create file in root - root_data = {"name": "root_test", "test": []} - root_file = Path(temp_dir) / "root.json" - with open(root_file, "w", encoding="utf-8") as f: - json.dump(root_data, f) - - # Create file in subdirectory - sub_data = {"name": "sub_test", "test": []} - sub_file = sub_dir / "sub.json" - with open(sub_file, "w", encoding="utf-8") as f: - json.dump(sub_data, f) - - # Non-recursive should find only root file - result_non_recursive = load_ddts(temp_dir, recursive=False) - assert len(result_non_recursive) == 1 - assert result_non_recursive[0]._name == "root_test" - - # Recursive should find both files - result_recursive = load_ddts(temp_dir, recursive=True) - assert len(result_recursive) == 2 - names = {test._name for test in result_recursive} - assert "root_test" in names - assert "sub_test" in names - - def test_load_with_parent_reference(self): - """Test loading files with parent references.""" - with tempfile.TemporaryDirectory() as temp_dir: - # Create parent file - parent_data = { - "name": "parent", - "defaults": { - "input": {"activity": {"type": "message", "locale": "en-US"}} - }, - } - parent_file = Path(temp_dir) / "parent.json" - with open(parent_file, "w", encoding="utf-8") as f: - json.dump(parent_data, f) - - # Create child file with parent reference - child_data = { - "name": "child", - "parent": str(parent_file), - "test": [{"type": "input", "activity": {"text": "Hello"}}], - } - child_file = Path(temp_dir) / "child.json" - with open(child_file, "w", encoding="utf-8") as f: - json.dump(child_data, f) - - result = load_ddts(temp_dir, recursive=False) - - # Should load both files - assert len(result) == 1 - - # Find the child test - child_test = next((t for t in result if t._name == "parent.child"), None) - assert child_test is not None - - # Child should have inherited defaults from parent - assert child_test._input_defaults == { - "activity": {"type": "message", "locale": "en-US"} - } - - def test_load_with_relative_parent_reference(self): - """Test loading files with relative parent references.""" - with tempfile.TemporaryDirectory() as temp_dir: - # Create parent file - parent_data = { - "name": "parent", - "defaults": {"input": {"activity": {"type": "message"}}}, - } - parent_file = Path(temp_dir) / "parent.yaml" - with open(parent_file, "w", encoding="utf-8") as f: - f.write( - "name: parent\ndefaults:\n input:\n activity:\n type: message\n" - ) - - # Create child file with relative parent reference - child_data = {"name": "child", "parent": "parent.yaml", "test": []} - child_file = Path(temp_dir) / "child.json" - with open(child_file, "w", encoding="utf-8") as f: - json.dump(child_data, f) - - # Change to temp_dir so relative path works - original_dir = os.getcwd() - try: - os.chdir(temp_dir) - result = load_ddts(temp_dir, recursive=False) - - assert len(result) == 1 - child_test = next( - (t for t in result if t._name == "parent.child"), None - ) - assert child_test is not None - finally: - os.chdir(original_dir) - - def test_load_with_nested_parent_references(self): - """Test loading files with nested parent references (grandparent -> parent -> child).""" - with tempfile.TemporaryDirectory() as temp_dir: - # Create grandparent file - grandparent_data = { - "name": "grandparent", - "defaults": {"input": {"activity": {"type": "message"}}}, - } - grandparent_file = Path(temp_dir) / "grandparent.json" - with open(grandparent_file, "w", encoding="utf-8") as f: - json.dump(grandparent_data, f) - - # Create parent file referencing grandparent - parent_data = { - "name": "parent", - "parent": str(grandparent_file), - "defaults": {"input": {"activity": {"locale": "en-US"}}}, - } - parent_file = Path(temp_dir) / "parent.json" - with open(parent_file, "w", encoding="utf-8") as f: - json.dump(parent_data, f) - - # Create child file referencing parent - child_data = {"name": "child", "parent": str(parent_file), "test": []} - child_file = Path(temp_dir) / "child.json" - with open(child_file, "w", encoding="utf-8") as f: - json.dump(child_data, f) - - result = load_ddts(temp_dir, recursive=False) - - # Should load all three files - assert len(result) == 1 - - # Verify child has inherited all defaults - child_test = next( - (t for t in result if t._name == "grandparent.parent.child"), None - ) - assert child_test is not None - - def test_load_with_missing_parent_raises_error(self): - """Test that referencing a non-existent parent file raises an error.""" - with tempfile.TemporaryDirectory() as temp_dir: - # Create child file with non-existent parent reference - child_data = { - "name": "child", - "parent": str(Path(temp_dir) / "nonexistent.json"), - "test": [], - } - child_file = Path(temp_dir) / "child.json" - with open(child_file, "w", encoding="utf-8") as f: - json.dump(child_data, f) - - with pytest.raises(Exception): - load_ddts(temp_dir, recursive=False) - - def test_load_sets_name_from_filename_when_missing(self): - """Test that name is set from filename when not provided in test data.""" - with tempfile.TemporaryDirectory() as temp_dir: - # Create file without name field - test_data = {"test": [{"type": "input", "activity": {"text": "Hello"}}]} - test_file = Path(temp_dir) / "my_test_file.json" - with open(test_file, "w", encoding="utf-8") as f: - json.dump(test_data, f) - - result = load_ddts(temp_dir, recursive=False) - - assert len(result) == 1 - assert result[0]._name == "my_test_file" - - def test_load_uses_current_working_directory_when_path_is_none(self): - """Test that load_ddts uses current working directory when path is None.""" - with tempfile.TemporaryDirectory() as temp_dir: - # Create test file - test_data = {"name": "test", "test": []} - test_file = Path(temp_dir) / "test.json" - with open(test_file, "w", encoding="utf-8") as f: - json.dump(test_data, f) - - # Change to temp_dir and load without path - original_dir = os.getcwd() - try: - os.chdir(temp_dir) - result = load_ddts(None, recursive=False) - - assert len(result) == 1 - assert result[0]._name == "test" - finally: - os.chdir(original_dir) - - def test_load_resolves_parent_to_absolute_path(self): - """Test that parent references are resolved to absolute paths.""" - with tempfile.TemporaryDirectory() as temp_dir: - # Create parent file - parent_data = { - "name": "parent", - "defaults": {"input": {"activity": {"type": "message"}}}, - } - parent_file = Path(temp_dir) / "parent.json" - with open(parent_file, "w", encoding="utf-8") as f: - json.dump(parent_data, f) - - # Create child with parent reference - child_data = {"name": "child", "parent": str(parent_file), "test": []} - child_file = Path(temp_dir) / "child.json" - with open(child_file, "w", encoding="utf-8") as f: - json.dump(child_data, f) - - result = load_ddts(temp_dir, recursive=False) - - # Find child and verify parent is a dict (resolved) - child_test = next((t for t in result if t._name == "parent.child"), None) - assert child_test is not None - - def test_load_handles_mixed_json_and_yaml_files(self): - """Test loading both JSON and YAML files together.""" - with tempfile.TemporaryDirectory() as temp_dir: - # Create JSON parent - parent_data = { - "name": "json_parent", - "defaults": {"input": {"activity": {"type": "message"}}}, - } - parent_file = Path(temp_dir) / "parent.json" - with open(parent_file, "w", encoding="utf-8") as f: - json.dump(parent_data, f) - - # Create YAML child referencing JSON parent - yaml_content = f"""name: yaml_child -parent: {parent_file} -test: [] -""" - child_file = Path(temp_dir) / "child.yaml" - with open(child_file, "w", encoding="utf-8") as f: - f.write(yaml_content) - - result = load_ddts(temp_dir, recursive=False) - - assert len(result) == 1 - names = {test._name for test in result} - assert "json_parent.yaml_child" in names - - def test_load_with_path_as_string(self): - """Test that path parameter accepts string type.""" - with tempfile.TemporaryDirectory() as temp_dir: - test_data = {"name": "test", "test": []} - test_file = Path(temp_dir) / "test.json" - with open(test_file, "w", encoding="utf-8") as f: - json.dump(test_data, f) - - # Pass path as string instead of Path object - result = load_ddts(str(temp_dir), recursive=False) - - assert len(result) == 1 - assert result[0]._name == "test" - - def test_load_with_path_as_path_object(self): - """Test that path parameter accepts Path object.""" - with tempfile.TemporaryDirectory() as temp_dir: - test_data = {"name": "test", "test": []} - test_file = Path(temp_dir) / "test.json" - with open(test_file, "w", encoding="utf-8") as f: - json.dump(test_data, f) - - # Pass path as Path object - result = load_ddts(Path(temp_dir), recursive=False) - - assert len(result) == 1 - assert result[0]._name == "test" diff --git a/dev/microsoft-agents-testing/tests/integration/core/test_application_runner.py b/dev/microsoft-agents-testing/tests/integration/test_application_runner.py similarity index 100% rename from dev/microsoft-agents-testing/tests/integration/core/test_application_runner.py rename to dev/microsoft-agents-testing/tests/integration/test_application_runner.py diff --git a/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_sample.py b/dev/microsoft-agents-testing/tests/integration/test_integration_from_sample.py similarity index 100% rename from dev/microsoft-agents-testing/tests/integration/core/test_integration_from_sample.py rename to dev/microsoft-agents-testing/tests/integration/test_integration_from_sample.py diff --git a/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py b/dev/microsoft-agents-testing/tests/integration/test_integration_from_service_url.py similarity index 100% rename from dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py rename to dev/microsoft-agents-testing/tests/integration/test_integration_from_service_url.py diff --git a/dev/payload.json b/dev/payload.json new file mode 100644 index 00000000..399023d6 --- /dev/null +++ b/dev/payload.json @@ -0,0 +1,20 @@ +{ + "channelId": "msteams", + "serviceUrl": "http://localhost:49231/_connector", + "delivery_mode": "expectReplies", + "recipient": { + "id": "00000000-0000-0000-0000-00000000000011", + "name": "Test Bot" + }, + "conversation": { + "id": "personal-chat-id", + "conversationType": "personal", + "tenantId": "00000000-0000-0000-0000-0000000000001" + }, + "from": { + "id": "user-id-0", + "aadObjectId": "00000000-0000-0000-0000-0000000000020" + }, + "type": "message", + "text": "Hello, Bot!" +} \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/integration/core/client/__init__.py b/dev/tests/E2E/__init__.py similarity index 100% rename from dev/microsoft-agents-testing/tests/integration/core/client/__init__.py rename to dev/tests/E2E/__init__.py diff --git a/dev/microsoft-agents-testing/tests/integration/data_driven/__init__.py b/dev/tests/E2E/basic_agent/__init__.py similarity index 100% rename from dev/microsoft-agents-testing/tests/integration/data_driven/__init__.py rename to dev/tests/E2E/basic_agent/__init__.py diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml b/dev/tests/E2E/basic_agent/directline/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml similarity index 100% rename from dev/integration/tests/basic_agent/directline/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml rename to dev/tests/E2E/basic_agent/directline/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_EndConversation_DeleteConversation.yaml b/dev/tests/E2E/basic_agent/directline/SendActivity_EndConversation_DeleteConversation.yaml similarity index 100% rename from dev/integration/tests/basic_agent/directline/SendActivity_EndConversation_DeleteConversation.yaml rename to dev/tests/E2E/basic_agent/directline/SendActivity_EndConversation_DeleteConversation.yaml diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml b/dev/tests/E2E/basic_agent/directline/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml similarity index 100% rename from dev/integration/tests/basic_agent/directline/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml rename to dev/tests/E2E/basic_agent/directline/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml b/dev/tests/E2E/basic_agent/directline/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml similarity index 100% rename from dev/integration/tests/basic_agent/directline/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml rename to dev/tests/E2E/basic_agent/directline/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml b/dev/tests/E2E/basic_agent/directline/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml similarity index 100% rename from dev/integration/tests/basic_agent/directline/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml rename to dev/tests/E2E/basic_agent/directline/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_SendsHi5_Returns5HiActivities.yaml b/dev/tests/E2E/basic_agent/directline/SendActivity_SendsHi5_Returns5HiActivities.yaml similarity index 99% rename from dev/integration/tests/basic_agent/directline/SendActivity_SendsHi5_Returns5HiActivities.yaml rename to dev/tests/E2E/basic_agent/directline/SendActivity_SendsHi5_Returns5HiActivities.yaml index 0227d47a..4368f378 100644 --- a/dev/integration/tests/basic_agent/directline/SendActivity_SendsHi5_Returns5HiActivities.yaml +++ b/dev/tests/E2E/basic_agent/directline/SendActivity_SendsHi5_Returns5HiActivities.yaml @@ -14,6 +14,7 @@ test: name: Test Bot text: hi 5 locale: en-US +- skip - type: assertion quantifier: one activity: diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml b/dev/tests/E2E/basic_agent/directline/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml similarity index 100% rename from dev/integration/tests/basic_agent/directline/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml rename to dev/tests/E2E/basic_agent/directline/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml b/dev/tests/E2E/basic_agent/directline/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml similarity index 70% rename from dev/integration/tests/basic_agent/directline/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml rename to dev/tests/E2E/basic_agent/directline/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml index e7d593c5..41398805 100644 --- a/dev/integration/tests/basic_agent/directline/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml +++ b/dev/tests/E2E/basic_agent/directline/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml @@ -13,12 +13,12 @@ test: name: Bot text: 'w: What''s the weather in Seattle today?' locale: en-US -- type: skip - type: assertion selector: - index: -1 + activity: + attachments: + - contentType: application/vnd.microsoft.card.adaptive activity: type: message attachments: - - contentType: application/vnd.microsoft.card.adaptive - content: ["RE_MATCH", "(๏ฟฝ|\\u00B0|Missing temperature inside adaptive card:)"] \ No newline at end of file + - content: ["RE_MATCH", "(๏ฟฝ|\\u00B0|Missing temperature inside adaptive card:)"] \ No newline at end of file diff --git a/dev/tests/E2E/basic_agent/directline/SendActivity_SendsText_ReturnsPoem.yaml b/dev/tests/E2E/basic_agent/directline/SendActivity_SendsText_ReturnsPoem.yaml new file mode 100644 index 00000000..bf0f93bd --- /dev/null +++ b/dev/tests/E2E/basic_agent/directline/SendActivity_SendsText_ReturnsPoem.yaml @@ -0,0 +1,27 @@ +test: +- type: input + activity: + type: message + channelId: directline + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + text: poem + locale: en-US +- type: assertion + selector: + activity: + type: typing + quantifier: any + activity: + text: ["CONTAINS", "Hold on for an awesome poem about Apollo"] +- type: assertion + selector: + index: -1 + activity: + text: ["CONTAINS", "Apollo"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml b/dev/tests/E2E/basic_agent/directline/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml similarity index 100% rename from dev/integration/tests/basic_agent/directline/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml rename to dev/tests/E2E/basic_agent/directline/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml diff --git a/dev/integration/tests/basic_agent/directline/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml b/dev/tests/E2E/basic_agent/directline/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml similarity index 83% rename from dev/integration/tests/basic_agent/directline/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml rename to dev/tests/E2E/basic_agent/directline/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml index 2477faea..f04b6a71 100644 --- a/dev/integration/tests/basic_agent/directline/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml +++ b/dev/tests/E2E/basic_agent/directline/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml @@ -14,12 +14,12 @@ test: name: Bot text: 'w: What''s the weather in Seattle today?''' locale: en-US -- type: skip - type: assertion selector: - index: -1 + activity: + attachments: + - contentType: application/vnd.microsoft.card.adaptive activity: type: message attachments: - - contentType: application/vnd.microsoft.card.adaptive content: ["RE_MATCH", "(๏ฟฝ|\\u00B0|Missing temperature inside adaptive card:)"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml b/dev/tests/E2E/basic_agent/directline/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml similarity index 100% rename from dev/integration/tests/basic_agent/directline/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml rename to dev/tests/E2E/basic_agent/directline/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml diff --git a/dev/integration/tests/basic_agent/directline/SendInvoke_QueryLink_ReturnsText.yaml b/dev/tests/E2E/basic_agent/directline/SendInvoke_QueryLink_ReturnsText.yaml similarity index 100% rename from dev/integration/tests/basic_agent/directline/SendInvoke_QueryLink_ReturnsText.yaml rename to dev/tests/E2E/basic_agent/directline/SendInvoke_QueryLink_ReturnsText.yaml diff --git a/dev/integration/tests/basic_agent/directline/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml b/dev/tests/E2E/basic_agent/directline/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml similarity index 100% rename from dev/integration/tests/basic_agent/directline/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml rename to dev/tests/E2E/basic_agent/directline/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml diff --git a/dev/integration/tests/basic_agent/directline/SendInvoke_SelectItem_ReceiveItem.yaml b/dev/tests/E2E/basic_agent/directline/SendInvoke_SelectItem_ReceiveItem.yaml similarity index 71% rename from dev/integration/tests/basic_agent/directline/SendInvoke_SelectItem_ReceiveItem.yaml rename to dev/tests/E2E/basic_agent/directline/SendInvoke_SelectItem_ReceiveItem.yaml index 4a843c50..3452742e 100644 --- a/dev/integration/tests/basic_agent/directline/SendInvoke_SelectItem_ReceiveItem.yaml +++ b/dev/tests/E2E/basic_agent/directline/SendInvoke_SelectItem_ReceiveItem.yaml @@ -22,10 +22,12 @@ test: name: composeExtension/selectItem locale: en-US - type: assertion - invokeResponse: - composeExtension: - type: result - text: ["CONTAINS", "Newtonsoft.Json"] - attachments: - contentType: application/vnd.microsoft.card.thumbnail -- type: skip \ No newline at end of file + activity: + type: invoke_response + value: + body: + composeExtension: + type: result + text: ["CONTAINS", "Newtonsoft.Json"] + attachments: + contentType: application/vnd.microsoft.card.thumbnail \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml b/dev/tests/E2E/basic_agent/directline/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml similarity index 77% rename from dev/integration/tests/basic_agent/directline/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml rename to dev/tests/E2E/basic_agent/directline/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml index 85f13369..5ab221e6 100644 --- a/dev/integration/tests/basic_agent/directline/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml +++ b/dev/tests/E2E/basic_agent/directline/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml @@ -30,9 +30,12 @@ test: parameters: - value: hi` - type: assertion - invokeResponse: - message: ["EQUALS", "Invoke received."] - status: 200 - data: - parameters: - - value: ["CONTAINS", "hi"] \ No newline at end of file + activity: + value: + body: + composeExtension: + message: ["EQUALS", "Invoke received."] + status: 200 + data: + parameters: + - value: ["CONTAINS", "hi"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml b/dev/tests/E2E/basic_agent/directline/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml similarity index 100% rename from dev/integration/tests/basic_agent/directline/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml rename to dev/tests/E2E/basic_agent/directline/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml diff --git a/dev/integration/tests/basic_agent/directline/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml b/dev/tests/E2E/basic_agent/directline/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml similarity index 100% rename from dev/integration/tests/basic_agent/directline/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml rename to dev/tests/E2E/basic_agent/directline/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml b/dev/tests/E2E/basic_agent/msteams/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml rename to dev/tests/E2E/basic_agent/msteams/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_EditMessage_ReceiveUpdate.yaml b/dev/tests/E2E/basic_agent/msteams/SendActivity_EditMessage_ReceiveUpdate.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendActivity_EditMessage_ReceiveUpdate.yaml rename to dev/tests/E2E/basic_agent/msteams/SendActivity_EditMessage_ReceiveUpdate.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_EndConversation_DeleteConversation.yaml b/dev/tests/E2E/basic_agent/msteams/SendActivity_EndConversation_DeleteConversation.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendActivity_EndConversation_DeleteConversation.yaml rename to dev/tests/E2E/basic_agent/msteams/SendActivity_EndConversation_DeleteConversation.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_EndTeamsMeeting_ExpectMessage.yaml b/dev/tests/E2E/basic_agent/msteams/SendActivity_EndTeamsMeeting_ExpectMessage.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendActivity_EndTeamsMeeting_ExpectMessage.yaml rename to dev/tests/E2E/basic_agent/msteams/SendActivity_EndTeamsMeeting_ExpectMessage.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_ParticipantJoinsTeamMeeting_ExpectMessage.yaml b/dev/tests/E2E/basic_agent/msteams/SendActivity_ParticipantJoinsTeamMeeting_ExpectMessage.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendActivity_ParticipantJoinsTeamMeeting_ExpectMessage.yaml rename to dev/tests/E2E/basic_agent/msteams/SendActivity_ParticipantJoinsTeamMeeting_ExpectMessage.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml b/dev/tests/E2E/basic_agent/msteams/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml rename to dev/tests/E2E/basic_agent/msteams/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml b/dev/tests/E2E/basic_agent/msteams/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml rename to dev/tests/E2E/basic_agent/msteams/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml b/dev/tests/E2E/basic_agent/msteams/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml rename to dev/tests/E2E/basic_agent/msteams/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_SendsHi5_Returns5HiActivities.yaml b/dev/tests/E2E/basic_agent/msteams/SendActivity_SendsHi5_Returns5HiActivities.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendActivity_SendsHi5_Returns5HiActivities.yaml rename to dev/tests/E2E/basic_agent/msteams/SendActivity_SendsHi5_Returns5HiActivities.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml b/dev/tests/E2E/basic_agent/msteams/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml rename to dev/tests/E2E/basic_agent/msteams/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml b/dev/tests/E2E/basic_agent/msteams/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml rename to dev/tests/E2E/basic_agent/msteams/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_SendsText_ReturnsPoem.yaml b/dev/tests/E2E/basic_agent/msteams/SendActivity_SendsText_ReturnsPoem.yaml similarity index 94% rename from dev/integration/tests/basic_agent/msteams/SendActivity_SendsText_ReturnsPoem.yaml rename to dev/tests/E2E/basic_agent/msteams/SendActivity_SendsText_ReturnsPoem.yaml index 5313bde1..7bf40833 100644 --- a/dev/integration/tests/basic_agent/msteams/SendActivity_SendsText_ReturnsPoem.yaml +++ b/dev/tests/E2E/basic_agent/msteams/SendActivity_SendsText_ReturnsPoem.yaml @@ -38,5 +38,4 @@ test: selector: index: -1 activity: - text: ["CONTAINS", "Apollo"] -- type: breakpoint \ No newline at end of file + text: ["CONTAINS", "Apollo"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml b/dev/tests/E2E/basic_agent/msteams/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml rename to dev/tests/E2E/basic_agent/msteams/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_StartTeamsMeeting_ExpectMessage.yaml b/dev/tests/E2E/basic_agent/msteams/SendActivity_StartTeamsMeeting_ExpectMessage.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendActivity_StartTeamsMeeting_ExpectMessage.yaml rename to dev/tests/E2E/basic_agent/msteams/SendActivity_StartTeamsMeeting_ExpectMessage.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml b/dev/tests/E2E/basic_agent/msteams/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml rename to dev/tests/E2E/basic_agent/msteams/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml b/dev/tests/E2E/basic_agent/msteams/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml rename to dev/tests/E2E/basic_agent/msteams/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendInvoke_QueryLink_ReturnsText.yaml b/dev/tests/E2E/basic_agent/msteams/SendInvoke_QueryLink_ReturnsText.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendInvoke_QueryLink_ReturnsText.yaml rename to dev/tests/E2E/basic_agent/msteams/SendInvoke_QueryLink_ReturnsText.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml b/dev/tests/E2E/basic_agent/msteams/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml rename to dev/tests/E2E/basic_agent/msteams/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendInvoke_SelectItem_ReceiveItem.yaml b/dev/tests/E2E/basic_agent/msteams/SendInvoke_SelectItem_ReceiveItem.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendInvoke_SelectItem_ReceiveItem.yaml rename to dev/tests/E2E/basic_agent/msteams/SendInvoke_SelectItem_ReceiveItem.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml b/dev/tests/E2E/basic_agent/msteams/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml rename to dev/tests/E2E/basic_agent/msteams/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml b/dev/tests/E2E/basic_agent/msteams/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml rename to dev/tests/E2E/basic_agent/msteams/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml diff --git a/dev/integration/tests/basic_agent/msteams/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml b/dev/tests/E2E/basic_agent/msteams/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml similarity index 100% rename from dev/integration/tests/basic_agent/msteams/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml rename to dev/tests/E2E/basic_agent/msteams/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml diff --git a/dev/integration/tests/basic_agent/test_basic_agent.py b/dev/tests/E2E/basic_agent/test_basic_agent.py similarity index 50% rename from dev/integration/tests/basic_agent/test_basic_agent.py rename to dev/tests/E2E/basic_agent/test_basic_agent.py index 840ab4cb..738c0269 100644 --- a/dev/integration/tests/basic_agent/test_basic_agent.py +++ b/dev/tests/E2E/basic_agent/test_basic_agent.py @@ -1,15 +1,16 @@ import pytest from microsoft_agents.testing import ( - ddt, Integration, ) +TEST_BASIC_AGENT = True -@ddt("tests/basic_agent/directline", prefix="directline") -@ddt("tests/basic_agent/webchat", prefix="webchat") -@ddt("tests/basic_agent/msteams", prefix="msteams") -class TestBasicAgent(Integration): + +@pytest.mark.skipif( + not TEST_BASIC_AGENT, reason="Skipping external agent tests for now." +) +class TestBasicAgentExternal(Integration): _agent_url = "http://localhost:3978/" _service_url = "http://localhost:8001/" _config_path = "agents/basic_agent/python/.env" diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml b/dev/tests/E2E/basic_agent/webchat/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml rename to dev/tests/E2E/basic_agent/webchat/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_EndConversation_DeleteConversation.yaml b/dev/tests/E2E/basic_agent/webchat/SendActivity_EndConversation_DeleteConversation.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendActivity_EndConversation_DeleteConversation.yaml rename to dev/tests/E2E/basic_agent/webchat/SendActivity_EndConversation_DeleteConversation.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml b/dev/tests/E2E/basic_agent/webchat/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml rename to dev/tests/E2E/basic_agent/webchat/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml b/dev/tests/E2E/basic_agent/webchat/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml rename to dev/tests/E2E/basic_agent/webchat/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml b/dev/tests/E2E/basic_agent/webchat/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml rename to dev/tests/E2E/basic_agent/webchat/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_SendsHi5_Returns5HiActivities.yaml b/dev/tests/E2E/basic_agent/webchat/SendActivity_SendsHi5_Returns5HiActivities.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendActivity_SendsHi5_Returns5HiActivities.yaml rename to dev/tests/E2E/basic_agent/webchat/SendActivity_SendsHi5_Returns5HiActivities.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml b/dev/tests/E2E/basic_agent/webchat/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml rename to dev/tests/E2E/basic_agent/webchat/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml b/dev/tests/E2E/basic_agent/webchat/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml rename to dev/tests/E2E/basic_agent/webchat/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_SendsText_ReturnsPoem.yaml b/dev/tests/E2E/basic_agent/webchat/SendActivity_SendsText_ReturnsPoem.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendActivity_SendsText_ReturnsPoem.yaml rename to dev/tests/E2E/basic_agent/webchat/SendActivity_SendsText_ReturnsPoem.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml b/dev/tests/E2E/basic_agent/webchat/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml rename to dev/tests/E2E/basic_agent/webchat/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml b/dev/tests/E2E/basic_agent/webchat/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml rename to dev/tests/E2E/basic_agent/webchat/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml b/dev/tests/E2E/basic_agent/webchat/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml rename to dev/tests/E2E/basic_agent/webchat/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendInvoke_QueryLink_ReturnsText.yaml b/dev/tests/E2E/basic_agent/webchat/SendInvoke_QueryLink_ReturnsText.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendInvoke_QueryLink_ReturnsText.yaml rename to dev/tests/E2E/basic_agent/webchat/SendInvoke_QueryLink_ReturnsText.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml b/dev/tests/E2E/basic_agent/webchat/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml rename to dev/tests/E2E/basic_agent/webchat/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendInvoke_SelectItem_ReceiveItem.yaml b/dev/tests/E2E/basic_agent/webchat/SendInvoke_SelectItem_ReceiveItem.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendInvoke_SelectItem_ReceiveItem.yaml rename to dev/tests/E2E/basic_agent/webchat/SendInvoke_SelectItem_ReceiveItem.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml b/dev/tests/E2E/basic_agent/webchat/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml rename to dev/tests/E2E/basic_agent/webchat/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml b/dev/tests/E2E/basic_agent/webchat/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml rename to dev/tests/E2E/basic_agent/webchat/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml diff --git a/dev/integration/tests/basic_agent/webchat/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml b/dev/tests/E2E/basic_agent/webchat/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml similarity index 100% rename from dev/integration/tests/basic_agent/webchat/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml rename to dev/tests/E2E/basic_agent/webchat/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml diff --git a/dev/tests/E2E/quickstart/__init__.py b/dev/tests/E2E/quickstart/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/tests/quickstart/directline/_parent.yaml b/dev/tests/E2E/quickstart/directline/_parent.yaml similarity index 100% rename from dev/integration/tests/quickstart/directline/_parent.yaml rename to dev/tests/E2E/quickstart/directline/_parent.yaml diff --git a/dev/integration/tests/quickstart/directline/conversation_update.yaml b/dev/tests/E2E/quickstart/directline/conversation_update.yaml similarity index 93% rename from dev/integration/tests/quickstart/directline/conversation_update.yaml rename to dev/tests/E2E/quickstart/directline/conversation_update.yaml index 3ff217c9..3a8824e4 100644 --- a/dev/integration/tests/quickstart/directline/conversation_update.yaml +++ b/dev/tests/E2E/quickstart/directline/conversation_update.yaml @@ -33,4 +33,4 @@ test: type: message activity: type: message - text: ["CONTAINS", "Welcome to the empty agent!"] + text: ["CONTAINS", "Welcome"] diff --git a/dev/integration/tests/quickstart/directline/send_hello.yaml b/dev/tests/E2E/quickstart/directline/send_hello.yaml similarity index 100% rename from dev/integration/tests/quickstart/directline/send_hello.yaml rename to dev/tests/E2E/quickstart/directline/send_hello.yaml diff --git a/dev/integration/tests/quickstart/directline/send_hi.yaml b/dev/tests/E2E/quickstart/directline/send_hi.yaml similarity index 100% rename from dev/integration/tests/quickstart/directline/send_hi.yaml rename to dev/tests/E2E/quickstart/directline/send_hi.yaml diff --git a/dev/tests/E2E/quickstart/test_quickstart.py b/dev/tests/E2E/quickstart/test_quickstart.py new file mode 100644 index 00000000..251e9822 --- /dev/null +++ b/dev/tests/E2E/quickstart/test_quickstart.py @@ -0,0 +1,107 @@ +import pytest +import asyncio + +from microsoft_agents.activity import Activity + +from microsoft_agents.testing import ( + Integration, + AiohttpEnvironment, + update_with_defaults, + AgentClient, + ResponseClient, + ModelQuery +) + +from ...samples import QuickstartSample + +class TestQuickstartBase(Integration): + _sample_cls = QuickstartSample + _environment_cls = AiohttpEnvironment + + OUTGOING_PARENT = { + "channel_id": "webchat", + "locale": "en-US", + "conversation": {"id": "conv1"}, + "from": {"id": "user1", "name": "User"}, + "recipient": {"id": "bot", "name": "Bot"}, + } + + def populate(self, input_data: dict | None = None, **kwargs) -> Activity: + if not input_data: + input_data = {} + input_data.update(kwargs) + update_with_defaults(input_data, self.OUTGOING_PARENT) + return Activity.model_validate(input_data) + + @pytest.mark.asyncio + async def test_conversation_update(self, agent_client: AgentClient, response_client: ResponseClient): + + input_activity = self.populate( + type="conversationUpdate", + members_added=[ + {"id": "bot-id", "name": "Bot"}, + {"id": "user1", "name": "User"}, + ], + textFormat="plain", + entities=[ + { + "type": "ClientCapabilities", + "requiresBotState": True, + "supportsTts": True + } + ], + channel_data={"clientActivityId": 123} + ) + + await agent_client.send_activity(input_activity) + + await asyncio.sleep(1) # Wait for processing + + responses = await response_client.pop() + + activity = ModelQuery(type="message").first(responses) + + assert activity is not None + assert "Welcome" in activity.text + + @pytest.mark.asyncio + async def test_send_hello(self, agent_client: AgentClient, response_client: ResponseClient): + + input_activity = self.populate( + type="message", + text="hello", + ) + + await agent_client.send_activity(input_activity) + + await asyncio.sleep(1) # Wait for processing + + responses = await response_client.pop() + + activity = ModelQuery(type="message").first(responses) + + assert activity is not None + assert activity.text == "Hello!" + + @pytest.mark.asyncio + async def test_send_hi(self, agent_client: AgentClient, response_client: ResponseClient): + + input_activity = self.populate( + type="message", + text="hi", + ) + + await agent_client.send_activity(input_activity) + + await asyncio.sleep(1) # Wait for processing + + responses = await response_client.pop() + assert len(responses) == 2 + + message_activity = ModelQuery(type="message").first(responses) + + assert message_activity is not None + assert message_activity.text == "you said: hi" + + typing_activity = ModelQuery(type="typing").first(responses) + assert typing_activity is not None \ No newline at end of file diff --git a/dev/tests/__init__.py b/dev/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/tests/agents/__init__.py b/dev/tests/agents/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/tests/agents/basic_agent/__init__.py b/dev/tests/agents/basic_agent/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/agents/basic_agent/python/README.md b/dev/tests/agents/basic_agent/python/README.md similarity index 100% rename from dev/integration/agents/basic_agent/python/README.md rename to dev/tests/agents/basic_agent/python/README.md diff --git a/dev/tests/agents/basic_agent/python/__init__.py b/dev/tests/agents/basic_agent/python/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/agents/basic_agent/python/env.TEMPLATE b/dev/tests/agents/basic_agent/python/env.TEMPLATE similarity index 100% rename from dev/integration/agents/basic_agent/python/env.TEMPLATE rename to dev/tests/agents/basic_agent/python/env.TEMPLATE diff --git a/dev/integration/agents/basic_agent/python/pre_requirements.txt b/dev/tests/agents/basic_agent/python/pre_requirements.txt similarity index 100% rename from dev/integration/agents/basic_agent/python/pre_requirements.txt rename to dev/tests/agents/basic_agent/python/pre_requirements.txt diff --git a/dev/integration/agents/basic_agent/python/requirements.txt b/dev/tests/agents/basic_agent/python/requirements.txt similarity index 100% rename from dev/integration/agents/basic_agent/python/requirements.txt rename to dev/tests/agents/basic_agent/python/requirements.txt diff --git a/dev/tests/agents/basic_agent/python/src/__init__.py b/dev/tests/agents/basic_agent/python/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/agents/basic_agent/python/src/agent.py b/dev/tests/agents/basic_agent/python/src/agent.py similarity index 100% rename from dev/integration/agents/basic_agent/python/src/agent.py rename to dev/tests/agents/basic_agent/python/src/agent.py diff --git a/dev/integration/agents/basic_agent/python/src/app.py b/dev/tests/agents/basic_agent/python/src/app.py similarity index 100% rename from dev/integration/agents/basic_agent/python/src/app.py rename to dev/tests/agents/basic_agent/python/src/app.py diff --git a/dev/integration/agents/basic_agent/python/src/config.py b/dev/tests/agents/basic_agent/python/src/config.py similarity index 100% rename from dev/integration/agents/basic_agent/python/src/config.py rename to dev/tests/agents/basic_agent/python/src/config.py diff --git a/dev/tests/agents/basic_agent/python/src/weather/__init__.py b/dev/tests/agents/basic_agent/python/src/weather/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/tests/agents/basic_agent/python/src/weather/agents/__init__.py b/dev/tests/agents/basic_agent/python/src/weather/agents/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/agents/basic_agent/python/src/weather/agents/weather_forecast_agent.py b/dev/tests/agents/basic_agent/python/src/weather/agents/weather_forecast_agent.py similarity index 100% rename from dev/integration/agents/basic_agent/python/src/weather/agents/weather_forecast_agent.py rename to dev/tests/agents/basic_agent/python/src/weather/agents/weather_forecast_agent.py diff --git a/dev/integration/agents/basic_agent/python/src/weather/plugins/__init__.py b/dev/tests/agents/basic_agent/python/src/weather/plugins/__init__.py similarity index 100% rename from dev/integration/agents/basic_agent/python/src/weather/plugins/__init__.py rename to dev/tests/agents/basic_agent/python/src/weather/plugins/__init__.py diff --git a/dev/integration/agents/basic_agent/python/src/weather/plugins/adaptive_card_plugin.py b/dev/tests/agents/basic_agent/python/src/weather/plugins/adaptive_card_plugin.py similarity index 100% rename from dev/integration/agents/basic_agent/python/src/weather/plugins/adaptive_card_plugin.py rename to dev/tests/agents/basic_agent/python/src/weather/plugins/adaptive_card_plugin.py diff --git a/dev/integration/agents/basic_agent/python/src/weather/plugins/date_time_plugin.py b/dev/tests/agents/basic_agent/python/src/weather/plugins/date_time_plugin.py similarity index 100% rename from dev/integration/agents/basic_agent/python/src/weather/plugins/date_time_plugin.py rename to dev/tests/agents/basic_agent/python/src/weather/plugins/date_time_plugin.py diff --git a/dev/integration/agents/basic_agent/python/src/weather/plugins/weather_forecast.py b/dev/tests/agents/basic_agent/python/src/weather/plugins/weather_forecast.py similarity index 100% rename from dev/integration/agents/basic_agent/python/src/weather/plugins/weather_forecast.py rename to dev/tests/agents/basic_agent/python/src/weather/plugins/weather_forecast.py diff --git a/dev/integration/agents/basic_agent/python/src/weather/plugins/weather_forecast_plugin.py b/dev/tests/agents/basic_agent/python/src/weather/plugins/weather_forecast_plugin.py similarity index 100% rename from dev/integration/agents/basic_agent/python/src/weather/plugins/weather_forecast_plugin.py rename to dev/tests/agents/basic_agent/python/src/weather/plugins/weather_forecast_plugin.py diff --git a/dev/tests/env.TEMPLATE b/dev/tests/env.TEMPLATE new file mode 100644 index 00000000..e69de29b diff --git a/dev/tests/integration/__init__.py b/dev/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/tests/integration/test_copilot_client.py b/dev/tests/integration/test_copilot_client.py new file mode 100644 index 00000000..c59207b1 --- /dev/null +++ b/dev/tests/integration/test_copilot_client.py @@ -0,0 +1,110 @@ +import pytest + +from typing import Awaitable, Callable, Iterable + +from aiohttp.web import Request, Response, Application, StreamResponse + +from microsoft_agents.activity import Activity + +from microsoft_agents.copilotstudio.client import ( + CopilotClient, + ConnectionSettings, + PowerPlatformEnvironment, +) + +from microsoft_agents.testing.integration import AiohttpRunner + +def mock_mcs_handler( + activities: Iterable[Activity], +) -> Callable[[Request], Awaitable[StreamResponse]]: + """Creates a mock handler for MCS endpoint returning SSE-formatted activity.""" + + async def handler(request: Request) -> StreamResponse: + response = StreamResponse(status=200) + response.headers["Content-Type"] = "text/event-stream" + response.headers["x-ms-conversationid"] = "test-conv-id" + # response.headers['Content-Length'] = str(len(activity_data)) + await response.prepare(request) + + # Proper SSE format + for activity in activities: + activity_data = activity.model_dump_json(exclude_unset=True) + await response.write(b"event: activity\n") + await response.write(f"data: {activity_data}\n\n".encode("utf-8")) + + await response.write_eof() + return response + + return handler + + +def mock_mcs_endpoint( + mocker, activities: Iterable[Activity], path: str, port: int +) -> AiohttpRunner: + """Mock MCS responses for testing.""" + + PowerPlatformEnvironment.get_copilot_studio_connection_url = mocker.MagicMock( + return_value=f"http://localhost:{port}{path}" + ) + + app = Application() + app.router.add_post(path, mock_mcs_handler(activities)) + + return AiohttpRunner(app, port=port) + + +@pytest.mark.asyncio +async def test_start_conversation_and_ask_question_large_message(mocker): + + activity = Activity( + type="message", text="*" * 1_000_000, conversation={"id": "conv-id"} + ) + + runner = mock_mcs_endpoint(mocker, [activity], "/mcs-endpoint", port=8081) + + async with runner: + settings = ConnectionSettings("environment-id", "agent-id") + client = CopilotClient(settings=settings, token="test-token") + + with pytest.raises(Exception, match="Chunk too big"): + async for conv_activity in client.start_conversation(): + assert conv_activity.type == "message" + +@pytest.mark.asyncio +async def test_start_conversation_and_ask_question_no_error(mocker): + + activity = Activity( + type="message", text="*" * 1_000_000, conversation={"id": "conv-id"} + ) + + runner = mock_mcs_endpoint(mocker, [activity], "/mcs-endpoint", port=8081) + + async with runner: + settings = ConnectionSettings("environment-id", "agent-id", + client_session_settings={"read_bufsize": 2**25}) + client = CopilotClient(settings=settings, token="test-token") + + async for conv_activity in client.start_conversation(): + assert conv_activity.type == "message" + + +def activity_generator(activity: Activity, n: int) -> Iterable[Activity]: + for i in range(n): + yield activity + + +@pytest.mark.asyncio +async def test_start_conversation_many(mocker): + + activity = Activity(type="message", conversation={"id": "conv-id"}) + activities = activity_generator(activity, 100_000) + + runner = mock_mcs_endpoint(mocker, activities, "/mcs-endpoint", port=8081) + + async with runner: + settings = ConnectionSettings("environment-id", "agent-id") + client = CopilotClient(settings=settings, token="test-token") + + for i in range(100): + async for conv_activity in client.start_conversation(): + assert conv_activity.type == "message" \ No newline at end of file diff --git a/dev/integration/tests/test_expect_replies.py b/dev/tests/integration/test_expect_replies.py similarity index 59% rename from dev/integration/tests/test_expect_replies.py rename to dev/tests/integration/test_expect_replies.py index 86d23cd7..152b2594 100644 --- a/dev/integration/tests/test_expect_replies.py +++ b/dev/tests/integration/test_expect_replies.py @@ -1,28 +1,17 @@ import pytest -import logging from microsoft_agents.activity import Activity from microsoft_agents.testing import ( - ddt, Integration, - AiohttpEnvironment, + AiohttpEnvironment ) -from ..samples import BasicSample +from ..samples import QuickstartSample -class BasicSampleWithLogging(BasicSample): - - async def init_app(self): - - logging.getLogger("microsoft_agents").setLevel(logging.DEBUG) - - await super().init_app() - - -class TestBasicDirectline(Integration): - _sample_cls = BasicSampleWithLogging +class TestExpectReplies(Integration): + _sample_cls = QuickstartSample _environment_cls = AiohttpEnvironment @pytest.mark.asyncio @@ -36,12 +25,12 @@ async def test_expect_replies_without_service_url( conversation={"id": "conv-id"}, channel_id="test", from_property={"id": "from-id"}, - to={"id": "to-id"}, + recipient={"id": "to-id"}, delivery_mode="expectReplies", locale="en-US", ) res = await agent_client.send_expect_replies(activity) - breakpoint() - res = Activity.model_validate(res) + assert len(res) > 0 + assert isinstance(res[0], Activity) diff --git a/dev/integration/pytest.ini b/dev/tests/pytest.ini similarity index 97% rename from dev/integration/pytest.ini rename to dev/tests/pytest.ini index 9908f4bf..2c3d00cb 100644 --- a/dev/integration/pytest.ini +++ b/dev/tests/pytest.ini @@ -7,7 +7,7 @@ filterwarnings = ignore::aiohttp.web.NotAppKeyWarning # Test discovery configuration -testpaths = tests +testpaths = ./ python_files = test_*.py *_test.py python_classes = Test* python_functions = test_* diff --git a/dev/integration/samples/__init__.py b/dev/tests/samples/__init__.py similarity index 60% rename from dev/integration/samples/__init__.py rename to dev/tests/samples/__init__.py index 4e712561..ab105497 100644 --- a/dev/integration/samples/__init__.py +++ b/dev/tests/samples/__init__.py @@ -1,7 +1,5 @@ -from .basic_sample import BasicSample from .quickstart_sample import QuickstartSample __all__ = [ - "BasicSample", "QuickstartSample", ] diff --git a/dev/tests/samples/quickstart_sample.py b/dev/tests/samples/quickstart_sample.py new file mode 100644 index 00000000..6bf65206 --- /dev/null +++ b/dev/tests/samples/quickstart_sample.py @@ -0,0 +1,55 @@ +import re +import os +import sys +import traceback + +from dotenv import load_dotenv + +from microsoft_agents.activity import ConversationUpdateTypes +from microsoft_agents.hosting.core import ( + AgentApplication, + TurnContext, + TurnState, +) +from microsoft_agents.testing.integration import Sample + + +class QuickstartSample(Sample): + """A quickstart sample implementation.""" + + @classmethod + async def get_config(cls) -> dict: + """Retrieve the configuration for the sample.""" + load_dotenv("./src/tests/.env") + return dict(os.environ) + + async def init_app(self): + """Initialize the application for the quickstart sample.""" + + app: AgentApplication[TurnState] = self.env.agent_application + + @app.conversation_update(ConversationUpdateTypes.MEMBERS_ADDED) + async def on_members_added(context: TurnContext, state: TurnState) -> None: + await context.send_activity( + "Welcome to the empty agent! " + "This agent is designed to be a starting point for your own agent development." + ) + + @app.message(re.compile(r"^hello$")) + async def on_hello(context: TurnContext, state: TurnState) -> None: + await context.send_activity("Hello!") + + @app.activity("message") + async def on_message(context: TurnContext, state: TurnState) -> None: + await context.send_activity(f"you said: {context.activity.text}") + + @app.error + async def on_error(context: TurnContext, error: Exception): + # This check writes out errors to console log .vs. app insights. + # NOTE: In production environment, you should consider logging this to Azure + # application insights. + print(f"\n [on_turn_error] unhandled error: {error}", file=sys.stderr) + traceback.print_exc() + + # Send a message to the user + await context.send_activity("The bot encountered an error or bug.")