diff --git a/eval_protocol/__init__.py b/eval_protocol/__init__.py index ebd4ab41..f912b7ff 100644 --- a/eval_protocol/__init__.py +++ b/eval_protocol/__init__.py @@ -12,7 +12,6 @@ from .auth import get_fireworks_account_id, get_fireworks_api_key from .common_utils import load_jsonl -from .config import RewardKitConfig, get_config, load_config from .mcp_env import ( AnthropicPolicy, FireworksPolicy, @@ -90,10 +89,6 @@ # Authentication "get_fireworks_api_key", "get_fireworks_account_id", - # Configuration - "load_config", - "get_config", - "RewardKitConfig", # Utilities "load_jsonl", # MCP Environment API diff --git a/eval_protocol/cli.py b/eval_protocol/cli.py index c0858f92..50665691 100644 --- a/eval_protocol/cli.py +++ b/eval_protocol/cli.py @@ -169,17 +169,17 @@ def parse_args(args=None): gcp_group.add_argument( "--gcp-project", required=False, - help="Google Cloud Project ID. Must be provided via CLI or rewardkit.yaml.", + help="Google Cloud Project ID. Required for GCP deployments.", ) gcp_group.add_argument( "--gcp-region", required=False, - help="Google Cloud Region for deployment (e.g., 'us-central1'). Must be provided via CLI or rewardkit.yaml.", + help="Google Cloud Region for deployment (e.g., 'us-central1'). Required for GCP deployments.", ) gcp_group.add_argument( "--gcp-ar-repo", required=False, - help="Google Artifact Registry repository name. Optional, defaults to value in rewardkit.yaml or 'eval-protocol-evaluators' if not specified.", + help="Google Artifact Registry repository name. Optional, defaults to 'eval-protocol-evaluators' if not specified.", ) gcp_group.add_argument( "--service-account", @@ -202,7 +202,7 @@ def parse_args(args=None): help="Authentication mode for the deployed GCP Cloud Run service. " "'open': Publicly accessible. " "'api-key': Service is publicly accessible but requires an API key in requests (handled by the application). " - "If not specified, defaults to value in rewardkit.yaml or 'api-key'. Optional.", + "If not specified, defaults to 'api-key'. Optional.", ) # Deploy MCP command @@ -218,11 +218,11 @@ def parse_args(args=None): ) deploy_mcp_parser.add_argument( "--gcp-project", - help="Google Cloud Project ID. Can also be set in rewardkit.yaml", + help="Google Cloud Project ID. Required for GCP deployments.", ) deploy_mcp_parser.add_argument( "--gcp-region", - help="Google Cloud Region (e.g., 'us-central1'). Can also be set in rewardkit.yaml", + help="Google Cloud Region (e.g., 'us-central1'). Required for GCP deployments.", ) deploy_mcp_parser.add_argument( "--gcp-ar-repo", diff --git a/eval_protocol/cli_commands/deploy.py b/eval_protocol/cli_commands/deploy.py index 1ae0313b..f7c08b63 100644 --- a/eval_protocol/cli_commands/deploy.py +++ b/eval_protocol/cli_commands/deploy.py @@ -12,8 +12,6 @@ from pathlib import Path # For path operations from typing import Any, Dict -import yaml # For saving config if save_config helper doesn't exist - # TODO: Consider moving subprocess_manager functions to a more central location if used by core CLI try: # Import functions with explicit names to match expected signatures @@ -79,12 +77,6 @@ def start_ngrok_and_get_url(local_port, log_path): from eval_protocol.auth import get_fireworks_account_id -from eval_protocol.config import ( - GCPCloudRunConfig, - RewardKitConfig, - _config_file_path as global_loaded_config_path, - get_config, -) from eval_protocol.evaluation import create_evaluation from eval_protocol.gcp_tools import ( build_and_push_docker_image, @@ -176,7 +168,7 @@ def _establish_local_server_and_tunnel(args): ) # URL, provider, server_pid, tunnel_pid -def _deploy_to_gcp_cloud_run(args, current_config, gcp_config_from_yaml): +def _deploy_to_gcp_cloud_run(args): """Handles the logic for --target gcp-cloud-run up to service deployment.""" print(f"Starting GCP Cloud Run deployment for evaluator '{args.id}'...") @@ -201,24 +193,18 @@ def _deploy_to_gcp_cloud_run(args, current_config, gcp_config_from_yaml): # Resolve GCP project_id gcp_project_id = args.gcp_project - if not gcp_project_id and gcp_config_from_yaml: - gcp_project_id = gcp_config_from_yaml.project_id if not gcp_project_id: - print("Error: GCP Project ID must be provided via --gcp-project argument or in rewardkit.yaml.") + print("Error: GCP Project ID must be provided via --gcp-project argument.") return None # Resolve GCP region gcp_region = args.gcp_region - if not gcp_region and gcp_config_from_yaml: - gcp_region = gcp_config_from_yaml.region if not gcp_region: - print("Error: GCP Region must be provided via --gcp-region argument or in rewardkit.yaml.") + print("Error: GCP Region must be provided via --gcp-region argument.") return None # Resolve GCP AR repo name gcp_ar_repo_name = args.gcp_ar_repo - if not gcp_ar_repo_name and gcp_config_from_yaml: - gcp_ar_repo_name = gcp_config_from_yaml.artifact_registry_repository if not gcp_ar_repo_name: gcp_ar_repo_name = "eval-protocol-evaluators" @@ -264,32 +250,15 @@ def _deploy_to_gcp_cloud_run(args, current_config, gcp_config_from_yaml): parsed_gcp_secrets: Dict[str, Any] = {} allow_unauthenticated_gcp = True - resolved_auth_mode = "api-key" - if gcp_config_from_yaml and gcp_config_from_yaml.default_auth_mode: - resolved_auth_mode = gcp_config_from_yaml.default_auth_mode - if args.gcp_auth_mode is not None: - resolved_auth_mode = args.gcp_auth_mode + resolved_auth_mode = args.gcp_auth_mode if args.gcp_auth_mode is not None else "api-key" print(f"Using GCP Auth Mode for service: {resolved_auth_mode}") if resolved_auth_mode == "api-key": print("Configuring GCP Cloud Run service for API key authentication (application layer).") evaluator_id = args.id - api_key_for_service = None # This is the key the service itself will use - config_path = global_loaded_config_path - - if current_config.evaluator_endpoint_keys and evaluator_id in current_config.evaluator_endpoint_keys: - api_key_for_service = current_config.evaluator_endpoint_keys[evaluator_id] - print(f"Using existing API key for '{evaluator_id}' from configuration for the service.") - else: - api_key_for_service = secrets.token_hex(32) - print(f"Generated new API key for '{evaluator_id}' for the service.") - if not current_config.evaluator_endpoint_keys: - current_config.evaluator_endpoint_keys = {} - current_config.evaluator_endpoint_keys[evaluator_id] = api_key_for_service - if config_path: - _save_config(current_config, config_path) - else: - print(f"Warning: No rewardkit.yaml found to save API key for '{evaluator_id}'.") + # Generate API key for the service + api_key_for_service = secrets.token_hex(32) + print(f"Generated new API key for '{evaluator_id}' for the service.") gcp_sanitized_eval_id = "".join(filter(lambda char: char.isalnum() or char in ["-", "_"], args.id)) if not gcp_sanitized_eval_id: @@ -349,17 +318,6 @@ def _deploy_to_gcp_cloud_run(args, current_config, gcp_config_from_yaml): return cloud_run_service_url -# Helper to save config (can be moved to config.py later) -def _save_config(config_data: RewardKitConfig, path: str): - # Basic save, ideally config.py would provide a robust method - try: - with open(path, "w") as f: - yaml.dump(config_data.model_dump(exclude_none=True), f, sort_keys=False) - print(f"Config updated and saved to {path}") - except Exception as e: - print(f"Warning: Failed to save updated config to {path}: {e}") - - def deploy_command(args): """Create and deploy an evaluator or register a remote one.""" @@ -390,10 +348,7 @@ def deploy_command(args): local_tunnel_pid_to_clean = None # Initialize here if args.target == "gcp-cloud-run": - current_config = get_config() # Needed by the helper - gcp_config_from_yaml = current_config.gcp_cloud_run if current_config.gcp_cloud_run else None - - cloud_run_service_url = _deploy_to_gcp_cloud_run(args, current_config, gcp_config_from_yaml) + cloud_run_service_url = _deploy_to_gcp_cloud_run(args) if not cloud_run_service_url: return 1 # Error already printed by helper service_url_to_register = cloud_run_service_url diff --git a/eval_protocol/cli_commands/deploy_mcp.py b/eval_protocol/cli_commands/deploy_mcp.py index 34cb6a6f..d8cfd763 100644 --- a/eval_protocol/cli_commands/deploy_mcp.py +++ b/eval_protocol/cli_commands/deploy_mcp.py @@ -9,12 +9,6 @@ from pathlib import Path from typing import Dict, Optional -from eval_protocol.config import ( - GCPCloudRunConfig, - RewardKitConfig, - _config_file_path as global_loaded_config_path, - get_config, -) from eval_protocol.gcp_tools import ( build_and_push_docker_image, deploy_to_cloud_run, @@ -129,7 +123,7 @@ def _generate_mcp_dockerfile_content( return dockerfile_content -def _deploy_mcp_to_gcp_cloud_run(args, current_config, gcp_config_from_yaml): +def _deploy_mcp_to_gcp_cloud_run(args): """Deploy MCP server to GCP Cloud Run.""" print(f"Starting MCP server deployment to GCP Cloud Run for '{args.id}'...") @@ -140,22 +134,16 @@ def _deploy_mcp_to_gcp_cloud_run(args, current_config, gcp_config_from_yaml): # Resolve GCP configuration gcp_project_id = args.gcp_project - if not gcp_project_id and gcp_config_from_yaml: - gcp_project_id = gcp_config_from_yaml.project_id if not gcp_project_id: - print("Error: GCP Project ID must be provided via --gcp-project or rewardkit.yaml.") + print("Error: GCP Project ID must be provided via --gcp-project argument.") return None gcp_region = args.gcp_region - if not gcp_region and gcp_config_from_yaml: - gcp_region = gcp_config_from_yaml.region if not gcp_region: - print("Error: GCP Region must be provided via --gcp-region or rewardkit.yaml.") + print("Error: GCP Region must be provided via --gcp-region argument.") return None gcp_ar_repo_name = args.gcp_ar_repo - if not gcp_ar_repo_name and gcp_config_from_yaml: - gcp_ar_repo_name = gcp_config_from_yaml.artifact_registry_repository if not gcp_ar_repo_name: gcp_ar_repo_name = "eval-protocol-mcp-servers" @@ -266,14 +254,8 @@ def deploy_mcp_command(args): return False try: - # Load configuration - current_config = get_config() - gcp_config_from_yaml: Optional[GCPCloudRunConfig] = None - if current_config and current_config.gcp_cloud_run: - gcp_config_from_yaml = current_config.gcp_cloud_run - # Deploy to GCP Cloud Run - service_url = _deploy_mcp_to_gcp_cloud_run(args, current_config, gcp_config_from_yaml) + service_url = _deploy_mcp_to_gcp_cloud_run(args) if service_url: print(f"✅ MCP server '{args.id}' successfully deployed!") diff --git a/eval_protocol/config.py b/eval_protocol/config.py deleted file mode 100644 index 7d75ab01..00000000 --- a/eval_protocol/config.py +++ /dev/null @@ -1,180 +0,0 @@ -import os -from typing import Dict, Literal, Optional - -import yaml -from pydantic import BaseModel, ValidationError - -CONFIG_FILE_NAME = "rewardkit.yaml" - -# --- Pydantic Models for Configuration Structure --- - - -class GCPCloudRunConfig(BaseModel): - project_id: Optional[str] = None # Default will be applied in deploy_command if not set - region: Optional[str] = None # Default will be applied in deploy_command if not set - artifact_registry_repository: Optional[str] = None # Default will be applied in deploy_command - service_name_template: Optional[str] = "rewardeval-{evaluator_id}" - default_auth_mode: Optional[Literal["api-key", "iam", "mtls-client-auth"]] = ( - "api-key" # Default auth mode if using GCP target and not specified - ) - secrets: Optional[Dict[str, str]] = {} # Maps ENV_VAR_NAME to GCP Secret Manager ID - - -class AWSLambdaConfig(BaseModel): - region: Optional[str] = None - function_name_template: Optional[str] = "rewardeval-{evaluator_id}" - default_auth_mode: Optional[Literal["api-key", "iam", "mtls-client-auth"]] = "api-key" - secrets: Optional[Dict[str, str]] = {} # Maps ENV_VAR_NAME to AWS Secret ARN - - -class RewardKitConfig(BaseModel): - default_deployment_target: Optional[Literal["gcp-cloud-run", "aws-lambda", "fireworks", "local"]] = "fireworks" - gcp_cloud_run: Optional[GCPCloudRunConfig] = GCPCloudRunConfig() - aws_lambda: Optional[AWSLambdaConfig] = AWSLambdaConfig() - evaluator_endpoint_keys: Optional[ - Dict[str, str] - ] = {} # Stores generated API keys for self-hosted evaluator endpoints - - -# --- Global variable to hold the loaded configuration --- -_loaded_config: Optional[RewardKitConfig] = None -_config_file_path: Optional[str] = None - - -def find_config_file(start_path: Optional[str] = None) -> Optional[str]: - """ - Finds the rewardkit.yaml file by searching upwards from start_path (or CWD). - """ - if start_path is None: - start_path = os.getcwd() - - current_path = os.path.abspath(start_path) - while True: - potential_path = os.path.join(current_path, CONFIG_FILE_NAME) - if os.path.isfile(potential_path): - return potential_path - - parent_path = os.path.dirname(current_path) - if parent_path == current_path: - return None - current_path = parent_path - - -def load_config(config_path: Optional[str] = None) -> RewardKitConfig: - """ - Loads the rewardkit.yaml configuration. - If already loaded, returns the cached version unless a new path is provided. - If no path is provided, it tries to find rewardkit.yaml in CWD or parent directories. - """ - global _loaded_config, _config_file_path - - if config_path: - pass - elif _loaded_config and not config_path: - return _loaded_config - else: - config_path = find_config_file() - - if not config_path: - _loaded_config = RewardKitConfig() - _config_file_path = None - return _loaded_config - - if _loaded_config and config_path == _config_file_path: - return _loaded_config - - try: - with open(config_path, "r") as f: - raw_config = yaml.safe_load(f) - - if raw_config is None: - _loaded_config = RewardKitConfig() - else: - _loaded_config = RewardKitConfig(**raw_config) - - _config_file_path = config_path - return _loaded_config - except FileNotFoundError: - _loaded_config = RewardKitConfig() - _config_file_path = None - return _loaded_config - except yaml.YAMLError as e: - print(f"Error parsing YAML from {config_path}: {e}") - # Decide: raise error or return default? For now, return default and warn. - _loaded_config = RewardKitConfig() - _config_file_path = config_path # So it doesn't try to reload this broken file again - return _loaded_config - except ValidationError as e: - print(f"Error validating configuration from {config_path}: {e}") - _loaded_config = RewardKitConfig() - _config_file_path = config_path - return _loaded_config - except Exception as e: - print(f"An unexpected error occurred while loading configuration from {config_path}: {e}") - _loaded_config = RewardKitConfig() - _config_file_path = config_path - return _loaded_config - - -def get_config() -> RewardKitConfig: - """ - Returns the loaded configuration. Loads it if not already loaded. - """ - if _loaded_config is None: - return load_config() - return _loaded_config - - -# Example usage (can be removed or kept for testing module directly) -if __name__ == "__main__": - # Create a dummy rewardkit.yaml for testing - dummy_config_content = """ -default_deployment_target: gcp-cloud-run - -gcp_cloud_run: - project_id: "test-gcp-project" - region: "europe-west1" - default_auth_mode: "iam" - secrets: - DB_PASSWORD: "projects/test-gcp-project/secrets/db-password/versions/latest" - -aws_lambda: - region: "eu-north-1" - -evaluator_endpoint_keys: - my_test_eval: "dummy_key_for_test_eval" -""" - dummy_file_path = os.path.join(os.getcwd(), CONFIG_FILE_NAME) - with open(dummy_file_path, "w") as f: - f.write(dummy_config_content) - - print(f"Created dummy {CONFIG_FILE_NAME} for testing.") - - config = get_config() - print("\nLoaded configuration:") - print(f" Default Target: {config.default_deployment_target}") - if config.gcp_cloud_run: - print(f" GCP Project ID: {config.gcp_cloud_run.project_id}") - print(f" GCP Region: {config.gcp_cloud_run.region}") - print(f" GCP Auth Mode: {config.gcp_cloud_run.default_auth_mode}") - print(f" GCP Secrets: {config.gcp_cloud_run.secrets}") - if config.aws_lambda: - print(f" AWS Region: {config.aws_lambda.region}") - print(f" Endpoint Keys: {config.evaluator_endpoint_keys}") - - # Test finding config - print("\nTesting find_config_file():") - found_path = find_config_file() - print(f" Found at: {found_path}") - - # Clean up dummy file - os.remove(dummy_file_path) - print(f"\nRemoved dummy {CONFIG_FILE_NAME}.") - - # Test loading with no file - _loaded_config = None # Reset cache - _config_file_path = None - print("\nTesting get_config() with no file present:") - config_no_file = get_config() - print(f" Default Target (no file): {config_no_file.default_deployment_target}") - print(f" GCP Config (no file): {config_no_file.gcp_cloud_run}") diff --git a/tests/cli_commands/test_deploy_cmd.py b/tests/cli_commands/test_deploy_cmd.py index fbd38ae8..eed8edc0 100644 --- a/tests/cli_commands/test_deploy_cmd.py +++ b/tests/cli_commands/test_deploy_cmd.py @@ -284,12 +284,8 @@ def test_deploy_gcp_mode_success( in captured.out ) - @patch("eval_protocol.cli_commands.deploy.get_config") - def test_deploy_gcp_mode_missing_args(self, mock_get_config, mock_check_environment, capsys): - # Mock empty config to test missing project/region scenarios - from eval_protocol.config import RewardKitConfig - - mock_get_config.return_value = RewardKitConfig() + def test_deploy_gcp_mode_missing_args(self, mock_check_environment, capsys): + # Test missing required args for GCP deployment args = MockArgs(target="gcp-cloud-run", id="gcp-eval-incomplete") # function_ref is missing, gcp_project, gcp_region also diff --git a/tests/test_config.py b/tests/test_config.py deleted file mode 100644 index 0670fb85..00000000 --- a/tests/test_config.py +++ /dev/null @@ -1,219 +0,0 @@ -import os -from unittest.mock import mock_open, patch - -import pytest -import yaml -from pydantic import ValidationError - -from eval_protocol import config as rk_config # Alias to avoid conflict -from eval_protocol.config import ( - CONFIG_FILE_NAME, - AWSLambdaConfig, - GCPCloudRunConfig, - RewardKitConfig, -) - - -# Helper to reset config state for isolation if module-level cache is an issue -@pytest.fixture(autouse=True) -def reset_config_module_state(): - rk_config._loaded_config = None - rk_config._config_file_path = None - yield - rk_config._loaded_config = None - rk_config._config_file_path = None - - -class TestFindConfigFile: - def test_find_in_current_dir(self, tmp_path): - """Test finding config in the current directory.""" - (tmp_path / CONFIG_FILE_NAME).write_text("test: content") - found_path = rk_config.find_config_file(start_path=str(tmp_path)) - assert found_path == str(tmp_path / CONFIG_FILE_NAME) - - def test_find_in_parent_dir(self, tmp_path): - """Test finding config in a parent directory.""" - parent_dir = tmp_path - child_dir = tmp_path / "subdir" - child_dir.mkdir() - (parent_dir / CONFIG_FILE_NAME).write_text("test: content") - - found_path = rk_config.find_config_file(start_path=str(child_dir)) - assert found_path == str(parent_dir / CONFIG_FILE_NAME) - - def test_not_found(self, tmp_path): - """Test when no config file is found up to root.""" - # To ensure find_config_file doesn't find any actual config file during this test, - # we mock os.path.isfile to always return False. - # The start_path (tmp_path) ensures it starts in an isolated directory. - with patch("os.path.isfile", return_value=False): - found_path = rk_config.find_config_file(start_path=str(tmp_path)) - assert found_path is None - - -class TestLoadConfig: - def test_load_no_config_file_found(self): - """Test loading config when no rewardkit.yaml exists.""" - with patch("eval_protocol.config.find_config_file", return_value=None): - cfg = rk_config.get_config() - assert isinstance(cfg, RewardKitConfig) - # Check for default values - assert cfg.default_deployment_target == "fireworks" - assert cfg.gcp_cloud_run is not None - assert cfg.gcp_cloud_run.project_id is None - assert cfg.aws_lambda is not None - assert cfg.evaluator_endpoint_keys == {} - - def test_load_empty_config_file(self, tmp_path): - """Test loading an empty rewardkit.yaml file.""" - config_file = tmp_path / CONFIG_FILE_NAME - config_file.write_text("") # Empty file - - cfg = rk_config.load_config(str(config_file)) - assert isinstance(cfg, RewardKitConfig) - assert cfg.default_deployment_target == "fireworks" # Should use Pydantic defaults - - def test_load_valid_full_config_file(self, tmp_path): - """Test loading a valid config file with all sections.""" - content = """ -default_deployment_target: gcp-cloud-run -gcp_cloud_run: - project_id: "my-gcp-proj" - region: "us-west1" - service_name_template: "eval-{evaluator_id}" - default_auth_mode: "iam" - secrets: - API_KEY: "gcp_secret_id_for_api_key" -aws_lambda: - region: "eu-central-1" - function_name_template: "lambda-eval-{evaluator_id}" - default_auth_mode: "mtls-client-auth" - secrets: - OTHER_KEY: "aws_secret_arn_for_other_key" -evaluator_endpoint_keys: - my_eval_1: "key123" -""" - config_file = tmp_path / CONFIG_FILE_NAME - config_file.write_text(content) - - cfg = rk_config.load_config(str(config_file)) - assert cfg.default_deployment_target == "gcp-cloud-run" - assert cfg.gcp_cloud_run.project_id == "my-gcp-proj" - assert cfg.gcp_cloud_run.region == "us-west1" - assert cfg.gcp_cloud_run.service_name_template == "eval-{evaluator_id}" - assert cfg.gcp_cloud_run.default_auth_mode == "iam" - assert cfg.gcp_cloud_run.secrets == {"API_KEY": "gcp_secret_id_for_api_key"} - assert cfg.aws_lambda.region == "eu-central-1" - assert cfg.aws_lambda.function_name_template == "lambda-eval-{evaluator_id}" - assert cfg.aws_lambda.default_auth_mode == "mtls-client-auth" - assert cfg.aws_lambda.secrets == {"OTHER_KEY": "aws_secret_arn_for_other_key"} - assert cfg.evaluator_endpoint_keys == {"my_eval_1": "key123"} - - def test_load_valid_partial_config_file(self, tmp_path): - """Test loading a valid config with some optional sections/fields missing.""" - content = """ -default_deployment_target: local -gcp_cloud_run: - project_id: "my-gcp-proj" -# aws_lambda is missing, should use default -# evaluator_endpoint_keys is missing, should use default -""" - config_file = tmp_path / CONFIG_FILE_NAME - config_file.write_text(content) - - cfg = rk_config.load_config(str(config_file)) - assert cfg.default_deployment_target == "local" - assert cfg.gcp_cloud_run.project_id == "my-gcp-proj" - assert cfg.gcp_cloud_run.region is None # Default for optional field - assert cfg.gcp_cloud_run.default_auth_mode == "api-key" # Default from model - assert cfg.aws_lambda is not None # Default AWSLambdaConfig object - assert cfg.aws_lambda.region is None - assert cfg.evaluator_endpoint_keys == {} # Default empty dict - - def test_load_malformed_yaml(self, tmp_path, capsys): - """Test loading a file with invalid YAML syntax.""" - config_file = tmp_path / CONFIG_FILE_NAME - config_file.write_text("default_deployment_target: gcp-cloud-run\n bad_indent: true") - - cfg = rk_config.load_config(str(config_file)) - captured = capsys.readouterr() - assert "Error parsing YAML" in captured.out - assert isinstance(cfg, RewardKitConfig) # Should return default - assert cfg.default_deployment_target == "fireworks" - - def test_load_schema_violation(self, tmp_path, capsys): - """Test loading valid YAML but with schema violations (wrong types).""" - content = """ -default_deployment_target: "invalid_target_option" # Not in Literal -gcp_cloud_run: - project_id: 12345 # Should be string -""" - config_file = tmp_path / CONFIG_FILE_NAME - config_file.write_text(content) - - cfg = rk_config.load_config(str(config_file)) - captured = capsys.readouterr() - assert "Error validating configuration" in captured.out - assert isinstance(cfg, RewardKitConfig) # Should return default - assert cfg.default_deployment_target == "fireworks" - - def test_config_caching(self, tmp_path): - """Test that get_config() returns a cached instance.""" - config_file = tmp_path / CONFIG_FILE_NAME - config_file.write_text("default_deployment_target: local") - - # Patch find_config_file to control its result for this test - with patch("eval_protocol.config.find_config_file", return_value=str(config_file)) as mock_find: - cfg1 = rk_config.get_config() - assert cfg1.default_deployment_target == "local" - mock_find.assert_called_once() # find_config_file should be called - - # Call get_config() again, should use cache - cfg2 = rk_config.get_config() - assert cfg2 is cfg1 # Should be the same instance - mock_find.assert_called_once() # find_config_file should NOT be called again - - # Force reload by providing path to load_config - rk_config.load_config(str(config_file)) # This will reload - # If find_config_file was patched at module level, this call to load_config - # would use the patched version. The fixture resets _loaded_config. - # The number of calls to find_config_file depends on how load_config is structured. - # The current load_config structure: if path is given, it uses it. If not, it calls find_config_file. - # So, load_config(path) doesn't call find_config_file. - - # Reset and call get_config again after a direct load_config(path) - rk_config._loaded_config = None - rk_config._config_file_path = None # Ensure it's not using the path cache - - cfg3 = rk_config.get_config() - assert cfg3.default_deployment_target == "local" - # mock_find should now have been called twice (once for cfg1, once for cfg3) - assert mock_find.call_count == 2 - - def test_load_specific_path_overrides_cache_or_find(self, tmp_path): - """Test that load_config(specific_path) loads that file.""" - default_config_file = tmp_path / "default_loc" / CONFIG_FILE_NAME - default_config_file.parent.mkdir() - default_config_file.write_text("default_deployment_target: fireworks") - - specific_config_file = tmp_path / "specific_loc" / CONFIG_FILE_NAME - specific_config_file.parent.mkdir() - specific_config_file.write_text("default_deployment_target: gcp-cloud-run") - - # Patch find_config_file to return the "default" location - with patch( - "eval_protocol.config.find_config_file", - return_value=str(default_config_file), - ): - # First, get_config might load the one found by find_config_file - cfg_found = rk_config.get_config() - assert cfg_found.default_deployment_target == "fireworks" - - # Now, load a specific one - cfg_specific = rk_config.load_config(str(specific_config_file)) - assert cfg_specific.default_deployment_target == "gcp-cloud-run" - - # get_config() should now return the specifically loaded one - cfg_after_specific_load = rk_config.get_config() - assert cfg_after_specific_load is cfg_specific - assert cfg_after_specific_load.default_deployment_target == "gcp-cloud-run" diff --git a/tests/test_deploy_integration.py b/tests/test_deploy_integration.py index 6b7db1b0..d82612a4 100644 --- a/tests/test_deploy_integration.py +++ b/tests/test_deploy_integration.py @@ -9,7 +9,6 @@ import pytest from eval_protocol.cli_commands.deploy import deploy_command -from eval_protocol.config import GCPCloudRunConfig, RewardKitConfig # Constants for a dummy reward function module # This module will be created and deleted by tests needing it. @@ -144,7 +143,6 @@ def test_deploy_gcp_with_inline_requirements( # Mock all external dependencies of _deploy_to_gcp_cloud_run and deploy_command with ( patch("eval_protocol.cli_commands.deploy.check_environment", return_value=True) as mock_check_env, - patch("eval_protocol.cli_commands.deploy.get_config") as mock_get_config, patch( "eval_protocol.cli_commands.deploy.ensure_artifact_registry_repo_exists", return_value=True, @@ -174,17 +172,6 @@ def test_deploy_gcp_with_inline_requirements( return_value={"name": evaluator_id}, ) as mock_create_eval, ): - # Configure mock_get_config to return a basic config - mock_config_instance = RewardKitConfig( - gcp_cloud_run=GCPCloudRunConfig( - project_id="test-gcp-project-yaml", # Test CLI override - region="us-west1-yaml", # Test CLI override - default_auth_mode="api-key", - ), - evaluator_endpoint_keys={}, - ) - mock_get_config.return_value = mock_config_instance - # Call the deploy command result_code = deploy_command(args)