From 9a3cb69e0a45809e6698d899884e410b31ce7f10 Mon Sep 17 00:00:00 2001 From: Judith <39854388+felix-20@users.noreply.github.com> Date: Mon, 11 Apr 2022 18:25:24 +0200 Subject: [PATCH 01/92] merge multiple experiments commit 8652a678520884a4c7325c87965cc4cfe42ffb48 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Mon Apr 11 18:17:20 2022 +0200 missed `handle_requests` on last commit commit a354f890afa46be9f678ffa88dfe9d3e7815a838 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Mon Apr 11 18:16:15 2022 +0200 implement #356 commit fd54365178e6b65c607c0fbb45a3ad8dca023e12 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Mon Apr 11 17:56:35 2022 +0200 docstrings commit 9f43c9dbfacc50764c5e7d1062b60a3d94f64057 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Mon Apr 11 15:07:47 2022 +0200 fix docker test commit b3e6e80628f9743f8de3d2cb99a6e8839ab0f74e Merge: 6f64f9e 091eee6 Author: felix-20 <39854388+felix-20@users.noreply.github.com> Date: Mon Apr 11 14:31:39 2022 +0200 Merge branch 'development' into 379-multiple-experiments commit 6f64f9ec9a5e56798594762ed03310fb4bff39da Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Mon Apr 11 14:20:47 2022 +0200 clean up docker port commit 8c35e8a6634c6516af71a6eeb959f880c01cd8ba Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Mon Apr 11 11:33:30 2022 +0200 merge development commit 091eee66ed4e2870d860059ed0a1ec9ca515ee6d Author: Nikkel Mollenhauer <57323886+NikkelM@users.noreply.github.com> Date: Mon Apr 11 10:43:00 2022 +0200 Tests for `config_validation.py` (#404) * Don't get default data (shouldn't be necessary) * Unpack default data * Refactored utils functions to return dicts instead of strings * Adapted to new mock format * Adapted to new mock format * Some first tests * More tests * Fixed testcase-names * Moved file-endings to initial function call * Fixed tests * More asserts * More tests * More tests * More tests * `validate_sub_keys`-tests commit f8bc1622e763f3502d103b77174517383c8fa8a2 Author: jannikgro <52510222+jannikgro@users.noreply.github.com> Date: Fri Apr 8 13:20:42 2022 +0200 [D] Stable baselines integration (#384) * refactored reinforcement learning agent to accept marketplace * adapted test_exampleprinter.py to marketplace initialization * add market option to accept continuos actions * fixed action space check * initial stable baselines integration * Agent init by env (#390) * introduced self.network in actorcritic_agent * added network_architecture in QLearningAgent * changed actorcritic_agent to network_architecture * set back training_scenario * am_configuration initialize rl-agent via marketplace * added final analyse to stable baselines training * added more stable baselines algorithms * added ppo algorithm * introduced stable_baselines_folder * renamed training to callback * satisfied linter * fixed loading problem * try to make tqdm run in stable_baselines * make tqdm running * reduced pmonitoring episodes in sb training * save model only if significantly better * fixed too long test time bug * moved back to 250 episodes testing * set timeout to 15 minutes * added first batch of fixes to @NikkelM feedback * added type annotations and asserts in stable_baselines_model * added sbtraining to training_scenario * applied comments in am_configuration * solved .dat problem and fixed crashing asserts * reintroduced _end_of_training * removed deprecated if * Moved '.dat' to function call instead of appending within function * Fixed assert * fixed model file ending bug * Add short explanation docstring Co-authored-by: Johann Schulze Tast <35633229+blackjack2693@users.noreply.github.com> * fixed wrong docstring * Fixed tests Co-authored-by: NikkelM Co-authored-by: Johann Schulze Tast <35633229+blackjack2693@users.noreply.github.com> commit 018387606ffe99bdeaf65c91720dce75149f59e1 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Mon Apr 11 11:31:15 2022 +0200 name from `names` for container commit ad70b7ca95d876889965eb5d9c5125e91b860434 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Sun Apr 10 16:32:47 2022 +0200 fix #380 commit 8e1314ca0f8cd631dce0959dacefaed9b51dd292 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Sun Apr 10 16:05:28 2022 +0200 multiple experiments are supported on the webserver commit 53a40007f7c6ddcf2dc4be0930298234269400d3 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Fri Apr 8 14:36:44 2022 +0200 support for starting multiple container on docker side commit 3c94f4325123f82d8bb2fe25554004560ebd0c3e Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Fri Apr 8 11:46:07 2022 +0200 ability to add `DockerInfo` for multiple container --- docker/app.py | 29 +- docker/docker_manager.py | 42 +- docker/test_docker_manager.py | 70 ++ setup.cfg | 1 + tests/test_config_validation.py | 774 +++++++++--------- webserver/alpha_business_app/buttons.py | 46 +- .../alpha_business_app/container_parser.py | 53 ++ webserver/alpha_business_app/handle_files.py | 7 +- .../alpha_business_app/handle_requests.py | 30 +- .../alpha_business_app/models/container.py | 11 +- .../tests/constant_tests.py | 1 + .../tests/test_api_interaction.py | 71 +- webserver/alpha_business_app/views.py | 2 + webserver/templates/configurator.html | 6 +- 14 files changed, 670 insertions(+), 473 deletions(-) create mode 100644 webserver/alpha_business_app/container_parser.py diff --git a/docker/app.py b/docker/app.py index 3df5620d..dbab6275 100644 --- a/docker/app.py +++ b/docker/app.py @@ -1,7 +1,7 @@ # app.py import uvicorn -from docker_manager import DockerManager +from docker_manager import DockerInfo, DockerManager from fastapi import FastAPI, Request from fastapi.responses import JSONResponse, StreamingResponse @@ -41,21 +41,30 @@ def is_invalid_status(status: str) -> bool: @app.post('/start') -async def start_container(config: Request) -> JSONResponse: +async def start_container(num_experiments: int, config: Request) -> JSONResponse: """ Start a container with the specified config.json and perform a command on it. Args: - config (Request): The combined hyperparameter_config.json and environment_config_command.json files that should be sent to the container. + num_experiments (int): the number of container, that should be started with this configuration + config (Request): The combined hyperparameter_config.json and environment_config_command.json files that should be sent to the container. Returns: - JSONResponse: The response of the Docker start request. Contains the port used on the host in the data-field. - """ - container_info = manager.start(config=await config.json()) - if (is_invalid_status(container_info.status) or container_info.data is False): - return JSONResponse(status_code=404, content=vars(container_info)) - else: - return JSONResponse(vars(container_info)) + JSONResponse: If starting was successfull the response contains multiple dicts, one for each started container. + If not, there will be one dict with an error message + """ + all_container_infos = manager.start(config=await config.json(), count=num_experiments) + # check if all prerequisite were met + if type(all_container_infos) == DockerInfo: + return JSONResponse(status_code=404, content=vars(all_container_infos)) + + return_dict = {} + for index in range(num_experiments): + if (is_invalid_status(all_container_infos[index].status) or all_container_infos[index].data is False): + return JSONResponse(status_code=404, content=vars(all_container_infos[index])) + return_dict[index] = vars(all_container_infos[index]) + print(f'successfully started {num_experiments} container') + return JSONResponse(return_dict) @app.get('/health/') diff --git a/docker/docker_manager.py b/docker/docker_manager.py index 2bd97c44..0f56002d 100644 --- a/docker/docker_manager.py +++ b/docker/docker_manager.py @@ -32,6 +32,15 @@ def __init__(self, id: str, status: str, data=None, stream: GeneratorType = None self.data = data self.stream = stream + def __eq__(self, other: object) -> bool: + if not isinstance(other, DockerInfo): + # don't attempt to compare against unrelated types + return NotImplemented + return self.id == other.id \ + and self.status == other.status \ + and self.data == other.data \ + and self.stream == other.stream + class DockerManager(): """ @@ -63,15 +72,17 @@ def __new__(cls): cls._update_port_mapping() return cls._instance - def start(self, config: dict) -> DockerInfo: + def start(self, config: dict, count: int) -> DockerInfo or list: """ To be called by the REST API. Create and start a new docker container from the image of the specified command. Args: - config (dict): The combined hyperparameter_config and environment_config_command dictionaries that should be sent to the container. + config (dict): The combined hyperparameter_config and environment_config_command dicts that should be sent to the container. + count (int): number of containers that should be started Returns: - DockerInfo: A JSON serializable object containing the id and the status of the new container. + DockerInfo or list: A JSON serializable object containing the error messages if the prerequisite were not met, or a list of + DockerInfos for the container(s) """ if 'hyperparameter' not in config: return DockerInfo(id='No container was started', status='The config is missing the "hyperparameter"-field') @@ -84,17 +95,20 @@ def start(self, config: dict) -> DockerInfo: print(f'Command with ID {command_id} not allowed') return DockerInfo(id='No container was started', status=f'Command not allowed: {command_id}') - image_id = self._confirm_image_exists() - if image_id is None: - return DockerInfo(None, 'Image build failed') - - # start a container for the image of the requested command - container_info: DockerInfo = self._create_container(command_id, config, use_gpu=False) - if container_info.status.__contains__('Image not found') or container_info.data is False: - self.remove_container(container_info.id) - return container_info - - return self._start_container(container_info.id) + if not self._confirm_image_exists(): + return DockerInfo(id='No container was started', status='Image build failed') + + all_container_infos = [] + for _ in range(count): + # start a container for the image of the requested command + container_info: DockerInfo = self._create_container(command_id, config, use_gpu=False) + if 'Image not found' in container_info.status or container_info.data is False: + # something is wrong with our container + self.remove_container(container_info.id) + return container_info + # the container is fine, we can start the container now + all_container_infos += [self._start_container(container_info.id)] + return all_container_infos def health(self, container_id: str) -> DockerInfo: """ diff --git a/docker/test_docker_manager.py b/docker/test_docker_manager.py index a5b83be4..48ce90a9 100644 --- a/docker/test_docker_manager.py +++ b/docker/test_docker_manager.py @@ -94,3 +94,73 @@ def test_port_mapping_initialization(): def test_allowed_commands_is_up_to_date(): assert set(manager._allowed_commands) == {'training', 'exampleprinter', 'agent_monitoring'}, \ 'The set of allowed commands has changed, please update this and all the other tests!' + + +invalid_start_parameter_testcases = [ + ({'environment': {'task': 'fasel'}, 'hyperparameter': {'test': 'fasel'}}, + {'id': 'No container was started', 'status': 'Command not allowed: fasel'}), + ({'environment': {'task': 'fasel'}}, {'id': 'No container was started', 'status': 'The config is missing the \"hyperparameter\"-field'}), + ({'hyperparameter': {'task': 'fasel'}}, {'id': 'No container was started', 'status': 'The config is missing the \"environment\"-field'}) +] + + +@pytest.mark.parametrize('test_config, expected_docker_info_params', invalid_start_parameter_testcases) +def test_start_with_invalid_command(test_config, expected_docker_info_params): + expected_docker_info = docker_manager.DockerInfo(**expected_docker_info_params) + + actual_docker_info = docker_manager.DockerManager().start(test_config, 2) + assert expected_docker_info == actual_docker_info + + +def test_start_but_no_image(): + test_config = {'environment': {'task': 'training'}, 'hyperparameter': {'bal': 'fasel'}} + expected_docker_info = docker_manager.DockerInfo(id='No container was started', status='Image build failed') + + with patch('docker_manager.DockerManager._confirm_image_exists') as image_exists_mock: + image_exists_mock.return_value = None + + actual_docker_info = docker_manager.DockerManager().start(test_config, 2) + assert expected_docker_info == actual_docker_info + image_exists_mock.assert_called_once() + + +invalid_create_container_status_testcases = [ + ({'id': 'test', 'status': 'Image not found bla'}), + ({'id': 'test', 'status': 'test status', 'data': False}) +] + + +@pytest.mark.parametrize('docker_info_mock_parameter', invalid_create_container_status_testcases) +def test_start_create_container_failed(docker_info_mock_parameter): + test_config = {'environment': {'task': 'training'}, 'hyperparameter': {'bal': 'fasel'}} + expected_docker_info = docker_manager.DockerInfo(**docker_info_mock_parameter) + + with patch('docker_manager.DockerManager._confirm_image_exists') as image_exists_mock, \ + patch('docker_manager.DockerManager._create_container') as create_container_mock, \ + patch('docker_manager.DockerManager.remove_container') as remove_container_mock: + image_exists_mock.return_value = '12345' + create_container_mock.return_value = docker_manager.DockerInfo(**docker_info_mock_parameter) + + actual_docker_info = docker_manager.DockerManager().start(test_config, 2) + + assert expected_docker_info == actual_docker_info + image_exists_mock.assert_called_once() + remove_container_mock.assert_called_once_with('test') + + +def test_start_container_works(): + test_config = {'environment': {'task': 'training'}, 'hyperparameter': {'bal': 'fasel'}} + docker_info = docker_manager.DockerInfo(id='test1', status='have a wonderful day') + + with patch('docker_manager.DockerManager._confirm_image_exists') as image_exists_mock, \ + patch('docker_manager.DockerManager._create_container') as create_container_mock, \ + patch('docker_manager.DockerManager.remove_container') as remove_container_mock, \ + patch('docker_manager.DockerManager._start_container') as start_container_mock: + image_exists_mock.return_value = '12345' + create_container_mock.return_value = docker_info + actual_docker_infos = docker_manager.DockerManager().start(test_config, 2) + + assert 2 == len(actual_docker_infos) + image_exists_mock.assert_called_once() + remove_container_mock.assert_not_called() + start_container_mock.assert_called() diff --git a/setup.cfg b/setup.cfg index 4378e185..55419dca 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,7 @@ install_requires = pytest-randomly>=3.11.0 pytest-xdist>=2.5.0 stable-baselines3[extra]>=1.5.0 + names>=0.3.0 python_requires = >=3.8 [options.entry_points] diff --git a/tests/test_config_validation.py b/tests/test_config_validation.py index eda738e7..dd0a816b 100644 --- a/tests/test_config_validation.py +++ b/tests/test_config_validation.py @@ -1,387 +1,387 @@ -import pytest -import utils_tests as ut_t - -import recommerce.configuration.config_validation as config_validation -from recommerce.configuration.environment_config import EnvironmentConfig -from recommerce.configuration.hyperparameter_config import HyperparameterConfig - -########## -# Tests with already combined configs (== hyperparameter and/or environment key on the top-level) -########## -validate_config_valid_combined_final_testcases = [ - ut_t.create_combined_mock_dict(), - ut_t.create_combined_mock_dict(hyperparameter=ut_t.create_hyperparameter_mock_dict(rl=ut_t.create_hyperparameter_mock_dict_rl(gamma=0.5))), - ut_t.create_combined_mock_dict(hyperparameter=ut_t.create_hyperparameter_mock_dict( - sim_market=ut_t.create_hyperparameter_mock_dict_sim_market(max_price=25))), - ut_t.create_combined_mock_dict(environment=ut_t.create_environment_mock_dict(task='exampleprinter')), - ut_t.create_combined_mock_dict(environment=ut_t.create_environment_mock_dict(agents=[ - { - 'name': 'Test_agent', - 'agent_class': 'recommerce.rl.q_learning.q_learning_agent.QLearningCERebuyAgent', - 'argument': '' - }, - { - 'name': 'Test_agent2', - 'agent_class': 'recommerce.market.circular.circular_vendors.RuleBasedCERebuyAgent', - 'argument': '' - } - ])), -] - - -@pytest.mark.parametrize('config', validate_config_valid_combined_final_testcases) -def test_validate_config_valid_combined_final(config): - # If the config is valid, the first member of the tuple returned will be True - validate_status, validate_data = config_validation.validate_config(config, True) - assert validate_status, validate_data - assert isinstance(validate_data, tuple) - assert 'rl' in validate_data[0] - assert 'sim_market' in validate_data[0] - assert 'gamma' in validate_data[0]['rl'] - assert 'max_price' in validate_data[0]['sim_market'] - assert 'task' in validate_data[1] - assert 'agents' in validate_data[1] - - -# These testcases do not cover everything, nor should they, there are simply too many combinations -validate_config_valid_combined_not_final_testcases = [ - ut_t.create_combined_mock_dict( - hyperparameter=ut_t.remove_key('rl', ut_t.create_hyperparameter_mock_dict())), - ut_t.create_combined_mock_dict( - hyperparameter=ut_t.create_hyperparameter_mock_dict( - rl=ut_t.remove_key('learning_rate', ut_t.create_hyperparameter_mock_dict_rl(gamma=0.5)))), - ut_t.create_combined_mock_dict( - hyperparameter=ut_t.create_hyperparameter_mock_dict( - rl=ut_t.remove_key('epsilon_start', ut_t.remove_key('learning_rate', ut_t.create_hyperparameter_mock_dict_rl())))), - ut_t.create_combined_mock_dict(environment=ut_t.remove_key('task', ut_t.create_environment_mock_dict())), - ut_t.create_combined_mock_dict(environment=ut_t.remove_key('agents', ut_t.remove_key('task', ut_t.create_environment_mock_dict()))), -] + validate_config_valid_combined_final_testcases - - -@pytest.mark.parametrize('config', validate_config_valid_combined_not_final_testcases) -def test_validate_config_valid_combined_not_final(config): - # If the config is valid, the first member of the returned tuple will be True - validate_status, validate_data = config_validation.validate_config(config, False) - assert validate_status, validate_data - - -validate_config_one_top_key_missing_testcases = [ - (ut_t.create_combined_mock_dict(hyperparameter=None), True), - (ut_t.create_combined_mock_dict(environment=None), True), - (ut_t.create_combined_mock_dict(hyperparameter=None), False), - (ut_t.create_combined_mock_dict(environment=None), False) -] - - -@pytest.mark.parametrize('config, is_final', validate_config_one_top_key_missing_testcases) -def test_validate_config_one_top_key_missing(config, is_final): - validate_status, validate_data = config_validation.validate_config(config, is_final) - assert not validate_status, validate_data - assert 'If your config contains one of "environment" or "hyperparameter" it must also contain the other' == validate_data - - -validate_config_too_many_keys_testcases = [ - True, - False -] - - -@pytest.mark.parametrize('is_final', validate_config_too_many_keys_testcases) -def test_validate_config_too_many_keys(is_final): - test_config = ut_t.create_combined_mock_dict() - test_config['additional_key'] = "this should'nt be allowed" - validate_status, validate_data = config_validation.validate_config(test_config, is_final) - assert not validate_status, validate_data - assert 'Your config should not contain keys other than "environment" and "hyperparameter"' == validate_data -########## -# End of tests with already combined configs (== hyperparameter and/or environment key on the top-level) -########## - - -########## -# Tests without the already split top-level (config keys are mixed and need to be matched) -########## -# These are singular dicts that will get combined for the actual testcases -validate_config_valid_not_final_dicts = [ - { - 'rl': { - 'gamma': 0.5, - 'epsilon_start': 0.9 - } - }, - { - 'sim_market': { - 'max_price': 40 - } - }, - { - 'task': 'training' - }, - { - 'marketplace': 'recommerce.market.circular.circular_sim_market.CircularEconomyRebuyPriceMonopolyScenario' - }, - { - 'agents': [ - { - 'name': 'Rule_Based Agent', - 'agent_class': 'recommerce.market.circular.circular_vendors.RuleBasedCERebuyAgent', - 'argument': '' - }, - { - 'name': 'CE Rebuy Agent (QLearning)', - 'agent_class': 'recommerce.rl.q_learning.q_learning_agent.QLearningCERebuyAgent', - 'argument': 'CircularEconomyRebuyPriceMonopolyScenario_QLearningCERebuyAgent.dat' - } - ] - }, - { - 'agents': [ - { - 'name': 'Rule_Based Agent', - 'agent_class': 'recommerce.market.circular.circular_vendors.RuleBasedCERebuyAgent', - 'argument': '' - } - ] - } -] - - -# get all combinations of the dicts defined above to mix and match as much as possible -mixed_configs = [ - {**dict1, **dict2} for dict1 in validate_config_valid_not_final_dicts for dict2 in validate_config_valid_not_final_dicts -] - - -@pytest.mark.parametrize('config', mixed_configs) -def test_validate_config_valid_not_final(config): - validate_status, validate_data = config_validation.validate_config(config, False) - assert validate_status, f'Test failed with error: {validate_data} on config: {config}' - - -validate_config_valid_final_testcases = [ - {**ut_t.create_hyperparameter_mock_dict(), **ut_t.create_environment_mock_dict()}, - {**ut_t.create_hyperparameter_mock_dict(rl=ut_t.create_hyperparameter_mock_dict_rl(gamma=0.2)), **ut_t.create_environment_mock_dict()}, - {**ut_t.create_hyperparameter_mock_dict(), **ut_t.create_environment_mock_dict(episodes=20)} -] - - -@pytest.mark.parametrize('config', validate_config_valid_final_testcases) -def test_validate_config_valid_final(config): - validate_status, validate_data = config_validation.validate_config(config, True) - assert validate_status, f'Test failed with error: {validate_data} on config: {config}' - assert 'rl' in validate_data[0] - assert 'sim_market' in validate_data[0] - assert 'agents' in validate_data[1] - - -@pytest.mark.parametrize('config', mixed_configs) -def test_split_mixed_config_valid(config): - config_validation.split_mixed_config(config) - - -split_mixed_config_invalid_testcases = [ - { - 'invalid_key': 2 - }, - { - 'rl': { - 'gamma': 0.5 - }, - 'invalid_key': 2 - }, - { - 'agents': [ - { - 'name': 'test', - 'agent_class': 'recommerce.market.circular.circular_vendors.RuleBasedCERebuyAgent', - 'argument': '' - } - ], - 'invalid_key': 2 - } -] - - -@pytest.mark.parametrize('config', split_mixed_config_invalid_testcases) -def test_split_mixed_config_invalid(config): - with pytest.raises(AssertionError) as error_message: - config_validation.split_mixed_config(config) - assert 'Your config contains an invalid key:' in str(error_message.value) - - -validate_sub_keys_invalid_keys_hyperparameter_testcases = [ - { - 'rl': { - 'gamma': 0.5, - 'invalid_key': 2 - } - }, - { - 'sim_market': { - 'max_price': 50, - 'invalid_key': 2 - } - }, - { - 'rl': { - 'gamma': 0.5, - 'invalid_key': 2 - }, - 'sim_market': { - 'max_price': 50, - 'invalid_key': 2 - } - }, - { - 'rl': { - 'gamma': 0.5 - }, - 'sim_market': { - 'max_price': 50, - 'invalid_key': 2 - } - } -] - - -@pytest.mark.parametrize('config', validate_sub_keys_invalid_keys_hyperparameter_testcases) -def test_validate_sub_keys_invalid_keys_hyperparameter(config): - with pytest.raises(AssertionError) as error_message: - top_level_keys = HyperparameterConfig.get_required_fields('top-dict') - config_validation.validate_sub_keys(HyperparameterConfig, config, top_level_keys) - assert 'The key "invalid_key" should not exist within a HyperparameterConfig config' in str(error_message.value) - - -validate_sub_keys_agents_invalid_keys_testcases = [ - { - 'task': 'training', - 'agents': [ - { - 'name': 'name', - 'invalid_key': 2 - } - ] - }, - { - 'agents': [ - { - 'name': '', - 'argument': '', - 'invalid_key': 2 - } - ] - }, - { - 'agents': [ - { - 'argument': '' - }, - { - 'name': '', - 'agent_class': '', - 'argument': '', - 'invalid_key': 2 - } - ] - } -] - - -@pytest.mark.parametrize('config', validate_sub_keys_agents_invalid_keys_testcases) -def test_validate_sub_keys_agents_invalid_keys(config): - with pytest.raises(AssertionError) as error_message: - top_level_keys = EnvironmentConfig.get_required_fields('top-dict') - config_validation.validate_sub_keys(EnvironmentConfig, config, top_level_keys) - assert 'An invalid key for agents was provided:' in str(error_message.value) - - -validate_sub_keys_agents_wrong_type_testcases = [ - { - 'agents': 2 - }, - { - 'agents': 'string' - }, - { - 'agents': 2.0 - }, - { - 'agents': {} - } -] - - -@pytest.mark.parametrize('config', validate_sub_keys_agents_wrong_type_testcases) -def test_validate_sub_keys_agents_wrong_type(config): - with pytest.raises(AssertionError) as error_message: - top_level_keys = EnvironmentConfig.get_required_fields('top-dict') - config_validation.validate_sub_keys(EnvironmentConfig, config, top_level_keys) - assert 'The "agents" key must have a value of type list, but was' in str(error_message.value) - - -validate_sub_keys_agents_wrong_type_testcases = [ - { - 'agents': [ - 2 - ] - }, - { - 'agents': [ - 'string' - ] - }, - { - 'agents': [ - 2.0 - ] - }, - { - 'agents': [ - [] - ] - } -] - - -@pytest.mark.parametrize('config', validate_sub_keys_agents_wrong_type_testcases) -def test_validate_sub_keys_agents_wrong_subtype(config): - with pytest.raises(AssertionError) as error_message: - top_level_keys = EnvironmentConfig.get_required_fields('top-dict') - config_validation.validate_sub_keys(EnvironmentConfig, config, top_level_keys) - assert 'All agents must be of type dict, but this one was' in str(error_message.value) - - -validate_sub_keys_wrong_type_hyperparameter_testcases = [ - { - 'rl': [] - }, - { - 'sim_market': [] - }, - { - 'rl': 2 - }, - { - 'sim_market': 2 - }, - { - 'rl': 'string' - }, - { - 'sim_market': 'string' - }, - { - 'rl': 2.0 - }, - { - 'sim_market': 2.0 - }, -] - - -@pytest.mark.parametrize('config', validate_sub_keys_wrong_type_hyperparameter_testcases) -def test_validate_sub_keys_wrong_type_hyperparameter(config): - with pytest.raises(AssertionError) as error_message: - top_level_keys = HyperparameterConfig.get_required_fields('top-dict') - config_validation.validate_sub_keys(HyperparameterConfig, config, top_level_keys) - assert 'The value of this key must be of type dict:' in str(error_message.value) +import pytest +import utils_tests as ut_t + +import recommerce.configuration.config_validation as config_validation +from recommerce.configuration.environment_config import EnvironmentConfig +from recommerce.configuration.hyperparameter_config import HyperparameterConfig + +########## +# Tests with already combined configs (== hyperparameter and/or environment key on the top-level) +########## +validate_config_valid_combined_final_testcases = [ + ut_t.create_combined_mock_dict(), + ut_t.create_combined_mock_dict(hyperparameter=ut_t.create_hyperparameter_mock_dict(rl=ut_t.create_hyperparameter_mock_dict_rl(gamma=0.5))), + ut_t.create_combined_mock_dict(hyperparameter=ut_t.create_hyperparameter_mock_dict( + sim_market=ut_t.create_hyperparameter_mock_dict_sim_market(max_price=25))), + ut_t.create_combined_mock_dict(environment=ut_t.create_environment_mock_dict(task='exampleprinter')), + ut_t.create_combined_mock_dict(environment=ut_t.create_environment_mock_dict(agents=[ + { + 'name': 'Test_agent', + 'agent_class': 'recommerce.rl.q_learning.q_learning_agent.QLearningCERebuyAgent', + 'argument': '' + }, + { + 'name': 'Test_agent2', + 'agent_class': 'recommerce.market.circular.circular_vendors.RuleBasedCERebuyAgent', + 'argument': '' + } + ])), +] + + +@pytest.mark.parametrize('config', validate_config_valid_combined_final_testcases) +def test_validate_config_valid_combined_final(config): + # If the config is valid, the first member of the tuple returned will be True + validate_status, validate_data = config_validation.validate_config(config, True) + assert validate_status, validate_data + assert isinstance(validate_data, tuple) + assert 'rl' in validate_data[0] + assert 'sim_market' in validate_data[0] + assert 'gamma' in validate_data[0]['rl'] + assert 'max_price' in validate_data[0]['sim_market'] + assert 'task' in validate_data[1] + assert 'agents' in validate_data[1] + + +# These testcases do not cover everything, nor should they, there are simply too many combinations +validate_config_valid_combined_not_final_testcases = [ + ut_t.create_combined_mock_dict( + hyperparameter=ut_t.remove_key('rl', ut_t.create_hyperparameter_mock_dict())), + ut_t.create_combined_mock_dict( + hyperparameter=ut_t.create_hyperparameter_mock_dict( + rl=ut_t.remove_key('learning_rate', ut_t.create_hyperparameter_mock_dict_rl(gamma=0.5)))), + ut_t.create_combined_mock_dict( + hyperparameter=ut_t.create_hyperparameter_mock_dict( + rl=ut_t.remove_key('epsilon_start', ut_t.remove_key('learning_rate', ut_t.create_hyperparameter_mock_dict_rl())))), + ut_t.create_combined_mock_dict(environment=ut_t.remove_key('task', ut_t.create_environment_mock_dict())), + ut_t.create_combined_mock_dict(environment=ut_t.remove_key('agents', ut_t.remove_key('task', ut_t.create_environment_mock_dict()))), +] + validate_config_valid_combined_final_testcases + + +@pytest.mark.parametrize('config', validate_config_valid_combined_not_final_testcases) +def test_validate_config_valid_combined_not_final(config): + # If the config is valid, the first member of the returned tuple will be True + validate_status, validate_data = config_validation.validate_config(config, False) + assert validate_status, validate_data + + +validate_config_one_top_key_missing_testcases = [ + (ut_t.create_combined_mock_dict(hyperparameter=None), True), + (ut_t.create_combined_mock_dict(environment=None), True), + (ut_t.create_combined_mock_dict(hyperparameter=None), False), + (ut_t.create_combined_mock_dict(environment=None), False) +] + + +@pytest.mark.parametrize('config, is_final', validate_config_one_top_key_missing_testcases) +def test_validate_config_one_top_key_missing(config, is_final): + validate_status, validate_data = config_validation.validate_config(config, is_final) + assert not validate_status, validate_data + assert 'If your config contains one of "environment" or "hyperparameter" it must also contain the other' == validate_data + + +validate_config_too_many_keys_testcases = [ + True, + False +] + + +@pytest.mark.parametrize('is_final', validate_config_too_many_keys_testcases) +def test_validate_config_too_many_keys(is_final): + test_config = ut_t.create_combined_mock_dict() + test_config['additional_key'] = "this should'nt be allowed" + validate_status, validate_data = config_validation.validate_config(test_config, is_final) + assert not validate_status, validate_data + assert 'Your config should not contain keys other than "environment" and "hyperparameter"' == validate_data +########## +# End of tests with already combined configs (== hyperparameter and/or environment key on the top-level) +########## + + +########## +# Tests without the already split top-level (config keys are mixed and need to be matched) +########## +# These are singular dicts that will get combined for the actual testcases +validate_config_valid_not_final_dicts = [ + { + 'rl': { + 'gamma': 0.5, + 'epsilon_start': 0.9 + } + }, + { + 'sim_market': { + 'max_price': 40 + } + }, + { + 'task': 'training' + }, + { + 'marketplace': 'recommerce.market.circular.circular_sim_market.CircularEconomyRebuyPriceMonopolyScenario' + }, + { + 'agents': [ + { + 'name': 'Rule_Based Agent', + 'agent_class': 'recommerce.market.circular.circular_vendors.RuleBasedCERebuyAgent', + 'argument': '' + }, + { + 'name': 'CE Rebuy Agent (QLearning)', + 'agent_class': 'recommerce.rl.q_learning.q_learning_agent.QLearningCERebuyAgent', + 'argument': 'CircularEconomyRebuyPriceMonopolyScenario_QLearningCERebuyAgent.dat' + } + ] + }, + { + 'agents': [ + { + 'name': 'Rule_Based Agent', + 'agent_class': 'recommerce.market.circular.circular_vendors.RuleBasedCERebuyAgent', + 'argument': '' + } + ] + } +] + + +# get all combinations of the dicts defined above to mix and match as much as possible +mixed_configs = [ + {**dict1, **dict2} for dict1 in validate_config_valid_not_final_dicts for dict2 in validate_config_valid_not_final_dicts +] + + +@pytest.mark.parametrize('config', mixed_configs) +def test_validate_config_valid_not_final(config): + validate_status, validate_data = config_validation.validate_config(config, False) + assert validate_status, f'Test failed with error: {validate_data} on config: {config}' + + +validate_config_valid_final_testcases = [ + {**ut_t.create_hyperparameter_mock_dict(), **ut_t.create_environment_mock_dict()}, + {**ut_t.create_hyperparameter_mock_dict(rl=ut_t.create_hyperparameter_mock_dict_rl(gamma=0.2)), **ut_t.create_environment_mock_dict()}, + {**ut_t.create_hyperparameter_mock_dict(), **ut_t.create_environment_mock_dict(episodes=20)} +] + + +@pytest.mark.parametrize('config', validate_config_valid_final_testcases) +def test_validate_config_valid_final(config): + validate_status, validate_data = config_validation.validate_config(config, True) + assert validate_status, f'Test failed with error: {validate_data} on config: {config}' + assert 'rl' in validate_data[0] + assert 'sim_market' in validate_data[0] + assert 'agents' in validate_data[1] + + +@pytest.mark.parametrize('config', mixed_configs) +def test_split_mixed_config_valid(config): + config_validation.split_mixed_config(config) + + +split_mixed_config_invalid_testcases = [ + { + 'invalid_key': 2 + }, + { + 'rl': { + 'gamma': 0.5 + }, + 'invalid_key': 2 + }, + { + 'agents': [ + { + 'name': 'test', + 'agent_class': 'recommerce.market.circular.circular_vendors.RuleBasedCERebuyAgent', + 'argument': '' + } + ], + 'invalid_key': 2 + } +] + + +@pytest.mark.parametrize('config', split_mixed_config_invalid_testcases) +def test_split_mixed_config_invalid(config): + with pytest.raises(AssertionError) as error_message: + config_validation.split_mixed_config(config) + assert 'Your config contains an invalid key:' in str(error_message.value) + + +validate_sub_keys_invalid_keys_hyperparameter_testcases = [ + { + 'rl': { + 'gamma': 0.5, + 'invalid_key': 2 + } + }, + { + 'sim_market': { + 'max_price': 50, + 'invalid_key': 2 + } + }, + { + 'rl': { + 'gamma': 0.5, + 'invalid_key': 2 + }, + 'sim_market': { + 'max_price': 50, + 'invalid_key': 2 + } + }, + { + 'rl': { + 'gamma': 0.5 + }, + 'sim_market': { + 'max_price': 50, + 'invalid_key': 2 + } + } +] + + +@pytest.mark.parametrize('config', validate_sub_keys_invalid_keys_hyperparameter_testcases) +def test_validate_sub_keys_invalid_keys_hyperparameter(config): + with pytest.raises(AssertionError) as error_message: + top_level_keys = HyperparameterConfig.get_required_fields('top-dict') + config_validation.validate_sub_keys(HyperparameterConfig, config, top_level_keys) + assert 'The key "invalid_key" should not exist within a HyperparameterConfig config' in str(error_message.value) + + +validate_sub_keys_agents_invalid_keys_testcases = [ + { + 'task': 'training', + 'agents': [ + { + 'name': 'name', + 'invalid_key': 2 + } + ] + }, + { + 'agents': [ + { + 'name': '', + 'argument': '', + 'invalid_key': 2 + } + ] + }, + { + 'agents': [ + { + 'argument': '' + }, + { + 'name': '', + 'agent_class': '', + 'argument': '', + 'invalid_key': 2 + } + ] + } +] + + +@pytest.mark.parametrize('config', validate_sub_keys_agents_invalid_keys_testcases) +def test_validate_sub_keys_agents_invalid_keys(config): + with pytest.raises(AssertionError) as error_message: + top_level_keys = EnvironmentConfig.get_required_fields('top-dict') + config_validation.validate_sub_keys(EnvironmentConfig, config, top_level_keys) + assert 'An invalid key for agents was provided:' in str(error_message.value) + + +validate_sub_keys_agents_wrong_type_testcases = [ + { + 'agents': 2 + }, + { + 'agents': 'string' + }, + { + 'agents': 2.0 + }, + { + 'agents': {} + } +] + + +@pytest.mark.parametrize('config', validate_sub_keys_agents_wrong_type_testcases) +def test_validate_sub_keys_agents_wrong_type(config): + with pytest.raises(AssertionError) as error_message: + top_level_keys = EnvironmentConfig.get_required_fields('top-dict') + config_validation.validate_sub_keys(EnvironmentConfig, config, top_level_keys) + assert 'The "agents" key must have a value of type list, but was' in str(error_message.value) + + +validate_sub_keys_agents_wrong_type_testcases = [ + { + 'agents': [ + 2 + ] + }, + { + 'agents': [ + 'string' + ] + }, + { + 'agents': [ + 2.0 + ] + }, + { + 'agents': [ + [] + ] + } +] + + +@pytest.mark.parametrize('config', validate_sub_keys_agents_wrong_type_testcases) +def test_validate_sub_keys_agents_wrong_subtype(config): + with pytest.raises(AssertionError) as error_message: + top_level_keys = EnvironmentConfig.get_required_fields('top-dict') + config_validation.validate_sub_keys(EnvironmentConfig, config, top_level_keys) + assert 'All agents must be of type dict, but this one was' in str(error_message.value) + + +validate_sub_keys_wrong_type_hyperparameter_testcases = [ + { + 'rl': [] + }, + { + 'sim_market': [] + }, + { + 'rl': 2 + }, + { + 'sim_market': 2 + }, + { + 'rl': 'string' + }, + { + 'sim_market': 'string' + }, + { + 'rl': 2.0 + }, + { + 'sim_market': 2.0 + }, +] + + +@pytest.mark.parametrize('config', validate_sub_keys_wrong_type_hyperparameter_testcases) +def test_validate_sub_keys_wrong_type_hyperparameter(config): + with pytest.raises(AssertionError) as error_message: + top_level_keys = HyperparameterConfig.get_required_fields('top-dict') + config_validation.validate_sub_keys(HyperparameterConfig, config, top_level_keys) + assert 'The value of this key must be of type dict:' in str(error_message.value) diff --git a/webserver/alpha_business_app/buttons.py b/webserver/alpha_business_app/buttons.py index 43dd0dfa..4a911934 100644 --- a/webserver/alpha_business_app/buttons.py +++ b/webserver/alpha_business_app/buttons.py @@ -1,5 +1,3 @@ -import copy - from django.http import HttpResponse from django.shortcuts import redirect, render from django.utils import timezone @@ -7,7 +5,8 @@ from recommerce.configuration.config_validation import validate_config from .config_merger import ConfigMerger -from .config_parser import ConfigFlatDictParser, ConfigModelParser +from .config_parser import ConfigFlatDictParser +from .container_parser import parse_response_to_database from .handle_files import download_file from .handle_requests import send_get_request, send_get_request_with_streaming, send_post_request, stop_container from .models.config import Config @@ -181,9 +180,8 @@ def _delete_container(self) -> HttpResponse: Returns: HttpResponse: a defined rendering. """ - raw_data = {'container_id': self.wanted_container.id} if not self.wanted_container.is_archived(): - self.message = stop_container(raw_data).status() + self.message = stop_container(self.wanted_container.id).status() if self.message[0] == 'success' or self.wanted_container.is_archived(): self.wanted_container.delete() @@ -216,7 +214,7 @@ def _health(self) -> HttpResponse: Returns: HttpResponse: A default response with default values or a response containing the error field. """ - response = send_get_request('health', self.request.POST) + response = send_get_request('health', self.request.POST['container_id']) if response.ok(): response = response.content update_container(response['id'], {'last_check_at': timezone.now(), 'health_status': response['status']}) @@ -234,7 +232,7 @@ def _logs(self) -> HttpResponse: Returns: HttpResponse: A default response with default values or a response containing the error field. """ - response = send_get_request('logs', self.request.POST) + response = send_get_request('logs', self.request.POST['container_id']) self.data = '' if response.ok(): # reverse the output for better readability @@ -282,7 +280,7 @@ def _remove(self) -> HttpResponse: Returns: HttpResponse: An appropriate rendering """ - self.message = stop_container(self.request.POST).status() + self.message = stop_container(self.request.POST['container_id']).status() return self._decide_rendering() def _start(self) -> HttpResponse: @@ -302,24 +300,18 @@ def _start(self) -> HttpResponse: self.message = ['error', validate_data] return self._decide_rendering() - response = send_post_request('start', config_dict) + num_experiments = post_request['num_experiments'][0] if post_request['num_experiments'][0] else 1 + response = send_post_request('start', config_dict, num_experiments) + if response.ok(): # put container into database - response = response.content - # check if a container with the same id already exists - if Container.objects.filter(id=response['id']).exists(): - # we will kindly ask the user to try it again and stop the container - # TODO insert better handling here - self.message = ['error', 'The new container has the same id as an already existing container, please try again.'] - return self._remove() - # get all necessary parameters for container object - container_name = self.request.POST['experiment_name'] - container_name = container_name if container_name != '' else response['id'][:10] - config_object = ConfigModelParser().parse_config(copy.deepcopy(config_dict)) - command = config_object.environment.task - Container.objects.create(id=response['id'], config=config_object, name=container_name, command=command) - config_object.name = f'Config for {container_name}' - config_object.save() + container_name = post_request['experiment_name'][0] + was_successfull, error_container_ids, data = parse_response_to_database(response, config_dict, container_name) + if not was_successfull: + self.message = ['error', data] + for error_container_id in error_container_ids: + stop_container(error_container_id) + return self._decide_rendering() return redirect('/observe', {'success': 'You successfully launched an experiment'}) else: self.message = response.status() @@ -335,7 +327,7 @@ def _tensorboard_link(self) -> HttpResponse: """ if self.wanted_container.has_tensorboard_link(): return redirect(self.wanted_container.tensorboard_link) - response = send_get_request('data/tensorboard', self.request.POST) + response = send_get_request('data/tensorboard', self.request.POST['container_id']) if response.ok(): update_container(self.wanted_container.id, {'tensorboard_link': response.content['data']}) return redirect(response.content['data']) @@ -352,9 +344,9 @@ def _toggle_pause(self) -> HttpResponse: """ # check, whether the request wants to pause or to unpause the container if self.wanted_container.is_paused(): - response = send_get_request('unpause', self.request.POST) + response = send_get_request('unpause', self.request.POST['container_id']) else: - response = send_get_request('pause', self.request.POST) + response = send_get_request('pause', self.request.POST['container_id']) if response.ok(): response = response.content diff --git a/webserver/alpha_business_app/container_parser.py b/webserver/alpha_business_app/container_parser.py new file mode 100644 index 00000000..7025be4e --- /dev/null +++ b/webserver/alpha_business_app/container_parser.py @@ -0,0 +1,53 @@ +import copy + +import names + +from .config_parser import ConfigModelParser +from .models.container import Container + + +def parse_response_to_database(api_response, config_dict: dict, given_name: str) -> None: + """ + Parses an API response containing multiple container to the database. + + Args: + api_response (APIResponse): The converted response from the docker API. + config_dict (dict): The dict the container have been started with. + given_name (str): the name the user put into the field + """ + started_container = api_response.content + # check if the api response is correct + for _, container_info in started_container.items(): + if type(container_info) != dict: + return False, [], 'The API answer was wrong, please try' + + num_experiments = len(started_container) + name = names.get_first_name() if not given_name else given_name + + # save the used config + config_object = ConfigModelParser().parse_config(copy.deepcopy(config_dict)) + config_object.name = f'Config for {name}' + config_object.save() + + command = config_object.environment.task + + for container_count, container_info in started_container.items(): + # check if a container with the same id already exists + if Container.objects.filter(id=container_info['id']).exists(): + # we will kindly ask the user to try it again and stop the container + # TODO insert better handling here + all_container_ids = [info['id'] for _, info in started_container.items()] + return False, all_container_ids, 'The new container has the same id as an already existing container, please try again.' + # get name for container + if num_experiments != 1: + current_container_name = name + f' ({container_count})' + else: + current_container_name = name + + # create the container + Container.objects.create(id=container_info['id'], + command=command, + config=config_object, + health_status=container_info['status'], + name=current_container_name) + return True, [], name diff --git a/webserver/alpha_business_app/handle_files.py b/webserver/alpha_business_app/handle_files.py index 41ec243b..30dbd7c1 100644 --- a/webserver/alpha_business_app/handle_files.py +++ b/webserver/alpha_business_app/handle_files.py @@ -32,8 +32,9 @@ def handle_uploaded_file(request, uploaded_config) -> HttpResponse: Checks if an uploaded config file is valid and parses it to the datastructure. Args: - request (Request): + request (Request): post request by the user uploaded_config (InMemoryUploadedFile): by user uploaded config file + filename (str, optional): the filename of the uploaded file. Defaults to ''. Returns: HttpResponse: either a redirect to the configurator or a render for the upload with an error message @@ -69,7 +70,9 @@ def handle_uploaded_file(request, uploaded_config) -> HttpResponse: except ValueError: return render(request, 'upload.html', {'error': 'Your config is wrong'}) - Config.objects.create(environment=web_environment_config, hyperparameter=web_hyperparameter_config, name=request.POST['config_name']) + given_name = request.POST['config_name'] + config_name = given_name if given_name else uploaded_config.name + Config.objects.create(environment=web_environment_config, hyperparameter=web_hyperparameter_config, name=config_name) return redirect('/configurator', {'success': 'You successfully uploaded a config file'}) diff --git a/webserver/alpha_business_app/handle_requests.py b/webserver/alpha_business_app/handle_requests.py index ac3e52af..41c65005 100644 --- a/webserver/alpha_business_app/handle_requests.py +++ b/webserver/alpha_business_app/handle_requests.py @@ -6,21 +6,19 @@ DOCKER_API = 'http://127.0.0.1:8000' # remember to include the port and the protocol, i.e. http:// -def send_get_request(wanted_action: str, raw_data: dict) -> APIResponse: +def send_get_request(wanted_action: str, container_id: str) -> APIResponse: """ Sends a get request to the API with the wanted action for a wanted container. Args: wanted_action (str): The API call that should be performed. Needs to be a key in `raw_data` - raw_data (dict): various post parameters, - must include the wanted action as key and the container_id as the value for this key. + container_id (str): id of container the action should be performed on Returns: APIResponse: Response from the API converted into our special format. """ - wanted_container = raw_data['container_id'] try: - response = requests.get(f'{DOCKER_API}/{wanted_action}', params={'id': str(wanted_container)}) + response = requests.get(f'{DOCKER_API}/{wanted_action}', params={'id': str(container_id)}) except requests.exceptions.RequestException: return APIResponse('error', content='The API is unavailable') if response.ok: @@ -48,19 +46,9 @@ def send_get_request_with_streaming(wanted_action: str, wanted_container: str) - return _error_handling_API(response) -def send_post_request(route: str, body: dict) -> APIResponse: - """ - Sends a post request to the API with the requested parameter, a body and a command as parameter - - Args: - route (str): A post route from the API. - body (dict): The body that should be send to the API. - - Returns: - APIResponse: Response from the API converted into our special format. - """ +def send_post_request(route: str, body: dict, num_experiments: int) -> APIResponse: try: - response = requests.post(f'{DOCKER_API}/{route}', json=body) + response = requests.post(f'{DOCKER_API}/{route}', json=body, params={'num_experiments': num_experiments}) except requests.exceptions.RequestException: return APIResponse('error', content='The API is unavailable') if response.ok: @@ -68,20 +56,20 @@ def send_post_request(route: str, body: dict) -> APIResponse: return _error_handling_API(response) -def stop_container(post_request: dict) -> APIResponse: +def stop_container(container_id: str) -> APIResponse: """ Sends an API request to stop and remove the container on the remote machine. Args: - post_request (dict): parameters for the request, must include the key 'remove' and as its value the container_id + post_request (str): id of container that should be stopped Returns: APIResponse: Response from the API converted into our special format. """ - response = send_get_request('remove', post_request) + response = send_get_request('remove', container_id) if response.ok() or response.not_found(): # mark container as archived - update_container(post_request['container_id'], {'health_status': 'archived'}) + update_container(container_id, {'health_status': 'archived'}) return APIResponse('success', content='You successfully stopped the container') return response diff --git a/webserver/alpha_business_app/models/container.py b/webserver/alpha_business_app/models/container.py index 0727fd93..cbac7957 100644 --- a/webserver/alpha_business_app/models/container.py +++ b/webserver/alpha_business_app/models/container.py @@ -24,15 +24,22 @@ def has_tensorboard_link(self): return self.tensorboard_link != '' -def update_container(container_id: str, updated_values: dict) -> None: +def update_container(container_id: str, updated_values: dict) -> bool: """ This will update the container belonging to the given id with the data given in `updated_values`. Args: id (str): id for the container that should be updated updated_values (dict): All keys need to be member variables of `Container`. + + Returns: + bool: indicating if updating the container worked. """ - saved_container = Container.objects.get(id=container_id) + try: + saved_container = Container.objects.get(id=container_id) + except Exception: + return False for key, value in updated_values.items(): setattr(saved_container, key, value) saved_container.save() + return True diff --git a/webserver/alpha_business_app/tests/constant_tests.py b/webserver/alpha_business_app/tests/constant_tests.py index cdc77ce7..c5fdd6e8 100644 --- a/webserver/alpha_business_app/tests/constant_tests.py +++ b/webserver/alpha_business_app/tests/constant_tests.py @@ -2,6 +2,7 @@ 'csrfmiddlewaretoken': ['PHZ3VkxiJkrk2gnBCkgNfYJAdUsdb4V5e7CO26nJuENMtSas7BVapRGJJ0B3t9HZ'], 'action': ['start'], 'experiment_name': ['test_experiment'], + 'num_experiments': ['2'], 'environment-task': ['training'], 'environment-episodes': [''], 'environment-plot_interval': [''], diff --git a/webserver/alpha_business_app/tests/test_api_interaction.py b/webserver/alpha_business_app/tests/test_api_interaction.py index e422716a..28a2cee8 100644 --- a/webserver/alpha_business_app/tests/test_api_interaction.py +++ b/webserver/alpha_business_app/tests/test_api_interaction.py @@ -44,6 +44,7 @@ def test_health_button(self): actual_arguments[2]['all_saved_containers'] = list(actual_arguments[2]['all_saved_containers']) render_mock.assert_called_once() + get_request_mock.assert_called_once() assert expected_arguments == actual_arguments assert 'healthy :)' == Container.objects.get(id='1234').health_status @@ -230,24 +231,76 @@ def test_download_tar_data(self): def test_start_button(self): # mock a request that is sent when user presses a button - request = self._setup_request_with_parameters('/start_container', 'start', EXAMPLE_POST_REQUEST_ARGUMENTS) + request = self._setup_request_with_parameters('/configurator', 'start', EXAMPLE_POST_REQUEST_ARGUMENTS) # setup a button handler for this request - test_button_handler = self._setup_button_handler('download.html', request) + test_button_handler = self._setup_button_handler('configurator.html', request, rendering='config') with patch('alpha_business_app.buttons.send_post_request') as post_request_mock, \ - patch('alpha_business_app.buttons.redirect')as redirect_mock: - post_request_mock.return_value = APIResponse('success', content={'id': '12345'}) + patch('alpha_business_app.buttons.redirect') as redirect_mock: + api_response_dict = { + '0': { + 'id': '2cc1fcd41e69f60055962e89c764f5c442cb2f4b76a9c4c8316c2bb9a5ffcdc6', + 'status': 'running', + 'data': 6006, + 'stream': None + }, '1': { + 'id': 'ca166cff9b83bee9791b435e378574e52d71150c0f790df4fe793d02a86e031f', + 'status': 'sleeping', + 'data': 6007, + 'stream': None + } + } + post_request_mock.return_value = APIResponse('success', content=api_response_dict) test_button_handler.do_button_click() - post_request_mock.assert_called_once_with('start', EXAMPLE_HIERARCHY_DICT) + post_request_mock.assert_called_once_with('start', EXAMPLE_HIERARCHY_DICT, 2) redirect_mock.assert_called_once_with('/observe', {'success': 'You successfully launched an experiment'}) - config_object = Config.objects.all()[1] - assert 'Config for test_experiment' == config_object.name + # assert config exists + config_object = Config.objects.all()[1] + assert 'Config for test_experiment' == config_object.name + + # assert two container were created + container_set = Container.objects.filter(id='2cc1fcd41e69f60055962e89c764f5c442cb2f4b76a9c4c8316c2bb9a5ffcdc6') + assert 1 == len(container_set) + container1 = container_set[0] + assert container1 + assert 'test_experiment (0)' == container1.name + assert 'running' == container1.health_status + + container_set = Container.objects.filter(id='ca166cff9b83bee9791b435e378574e52d71150c0f790df4fe793d02a86e031f') + assert 1 == len(container_set) + container2 = container_set[0] + assert container2 + assert 'test_experiment (1)' == container2.name + assert 'sleeping' == container2.health_status + + def test_id_from_api_already_exists(self): + # mock a request that is sent when user presses a button + request = self._setup_request_with_parameters('/configurator', 'start', EXAMPLE_POST_REQUEST_ARGUMENTS) + # setup a button handler for this request + test_button_handler = self._setup_button_handler('configurator.html', request, rendering='config') + with patch('alpha_business_app.buttons.send_post_request') as post_request_mock, \ + patch('alpha_business_app.buttons.render') as render_mock, \ + patch('alpha_business_app.buttons.stop_container'): + api_response_dict = { + '0': { + 'id': '1234', + 'status': 'running', + 'data': 6006, + 'stream': None + } + } + post_request_mock.return_value = APIResponse('success', content=api_response_dict) + + test_button_handler.do_button_click() + post_request_mock.assert_called_once_with('start', EXAMPLE_HIERARCHY_DICT, 2) + render_mock.assert_called_once() + assert 1 == len(Container.objects.all()) - def _setup_button_handler(self, view: str, request: RequestFactory) -> ButtonHandler: + def _setup_button_handler(self, view: str, request: RequestFactory, rendering: str = 'default') -> ButtonHandler: return ButtonHandler(request, view=view, container=self.test_container, - rendering_method='default') + rendering_method=rendering) def _setup_request(self, view: str, action: str) -> RequestFactory: request = RequestFactory().post(view, {'action': action, 'container_id': '1234'}) diff --git a/webserver/alpha_business_app/views.py b/webserver/alpha_business_app/views.py index 3b63b0b1..5b4df947 100644 --- a/webserver/alpha_business_app/views.py +++ b/webserver/alpha_business_app/views.py @@ -47,6 +47,8 @@ def observe(request) -> HttpResponse: def upload(request) -> HttpResponse: if request.method == 'POST': form = UploadFileForm(request.POST, request.FILES) + if not request.FILES: + return render(request, 'upload.html', {'form': form, 'error': 'You need to upload a file before submitting'}) return handle_uploaded_file(request, request.FILES['upload_config']) else: form = UploadFileForm() diff --git a/webserver/templates/configurator.html b/webserver/templates/configurator.html index c91c468d..de43c29c 100644 --- a/webserver/templates/configurator.html +++ b/webserver/templates/configurator.html @@ -37,7 +37,11 @@

You can configure your experiments here

{% csrf_token %} {% include "configuration_items/config.html" with prefill=prefill should_show=True error_dict=error_dict %} - +
+ + +
+
From 44a03d6baec77c2f78701a3f2e6004d2c2d25f1d Mon Sep 17 00:00:00 2001 From: Judith <39854388+felix-20@users.noreply.github.com> Date: Wed, 20 Apr 2022 14:41:48 +0200 Subject: [PATCH 02/92] first attempt for websocket --- docker/docker_manager.py | 4 +++ docker/websocket.py | 22 +++++++++++++ .../alpha_business_app/handle_requests.py | 2 +- .../alpha_business_app/static/js/websocket.js | 8 +++++ webserver/alpha_business_app/urls.py | 4 ++- webserver/alpha_business_app/views.py | 4 +++ webserver/templates/base.html | 4 +++ webserver/templates/test.html | 31 +++++++++++++++++++ 8 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 docker/websocket.py create mode 100644 webserver/alpha_business_app/static/js/websocket.js create mode 100644 webserver/templates/test.html diff --git a/docker/docker_manager.py b/docker/docker_manager.py index 0f56002d..b18d9b09 100644 --- a/docker/docker_manager.py +++ b/docker/docker_manager.py @@ -72,6 +72,10 @@ def __new__(cls): cls._update_port_mapping() return cls._instance + def check_health_of_all_container(self) -> list: + print(self._get_client().containers) + return ['1'] + def start(self, config: dict, count: int) -> DockerInfo or list: """ To be called by the REST API. Create and start a new docker container from the image of the specified command. diff --git a/docker/websocket.py b/docker/websocket.py new file mode 100644 index 00000000..6ca24f1b --- /dev/null +++ b/docker/websocket.py @@ -0,0 +1,22 @@ +# app.py + +import uvicorn +from docker_manager import DockerManager +from fastapi import FastAPI, WebSocket + +manager = DockerManager() + +app = FastAPI() + + +@app.websocket('/ws') +async def websocket_endpoint(websocket: WebSocket): + await websocket.accept() + while True: + is_exited, exited_container_ids = manager.check_health_of_all_container() + if is_exited: + await websocket.send_text(exited_container_ids) + + +if __name__ == '__main__': + uvicorn.run('app:app', host='0.0.0.0', port=8000) diff --git a/webserver/alpha_business_app/handle_requests.py b/webserver/alpha_business_app/handle_requests.py index 41c65005..4bf87824 100644 --- a/webserver/alpha_business_app/handle_requests.py +++ b/webserver/alpha_business_app/handle_requests.py @@ -3,7 +3,7 @@ from .api_response import APIResponse from .models.container import update_container -DOCKER_API = 'http://127.0.0.1:8000' # remember to include the port and the protocol, i.e. http:// +DOCKER_API = 'http://192.168.159.134:8000' # remember to include the port and the protocol, i.e. http:// def send_get_request(wanted_action: str, container_id: str) -> APIResponse: diff --git a/webserver/alpha_business_app/static/js/websocket.js b/webserver/alpha_business_app/static/js/websocket.js new file mode 100644 index 00000000..7c93da08 --- /dev/null +++ b/webserver/alpha_business_app/static/js/websocket.js @@ -0,0 +1,8 @@ +var ws = new WebSocket("ws://192.168.159.134:8000/ws"); +ws.onmessage = function(event) { + var messages = document.getElementById('container-crashed-alert') + var message = document.createElement('div') + var content = document.createTextNode(event.data) + message.appendChild(content) + messages.appendChild(message) +}; diff --git a/webserver/alpha_business_app/urls.py b/webserver/alpha_business_app/urls.py index 60e10959..4dd3dce0 100644 --- a/webserver/alpha_business_app/urls.py +++ b/webserver/alpha_business_app/urls.py @@ -13,5 +13,7 @@ # AJAX relevant url's path('agent', views.agent, name='agent'), path('api_availability', views.api_availability, name='api_availability'), - path('validate_config', views.config_validation, name='config_validation') + path('validate_config', views.config_validation, name='config_validation'), + + path('test', views.test, name='test') ] diff --git a/webserver/alpha_business_app/views.py b/webserver/alpha_business_app/views.py index 5b4df947..9d04bc5e 100644 --- a/webserver/alpha_business_app/views.py +++ b/webserver/alpha_business_app/views.py @@ -110,3 +110,7 @@ def config_validation(request): if not validate_status: return render(request, 'notice_field.html', {'error': validate_data}) return render(request, 'notice_field.html', {'success': 'This config is valid'}) + + +def test(request): + return render(request, 'test.html') diff --git a/webserver/templates/base.html b/webserver/templates/base.html index e524c1f8..0d229aaf 100644 --- a/webserver/templates/base.html +++ b/webserver/templates/base.html @@ -11,6 +11,7 @@ +
@@ -35,6 +36,9 @@
  • Download +
    • +
  • +
  • diff --git a/webserver/templates/test.html b/webserver/templates/test.html new file mode 100644 index 00000000..2c38607d --- /dev/null +++ b/webserver/templates/test.html @@ -0,0 +1,31 @@ + + + + Chat + + +

    WebSocket Chat

    +
    + + +
    +
      +
    + + + From bb09c62dcd1a5327164205e686f9f8685a0dc152 Mon Sep 17 00:00:00 2001 From: Judith <39854388+felix-20@users.noreply.github.com> Date: Thu, 21 Apr 2022 21:09:51 +0200 Subject: [PATCH 03/92] websocket on docker site working --- docker/container_notification_websocket.py | 42 +++++++++++++++++++ docker/docker_manager.py | 20 +++++++-- docker/websocket.py | 22 ---------- .../alpha_business_app/static/js/websocket.js | 24 +++++++---- webserver/templates/base.html | 28 ++++++++++++- 5 files changed, 102 insertions(+), 34 deletions(-) create mode 100644 docker/container_notification_websocket.py delete mode 100644 docker/websocket.py diff --git a/docker/container_notification_websocket.py b/docker/container_notification_websocket.py new file mode 100644 index 00000000..76738d89 --- /dev/null +++ b/docker/container_notification_websocket.py @@ -0,0 +1,42 @@ +import asyncio +import json + +import uvicorn +from docker_manager import DockerManager +from fastapi import FastAPI, WebSocket, WebSocketDisconnect + + +class ConnectionManager: + def __init__(self): + self.active_connections = [] + + async def connect(self, websocket: WebSocket): + await websocket.accept() + self.active_connections.append(websocket) + + def disconnect(self, websocket: WebSocket): + self.active_connections.remove(websocket) + + +manager = DockerManager() +connection_manager = ConnectionManager() +app = FastAPI() + + +@app.websocket('/ws') +async def websocket_endpoint(websocket: WebSocket): + await connection_manager.connect(websocket) + last_docker_info = None + try: + while True: + await asyncio.sleep(1) + is_exited, docker_info = manager.check_health_of_all_container() + if is_exited and last_docker_info != docker_info: + await websocket.send_json(json.dumps(vars(docker_info - last_docker_info))) + last_docker_info = docker_info + except WebSocketDisconnect: + connection_manager.disconnect(websocket) + + +if __name__ == '__main__': + uvicorn.run('container_notification_websocket:app', host='0.0.0.0', port=8001) diff --git a/docker/docker_manager.py b/docker/docker_manager.py index b18d9b09..271ae95a 100644 --- a/docker/docker_manager.py +++ b/docker/docker_manager.py @@ -35,12 +35,21 @@ def __init__(self, id: str, status: str, data=None, stream: GeneratorType = None def __eq__(self, other: object) -> bool: if not isinstance(other, DockerInfo): # don't attempt to compare against unrelated types - return NotImplemented + return False return self.id == other.id \ and self.status == other.status \ and self.data == other.data \ and self.stream == other.stream + def __sub__(self, other_object): + if other_object is None: + return self + if not isinstance(other_object, DockerInfo): + raise NotImplementedError + resulting_ids = ';'.join(set(self.id.split(';')) - set(other_object.id.split(';'))) + resulting_status = ';'.join(set(self.status.split(';')) - set(other_object.status.split(';'))) + return DockerInfo(id=resulting_ids, status=resulting_status) + class DockerManager(): """ @@ -73,8 +82,13 @@ def __new__(cls): return cls._instance def check_health_of_all_container(self) -> list: - print(self._get_client().containers) - return ['1'] + exited_recommerce_containers = list(self._get_client().containers.list(filters={'label': 'recommerce', 'status': 'exited'})) + exited_container = [] + for container in exited_recommerce_containers: + exited_container += [(container.id, docker.APIClient().inspect_container(container.id)['State']['ExitCode'])] + all_container_ids = ';'.join([str(container_id) for container_id, exit_code in exited_container]) + all_container_status = ';'.join([str((container_id, exit_code)) for container_id, exit_code in exited_container]) + return exited_recommerce_containers != [], DockerInfo(id=all_container_ids, status=all_container_status) def start(self, config: dict, count: int) -> DockerInfo or list: """ diff --git a/docker/websocket.py b/docker/websocket.py deleted file mode 100644 index 6ca24f1b..00000000 --- a/docker/websocket.py +++ /dev/null @@ -1,22 +0,0 @@ -# app.py - -import uvicorn -from docker_manager import DockerManager -from fastapi import FastAPI, WebSocket - -manager = DockerManager() - -app = FastAPI() - - -@app.websocket('/ws') -async def websocket_endpoint(websocket: WebSocket): - await websocket.accept() - while True: - is_exited, exited_container_ids = manager.check_health_of_all_container() - if is_exited: - await websocket.send_text(exited_container_ids) - - -if __name__ == '__main__': - uvicorn.run('app:app', host='0.0.0.0', port=8000) diff --git a/webserver/alpha_business_app/static/js/websocket.js b/webserver/alpha_business_app/static/js/websocket.js index 7c93da08..16ddd352 100644 --- a/webserver/alpha_business_app/static/js/websocket.js +++ b/webserver/alpha_business_app/static/js/websocket.js @@ -1,8 +1,16 @@ -var ws = new WebSocket("ws://192.168.159.134:8000/ws"); -ws.onmessage = function(event) { - var messages = document.getElementById('container-crashed-alert') - var message = document.createElement('div') - var content = document.createTextNode(event.data) - message.appendChild(content) - messages.appendChild(message) -}; +// var ws = new WebSocket("ws://192.168.159.134:8001/ws"); +// ws.onopen = function (event) { +// console.log('connection geöffnet'); +// } +// ws.onmessage = function(event) { +// // var messages = document.getElementById('container-crashed-alert') +// // var message = document.createElement('div') +// // var content = document.createTextNode(event.data) +// // message.appendChild(content) +// // messages.appendChild(message) +// console.log(event.data); +// } + +// ws.onclose = function(event) { +// console.log('zu') +// } diff --git a/webserver/templates/base.html b/webserver/templates/base.html index 0d229aaf..0a333699 100644 --- a/webserver/templates/base.html +++ b/webserver/templates/base.html @@ -38,7 +38,33 @@ Download
  • - +

    WebSocket Chat

    +
    + + +
    +
      +
    +
  • From 6bcf3e726cf238778f63a6541f06ac3c93b5d19f Mon Sep 17 00:00:00 2001 From: Judith <39854388+felix-20@users.noreply.github.com> Date: Thu, 21 Apr 2022 21:16:05 +0200 Subject: [PATCH 04/92] just send all of it if things have changed --- docker/container_notification_websocket.py | 2 +- docker/docker_manager.py | 9 ------ .../alpha_business_app/static/js/websocket.js | 30 +++++++++---------- webserver/templates/base.html | 28 +---------------- 4 files changed, 17 insertions(+), 52 deletions(-) diff --git a/docker/container_notification_websocket.py b/docker/container_notification_websocket.py index 76738d89..be1ae9cd 100644 --- a/docker/container_notification_websocket.py +++ b/docker/container_notification_websocket.py @@ -32,7 +32,7 @@ async def websocket_endpoint(websocket: WebSocket): await asyncio.sleep(1) is_exited, docker_info = manager.check_health_of_all_container() if is_exited and last_docker_info != docker_info: - await websocket.send_json(json.dumps(vars(docker_info - last_docker_info))) + await websocket.send_json(json.dumps(vars(docker_info))) last_docker_info = docker_info except WebSocketDisconnect: connection_manager.disconnect(websocket) diff --git a/docker/docker_manager.py b/docker/docker_manager.py index 271ae95a..2bf11727 100644 --- a/docker/docker_manager.py +++ b/docker/docker_manager.py @@ -41,15 +41,6 @@ def __eq__(self, other: object) -> bool: and self.data == other.data \ and self.stream == other.stream - def __sub__(self, other_object): - if other_object is None: - return self - if not isinstance(other_object, DockerInfo): - raise NotImplementedError - resulting_ids = ';'.join(set(self.id.split(';')) - set(other_object.id.split(';'))) - resulting_status = ';'.join(set(self.status.split(';')) - set(other_object.status.split(';'))) - return DockerInfo(id=resulting_ids, status=resulting_status) - class DockerManager(): """ diff --git a/webserver/alpha_business_app/static/js/websocket.js b/webserver/alpha_business_app/static/js/websocket.js index 16ddd352..6c409fc3 100644 --- a/webserver/alpha_business_app/static/js/websocket.js +++ b/webserver/alpha_business_app/static/js/websocket.js @@ -1,16 +1,16 @@ -// var ws = new WebSocket("ws://192.168.159.134:8001/ws"); -// ws.onopen = function (event) { -// console.log('connection geöffnet'); -// } -// ws.onmessage = function(event) { -// // var messages = document.getElementById('container-crashed-alert') -// // var message = document.createElement('div') -// // var content = document.createTextNode(event.data) -// // message.appendChild(content) -// // messages.appendChild(message) -// console.log(event.data); -// } +var ws = new WebSocket("ws://192.168.159.134:8001/ws"); +ws.onopen = function (event) { + console.log('connection geöffnet'); +} +ws.onmessage = function(event) { + // var messages = document.getElementById('container-crashed-alert') + // var message = document.createElement('div') + // var content = document.createTextNode(event.data) + // message.appendChild(content) + // messages.appendChild(message) + console.log(event.data); +} -// ws.onclose = function(event) { -// console.log('zu') -// } +ws.onclose = function(event) { + console.log('zu') +} diff --git a/webserver/templates/base.html b/webserver/templates/base.html index 0a333699..0d229aaf 100644 --- a/webserver/templates/base.html +++ b/webserver/templates/base.html @@ -38,33 +38,7 @@ Download
  • -

    WebSocket Chat

    -
    - - -
    -
      -
    - +
  • From b743f075c9af5884f8da0b8be743119ae7c7a272 Mon Sep 17 00:00:00 2001 From: Judith <39854388+felix-20@users.noreply.github.com> Date: Thu, 21 Apr 2022 21:42:51 +0200 Subject: [PATCH 05/92] started on webserver site --- webserver/alpha_business_app/buttons.py | 2 +- ...ontainer_parser.py => container_helper.py} | 25 ++++++++++++++ .../alpha_business_app/static/js/custom.js | 34 ++++++++++++++++++- .../alpha_business_app/static/js/websocket.js | 16 --------- webserver/alpha_business_app/urls.py | 3 +- webserver/alpha_business_app/views.py | 7 ++-- webserver/templates/alert_field.html | 9 +++++ webserver/templates/base.html | 11 +++--- webserver/templates/notice_field.html | 20 +++++------ webserver/templates/test.html | 31 ----------------- 10 files changed, 88 insertions(+), 70 deletions(-) rename webserver/alpha_business_app/{container_parser.py => container_helper.py} (72%) create mode 100644 webserver/templates/alert_field.html delete mode 100644 webserver/templates/test.html diff --git a/webserver/alpha_business_app/buttons.py b/webserver/alpha_business_app/buttons.py index 4a911934..4f77174b 100644 --- a/webserver/alpha_business_app/buttons.py +++ b/webserver/alpha_business_app/buttons.py @@ -6,7 +6,7 @@ from .config_merger import ConfigMerger from .config_parser import ConfigFlatDictParser -from .container_parser import parse_response_to_database +from .container_helper import parse_response_to_database from .handle_files import download_file from .handle_requests import send_get_request, send_get_request_with_streaming, send_post_request, stop_container from .models.config import Config diff --git a/webserver/alpha_business_app/container_parser.py b/webserver/alpha_business_app/container_helper.py similarity index 72% rename from webserver/alpha_business_app/container_parser.py rename to webserver/alpha_business_app/container_helper.py index 7025be4e..05832120 100644 --- a/webserver/alpha_business_app/container_parser.py +++ b/webserver/alpha_business_app/container_helper.py @@ -1,4 +1,5 @@ import copy +import json import names @@ -51,3 +52,27 @@ def parse_response_to_database(api_response, config_dict: dict, given_name: str) health_status=container_info['status'], name=current_container_name) return True, [], name + + +def get_actually_stopped_container_from_api_notification(api_response: str) -> str: + try: + raw_data = json.loads(json.loads(api_response)) + except json.decoder.JSONDecodeError: + print(f'Could not parse API Response to valid JSON {api_response}') + return False, [] + + polished_data = [item[1:-1].split(',') for item in raw_data['status'].split(';')] + result = [] + for container_id, exit_code in polished_data: + try: + wanted_container = Container.objects.get(id=container_id) + except Container.DoesNotExist: + continue + if wanted_container.health_status.startswith('exit'): + continue + result += [wanted_container.name, exit_code] + print(result) + if not result: + return False, [] + + return True, result diff --git a/webserver/alpha_business_app/static/js/custom.js b/webserver/alpha_business_app/static/js/custom.js index e23de730..5ecd204f 100644 --- a/webserver/alpha_business_app/static/js/custom.js +++ b/webserver/alpha_business_app/static/js/custom.js @@ -68,8 +68,40 @@ $(document).ready(function() { formdata }, success: function (data) { - $("p.notice-field").replaceWith(data); + const notice_field = $('#notice-field'); + if (notice_field.length > 0) { + notice_field.replaceWith(data); + } else { + const endOfContent = document.getElementById('end-of-content'); + endOfContent.insertAdjacentHTML('afterend', data); + } } }); }); + + var url = "ws://192.168.159.134:8001/ws"; + var ws = new WebSocket(url); + ws.onopen = function (_) { + console.log("connection to ", url, "open"); + } + ws.onmessage = function(event) { + const csrftoken = getCookie('csrftoken'); + $.ajax({ + type: "POST", + url: "/container_notification", + data: { + csrfmiddlewaretoken: csrftoken, + api_response: event.data + }, + success: function (data) { + var nav_bar = document.getElementById('main-nav-bar'); + nav_bar.insertAdjacentHTML('afterend', data) + } + }); + console.log(event.data); + } + + ws.onclose = function(_) { + console.log("connection to ", url, "closed"); + } }); diff --git a/webserver/alpha_business_app/static/js/websocket.js b/webserver/alpha_business_app/static/js/websocket.js index 6c409fc3..e69de29b 100644 --- a/webserver/alpha_business_app/static/js/websocket.js +++ b/webserver/alpha_business_app/static/js/websocket.js @@ -1,16 +0,0 @@ -var ws = new WebSocket("ws://192.168.159.134:8001/ws"); -ws.onopen = function (event) { - console.log('connection geöffnet'); -} -ws.onmessage = function(event) { - // var messages = document.getElementById('container-crashed-alert') - // var message = document.createElement('div') - // var content = document.createTextNode(event.data) - // message.appendChild(content) - // messages.appendChild(message) - console.log(event.data); -} - -ws.onclose = function(event) { - console.log('zu') -} diff --git a/webserver/alpha_business_app/urls.py b/webserver/alpha_business_app/urls.py index 4dd3dce0..5fbbce90 100644 --- a/webserver/alpha_business_app/urls.py +++ b/webserver/alpha_business_app/urls.py @@ -14,6 +14,5 @@ path('agent', views.agent, name='agent'), path('api_availability', views.api_availability, name='api_availability'), path('validate_config', views.config_validation, name='config_validation'), - - path('test', views.test, name='test') + path('container_notification', views.container_notification, name='container_notification') ] diff --git a/webserver/alpha_business_app/views.py b/webserver/alpha_business_app/views.py index 9d04bc5e..04349f6b 100644 --- a/webserver/alpha_business_app/views.py +++ b/webserver/alpha_business_app/views.py @@ -9,6 +9,7 @@ from .buttons import ButtonHandler from .config_parser import ConfigFlatDictParser +from .container_helper import get_actually_stopped_container_from_api_notification from .forms import UploadFileForm from .handle_files import handle_uploaded_file from .handle_requests import DOCKER_API @@ -112,5 +113,7 @@ def config_validation(request): return render(request, 'notice_field.html', {'success': 'This config is valid'}) -def test(request): - return render(request, 'test.html') +def container_notification(request): + if request.method == 'POST': + is_notification_necessary, result = get_actually_stopped_container_from_api_notification(request.POST['api_response']) + return render(request, 'alert_field.html', {'warning': result, 'should_render': is_notification_necessary}) diff --git a/webserver/templates/alert_field.html b/webserver/templates/alert_field.html new file mode 100644 index 00000000..78939833 --- /dev/null +++ b/webserver/templates/alert_field.html @@ -0,0 +1,9 @@ +{% if should_render %} + +{% endif %} diff --git a/webserver/templates/base.html b/webserver/templates/base.html index 0d229aaf..b6c57be6 100644 --- a/webserver/templates/base.html +++ b/webserver/templates/base.html @@ -15,7 +15,7 @@
    -
  • Download -
    • -
  • -
  • @@ -48,11 +45,11 @@
    • + {% include "alert_field.html" %} {% block content %} {% endblock content %} -
    - {% include "notice_field.html" %} - +
    + {% include "notice_field.html"%}