From 55355cd5f4354524d8ac7d99de4db1e7aaec42e4 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Thu, 15 Aug 2024 14:31:31 +0100 Subject: [PATCH 1/5] feat(task_context): add task name to subs implicitly --- docs/reference/transforms/task_context.rst | 25 ++++++++++++++++++++-- src/taskgraph/transforms/run/run_task.py | 1 + src/taskgraph/transforms/task_context.py | 3 +++ test/test_transform_task_context.py | 4 ++-- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/docs/reference/transforms/task_context.rst b/docs/reference/transforms/task_context.rst index de13079c5..784590f23 100644 --- a/docs/reference/transforms/task_context.rst +++ b/docs/reference/transforms/task_context.rst @@ -128,10 +128,31 @@ For example: This will give build1 and build2 descriptions with their ``extra_desc`` included while allowing them to share the rest of their task definition. +Implicit Context +~~~~~~~~~~~~~~~~ + +Finally, the name of the task is added to the context implicitly. For example: + +.. code-block:: yaml + + task-defaults: + description: run {name} + task-context: + substitution-fields: + - description + + tasks: + foo: {} + bar: {} + +This will evaluate the description correctly, even though there are no +``task-context`` keys defined on the individual tasks. + Precedence ---------- -If the same key is found in multiple places the order of precedence -is as follows: ``from-parameters``, ``from-object`` keys, ``from-file``. +If the same key is found in multiple places the order of precedence is as +follows: ``from-parameters``, ``from-object`` keys, ``from-file`` and finally +implicit context. That is to say: parameters will always override anything else. diff --git a/src/taskgraph/transforms/run/run_task.py b/src/taskgraph/transforms/run/run_task.py index 3ce894e39..92fd71a89 100644 --- a/src/taskgraph/transforms/run/run_task.py +++ b/src/taskgraph/transforms/run/run_task.py @@ -82,6 +82,7 @@ def common_setup(config, task, taskdesc, command): for repo_config in repo_configs.values(): checkout_path = path.join(vcs_path, repo_config.path) command.append(f"--{repo_config.prefix}-checkout={checkout_path}") + taskdesc["worker"]["env"]["VCS_CHECKOUT_DIR_PATH"] = checkout_path if run["sparse-profile"]: command.append( diff --git a/src/taskgraph/transforms/task_context.py b/src/taskgraph/transforms/task_context.py index bd36d827a..da846d496 100644 --- a/src/taskgraph/transforms/task_context.py +++ b/src/taskgraph/transforms/task_context.py @@ -9,6 +9,7 @@ SCHEMA = Schema( { + Optional("name"): str, Required( "task-context", description=dedent( @@ -108,6 +109,8 @@ def render_task(config, tasks): # substitution key/value pairs. subs.update(sub_config.pop("from-object", {})) subs.update(params_context) + if "name" in task: + subs.setdefault("name", task["name"]) # Now that we have our combined context, we can substitute. for field in fields: diff --git a/test/test_transform_task_context.py b/test/test_transform_task_context.py index 4394cf84e..871867aaa 100644 --- a/test/test_transform_task_context.py +++ b/test/test_transform_task_context.py @@ -15,7 +15,7 @@ TASK_DEFAULTS = { "description": "fake description {object} {file} {param} {object_and_file}" - "{object_and_param} {file_and_param} {object_file_and_param} {param_fallback}", + "{object_and_param} {file_and_param} {object_file_and_param} {param_fallback} {name}", "name": "fake-task-name", "task-context": { "from-parameters": { @@ -80,5 +80,5 @@ def test_transforms(request, run_transform, graph_config): assert ( task["description"] == "fake description object file param object-overrides-file" - "param-overrides-object param-overrides-file param-overrides-all default" + "param-overrides-object param-overrides-file param-overrides-all default fake-task-name" ) From 0137d23978d5eb80129c53e2cc16e160700e8a5d Mon Sep 17 00:00:00 2001 From: Shiva Gupta Date: Mon, 19 Aug 2024 17:59:28 +0930 Subject: [PATCH 2/5] fix: save vcs checkout dir path if exists --- src/taskgraph/transforms/run/run_task.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/taskgraph/transforms/run/run_task.py b/src/taskgraph/transforms/run/run_task.py index 92fd71a89..bec2c17e7 100644 --- a/src/taskgraph/transforms/run/run_task.py +++ b/src/taskgraph/transforms/run/run_task.py @@ -82,7 +82,8 @@ def common_setup(config, task, taskdesc, command): for repo_config in repo_configs.values(): checkout_path = path.join(vcs_path, repo_config.path) command.append(f"--{repo_config.prefix}-checkout={checkout_path}") - taskdesc["worker"]["env"]["VCS_CHECKOUT_DIR_PATH"] = checkout_path + if vcs_path is None or not vcs_path.strip(): + taskdesc["worker"]["env"]["PRIMARY_REPO_PATH"] = checkout_path if run["sparse-profile"]: command.append( From 38a9c42f32b0e4c6ae555cf35f79653cf022d39e Mon Sep 17 00:00:00 2001 From: Shiva Gupta Date: Mon, 19 Aug 2024 18:33:51 +0930 Subject: [PATCH 3/5] fix: save if primary repo path exists --- src/taskgraph/transforms/run/run_task.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/taskgraph/transforms/run/run_task.py b/src/taskgraph/transforms/run/run_task.py index bec2c17e7..6883b7d0d 100644 --- a/src/taskgraph/transforms/run/run_task.py +++ b/src/taskgraph/transforms/run/run_task.py @@ -82,7 +82,8 @@ def common_setup(config, task, taskdesc, command): for repo_config in repo_configs.values(): checkout_path = path.join(vcs_path, repo_config.path) command.append(f"--{repo_config.prefix}-checkout={checkout_path}") - if vcs_path is None or not vcs_path.strip(): + primary_repo_path = taskdesc["worker"]["env"]["PRIMARY_REPO_PATH"] + if primary_repo_path is None or not vcs_path.strip(): taskdesc["worker"]["env"]["PRIMARY_REPO_PATH"] = checkout_path if run["sparse-profile"]: From 3d0ae4c5b92b12c542a8f56ac0aa60cd5470bc5b Mon Sep 17 00:00:00 2001 From: Shiva Gupta Date: Mon, 19 Aug 2024 21:07:02 +0930 Subject: [PATCH 4/5] fix: save vcs path dir with error checking --- src/taskgraph/transforms/run/run_task.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/taskgraph/transforms/run/run_task.py b/src/taskgraph/transforms/run/run_task.py index 6883b7d0d..c738a44b5 100644 --- a/src/taskgraph/transforms/run/run_task.py +++ b/src/taskgraph/transforms/run/run_task.py @@ -79,11 +79,16 @@ def common_setup(config, task, taskdesc, command): ) vcs_path = taskdesc["worker"]["env"]["VCS_PATH"] + + try: + primary_repo_path = taskdesc["worker"]["env"]["PRIMARY_REPO_PATH"] + except Exception: + primary_repo_path_exists = False + for repo_config in repo_configs.values(): checkout_path = path.join(vcs_path, repo_config.path) command.append(f"--{repo_config.prefix}-checkout={checkout_path}") - primary_repo_path = taskdesc["worker"]["env"]["PRIMARY_REPO_PATH"] - if primary_repo_path is None or not vcs_path.strip(): + if primary_repo_path_exists is False or not primary_repo_path.strip(): taskdesc["worker"]["env"]["PRIMARY_REPO_PATH"] = checkout_path if run["sparse-profile"]: From 5dec8718c4752703472075ec12d05070cf66af08 Mon Sep 17 00:00:00 2001 From: Shiva Gupta Date: Tue, 27 Aug 2024 00:05:53 +0930 Subject: [PATCH 5/5] fix: save primary repo path on first iteration --- src/taskgraph/transforms/run/run_task.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/taskgraph/transforms/run/run_task.py b/src/taskgraph/transforms/run/run_task.py index c738a44b5..56204112f 100644 --- a/src/taskgraph/transforms/run/run_task.py +++ b/src/taskgraph/transforms/run/run_task.py @@ -80,16 +80,11 @@ def common_setup(config, task, taskdesc, command): vcs_path = taskdesc["worker"]["env"]["VCS_PATH"] - try: - primary_repo_path = taskdesc["worker"]["env"]["PRIMARY_REPO_PATH"] - except Exception: - primary_repo_path_exists = False - - for repo_config in repo_configs.values(): + for repo_config, index in enumerate(repo_configs.values()): checkout_path = path.join(vcs_path, repo_config.path) command.append(f"--{repo_config.prefix}-checkout={checkout_path}") - if primary_repo_path_exists is False or not primary_repo_path.strip(): - taskdesc["worker"]["env"]["PRIMARY_REPO_PATH"] = checkout_path + if index == 0: + taskdesc["worker"]["env"].setdefault("PRIMARY_REPO_PATH", checkout_path) if run["sparse-profile"]: command.append(