From 8b3259c0532247ac06908319bc60da430514d276 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Mon, 31 Mar 2025 12:15:51 -0400 Subject: [PATCH 01/10] initial changes for R2 support --- .github/create_combined_ci_report.py | 2 +- .github/workflows/release_branches.yml | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/create_combined_ci_report.py b/.github/create_combined_ci_report.py index 07b6a80763bb..1eea1a7fd7da 100755 --- a/.github/create_combined_ci_report.py +++ b/.github/create_combined_ci_report.py @@ -276,7 +276,7 @@ def main(): exit(0) # Upload the report to S3 - s3_client = boto3.client("s3") + s3_client = boto3.client("s3", endpoint_url=os.getenv("S3_URL")) try: s3_client.put_object( diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 677d8e3865e8..eb5f2072701a 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -4,9 +4,13 @@ name: ReleaseBranchCI env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + # AWS region not needed for R2 + # AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + # Point S3_URL to R2 endpoint + S3_URL: ${{ secrets.R2_ENDPOINT }} + S3_DOWNLOAD: ${{ secrets.R2_ENDPOINT }} CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} From f7a5fb34724979a3cc38bc17218205ff3479d55d Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Mon, 31 Mar 2025 13:46:51 -0400 Subject: [PATCH 02/10] Fix creds in reusable_test --- .github/workflows/reusable_test.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml index 240bddd9411c..708649a67309 100644 --- a/.github/workflows/reusable_test.yml +++ b/.github/workflows/reusable_test.yml @@ -69,9 +69,11 @@ env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 CHECK_NAME: ${{inputs.test_name}} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + # Point S3_URL to R2 endpoint + S3_URL: ${{ secrets.R2_ENDPOINT }} + S3_DOWNLOAD: ${{ secrets.R2_ENDPOINT }} CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} From ef1ed6cccd10dcf179cc140f0a8b2365ae674079 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Mon, 31 Mar 2025 14:10:02 -0400 Subject: [PATCH 03/10] found some more secrets to change --- .github/workflows/docker_test_images.yml | 7 ++++--- .github/workflows/reusable_build.yml | 7 ++++--- .github/workflows/reusable_sign.yml | 7 ++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker_test_images.yml b/.github/workflows/docker_test_images.yml index 5a0178039ddc..ff000247ae77 100644 --- a/.github/workflows/docker_test_images.yml +++ b/.github/workflows/docker_test_images.yml @@ -23,9 +23,10 @@ name: Build docker images required: true env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + S3_URL: ${{ secrets.R2_ENDPOINT }} + S3_DOWNLOAD: ${{ secrets.R2_ENDPOINT }} jobs: DockerBuildAarch64: diff --git a/.github/workflows/reusable_build.yml b/.github/workflows/reusable_build.yml index b73fa4fa80a2..fa06fdf8374c 100644 --- a/.github/workflows/reusable_build.yml +++ b/.github/workflows/reusable_build.yml @@ -4,9 +4,10 @@ env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + S3_URL: ${{ secrets.R2_ENDPOINT }} + S3_DOWNLOAD: ${{ secrets.R2_ENDPOINT }} CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} diff --git a/.github/workflows/reusable_sign.yml b/.github/workflows/reusable_sign.yml index 7bfed2758359..88477bd298f2 100644 --- a/.github/workflows/reusable_sign.yml +++ b/.github/workflows/reusable_sign.yml @@ -57,9 +57,10 @@ env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 CHECK_NAME: ${{inputs.test_name}} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + S3_URL: ${{ secrets.R2_ENDPOINT }} + S3_DOWNLOAD: ${{ secrets.R2_ENDPOINT }} CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} From e1c4a4702ea957db978491ba2457c16a68cf941d Mon Sep 17 00:00:00 2001 From: MyroTk Date: Mon, 31 Mar 2025 17:13:24 -0400 Subject: [PATCH 04/10] use aws for sccache temporarily --- .github/workflows/reusable_build.yml | 2 ++ tests/ci/build_check.py | 10 +++++----- tests/ci/env_helper.py | 3 +++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/reusable_build.yml b/.github/workflows/reusable_build.yml index fa06fdf8374c..c57c3da98b31 100644 --- a/.github/workflows/reusable_build.yml +++ b/.github/workflows/reusable_build.yml @@ -13,6 +13,8 @@ env: CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} + SCCACHE_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + SCCACHE_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} name: Build ClickHouse 'on': diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 425b23bc5e93..93ab6aecc293 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -10,7 +10,7 @@ import docker_images_helper from ci_config import CI -from env_helper import REPO_COPY, S3_BUILDS_BUCKET, TEMP_PATH, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY +from env_helper import REPO_COPY, S3_BUILDS_BUCKET, TEMP_PATH, SCCACHE_ACCESS_KEY_ID, SCCACHE_SECRET_ACCESS_KEY from git_helper import Git from pr_info import PRInfo, EventType from report import FAILURE, SUCCESS, JobReport, StatusType @@ -67,8 +67,8 @@ def get_packager_cmd( cmd += " --cache=sccache" cmd += " --s3-rw-access" cmd += f" --s3-bucket={S3_BUILDS_BUCKET}" - cmd += f" --s3-access-key-id={S3_ACCESS_KEY_ID}" - cmd += f" --s3-secret-access-key={S3_SECRET_ACCESS_KEY}" + cmd += f" --s3-access-key-id={SCCACHE_ACCESS_KEY_ID}" + cmd += f" --s3-secret-access-key={SCCACHE_SECRET_ACCESS_KEY}" if build_config.additional_pkgs: cmd += " --additional-pkgs" @@ -238,7 +238,7 @@ def main(): pr_info.sha, build_name, ) - ) + ) src_path = temp_path / "build_source.src.tar.gz" s3_path = s3_path_prefix + "/clickhouse-" + version.string + ".src.tar.gz" logging.info("s3_path %s", s3_path) @@ -251,7 +251,7 @@ def main(): else: logging.info("Source tar doesn't exist") print("Source tar doesn't exist") - + if build_status != SUCCESS: # We check if docker works, because if it's down, it's infrastructure try: diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 10b4037e6575..044f5903fd19 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -34,6 +34,9 @@ S3_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "altinity-build-artifacts") +SCCACHE_ACCESS_KEY_ID = os.getenv("SCCACHE_ACCESS_KEY_ID") +SCCACHE_SECRET_ACCESS_KEY = os.getenv("SCCACHE_SECRET_ACCESS_KEY") + S3_URL = os.getenv("S3_URL", "https://s3.amazonaws.com") S3_DOWNLOAD = os.getenv("S3_DOWNLOAD", S3_URL) S3_ARTIFACT_DOWNLOAD_TEMPLATE = ( From 3afdf99d60add3657ec2367db859abff163ffcfe Mon Sep 17 00:00:00 2001 From: MyroTk Date: Tue, 1 Apr 2025 12:33:32 -0400 Subject: [PATCH 05/10] use sccache on r2 --- .github/workflows/reusable_build.yml | 2 -- docker/packager/packager | 3 ++- tests/ci/build_check.py | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/reusable_build.yml b/.github/workflows/reusable_build.yml index c57c3da98b31..fa06fdf8374c 100644 --- a/.github/workflows/reusable_build.yml +++ b/.github/workflows/reusable_build.yml @@ -13,8 +13,6 @@ env: CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} - SCCACHE_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - SCCACHE_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} name: Build ClickHouse 'on': diff --git a/docker/packager/packager b/docker/packager/packager index 28d8778d03d7..c1e8996510af 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -13,6 +13,7 @@ IMAGE_TYPE = "binary-builder" IMAGE_NAME = f"altinityinfra/{IMAGE_TYPE}" DEFAULT_TMP_PATH = SCRIPT_PATH.parent.absolute() / 'tmp' TEMP_PATH = Path(os.getenv("TEMP_PATH", DEFAULT_TMP_PATH)) +AWS_ENDPOINT_URL = os.getenv("S3_URL") class BuildException(Exception): pass @@ -98,7 +99,7 @@ def run_docker_image_with_env( for key, value in aws_secrets.items(): f.write(f"\n{key}={value}") - extra_parts = f"--volume={host_aws_config_file_path}:{env_part['AWS_CONFIG_FILE']}" + extra_parts = f"--volume={host_aws_config_file_path}:{env_part['AWS_CONFIG_FILE']} -e AWS_ENDPOINT_URL={AWS_ENDPOINT_URL}" env_part = " -e ".join(env_variables) if env_part: diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 93ab6aecc293..e8c41b5086a7 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -10,7 +10,7 @@ import docker_images_helper from ci_config import CI -from env_helper import REPO_COPY, S3_BUILDS_BUCKET, TEMP_PATH, SCCACHE_ACCESS_KEY_ID, SCCACHE_SECRET_ACCESS_KEY +from env_helper import REPO_COPY, S3_BUILDS_BUCKET, TEMP_PATH, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY from git_helper import Git from pr_info import PRInfo, EventType from report import FAILURE, SUCCESS, JobReport, StatusType @@ -67,8 +67,8 @@ def get_packager_cmd( cmd += " --cache=sccache" cmd += " --s3-rw-access" cmd += f" --s3-bucket={S3_BUILDS_BUCKET}" - cmd += f" --s3-access-key-id={SCCACHE_ACCESS_KEY_ID}" - cmd += f" --s3-secret-access-key={SCCACHE_SECRET_ACCESS_KEY}" + cmd += f" --s3-access-key-id={S3_ACCESS_KEY_ID}" + cmd += f" --s3-secret-access-key={S3_SECRET_ACCESS_KEY}" if build_config.additional_pkgs: cmd += " --additional-pkgs" From e92a734a22873fb1ad7c724386fbee16ad2a13ab Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Fri, 4 Apr 2025 14:49:51 -0400 Subject: [PATCH 06/10] Try SCCACHE_ENDPOINT --- docker/packager/packager | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index c1e8996510af..68598b2832f5 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -13,7 +13,7 @@ IMAGE_TYPE = "binary-builder" IMAGE_NAME = f"altinityinfra/{IMAGE_TYPE}" DEFAULT_TMP_PATH = SCRIPT_PATH.parent.absolute() / 'tmp' TEMP_PATH = Path(os.getenv("TEMP_PATH", DEFAULT_TMP_PATH)) -AWS_ENDPOINT_URL = os.getenv("S3_URL") +SCCACHE_ENDPOINT = os.getenv("S3_URL") class BuildException(Exception): pass @@ -99,7 +99,7 @@ def run_docker_image_with_env( for key, value in aws_secrets.items(): f.write(f"\n{key}={value}") - extra_parts = f"--volume={host_aws_config_file_path}:{env_part['AWS_CONFIG_FILE']} -e AWS_ENDPOINT_URL={AWS_ENDPOINT_URL}" + extra_parts = f"--volume={host_aws_config_file_path}:{env_part['AWS_CONFIG_FILE']} -e SCCACHE_ENDPOINT={SCCACHE_ENDPOINT} -e SCCACHE_REGION=auto" env_part = " -e ".join(env_variables) if env_part: From 12978359c6135e80bcde595c4c5b5711e279025b Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Fri, 4 Apr 2025 16:52:43 -0400 Subject: [PATCH 07/10] set ACL manually --- tests/ci/s3_helper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index d0aa034258a5..79dd2259fa09 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -103,6 +103,7 @@ def _upload_file_to_s3( logging.info("Processing file without compression") logging.info("File is too large, do not provide content type") + metadata["ACL"] = "public-read" self.client.upload_file(file_path, bucket_name, s3_path, ExtraArgs=metadata) url = self.s3_url(bucket_name, s3_path) logging.info("Upload %s to %s Meta: %s", file_path, url, metadata) From 460aec9863e456154be278661cf134e76b411103 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Fri, 4 Apr 2025 19:06:09 -0400 Subject: [PATCH 08/10] custom domain for r2 download --- .github/workflows/docker_test_images.yml | 2 +- .github/workflows/release_branches.yml | 2 +- .github/workflows/reusable_build.yml | 2 +- .github/workflows/reusable_sign.yml | 2 +- .github/workflows/reusable_test.yml | 2 +- tests/ci/s3_helper.py | 5 ++++- 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker_test_images.yml b/.github/workflows/docker_test_images.yml index ff000247ae77..e88fa466d1f8 100644 --- a/.github/workflows/docker_test_images.yml +++ b/.github/workflows/docker_test_images.yml @@ -26,7 +26,7 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} S3_URL: ${{ secrets.R2_ENDPOINT }} - S3_DOWNLOAD: ${{ secrets.R2_ENDPOINT }} + S3_DOWNLOAD: https://build-artifacts.altinity.io jobs: DockerBuildAarch64: diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index eb5f2072701a..f5cddb1cde1f 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -10,7 +10,7 @@ env: AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} # Point S3_URL to R2 endpoint S3_URL: ${{ secrets.R2_ENDPOINT }} - S3_DOWNLOAD: ${{ secrets.R2_ENDPOINT }} + S3_DOWNLOAD: https://build-artifacts.altinity.io CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} diff --git a/.github/workflows/reusable_build.yml b/.github/workflows/reusable_build.yml index fa06fdf8374c..a12b692cd5ed 100644 --- a/.github/workflows/reusable_build.yml +++ b/.github/workflows/reusable_build.yml @@ -7,7 +7,7 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} S3_URL: ${{ secrets.R2_ENDPOINT }} - S3_DOWNLOAD: ${{ secrets.R2_ENDPOINT }} + S3_DOWNLOAD: https://build-artifacts.altinity.io CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} diff --git a/.github/workflows/reusable_sign.yml b/.github/workflows/reusable_sign.yml index 88477bd298f2..20db4eeb48b6 100644 --- a/.github/workflows/reusable_sign.yml +++ b/.github/workflows/reusable_sign.yml @@ -60,7 +60,7 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} S3_URL: ${{ secrets.R2_ENDPOINT }} - S3_DOWNLOAD: ${{ secrets.R2_ENDPOINT }} + S3_DOWNLOAD: https://build-artifacts.altinity.io CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml index 708649a67309..6673f38ebcfa 100644 --- a/.github/workflows/reusable_test.yml +++ b/.github/workflows/reusable_test.yml @@ -73,7 +73,7 @@ env: AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} # Point S3_URL to R2 endpoint S3_URL: ${{ secrets.R2_ENDPOINT }} - S3_DOWNLOAD: ${{ secrets.R2_ENDPOINT }} + S3_DOWNLOAD: https://build-artifacts.altinity.io CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }} diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index 79dd2259fa09..a1a54b6431d4 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -361,7 +361,10 @@ def get_url(bucket: str, key: str) -> str: @staticmethod def s3_url(bucket: str, key: str) -> str: - url = f"{S3_DOWNLOAD}/{bucket}/{key}" + if 'r2.cloudflarestorage.com' in S3_URL: + url = f"{S3_DOWNLOAD}/{key}" + else: + url = f"{S3_DOWNLOAD}/{bucket}/{key}" # last two replacements are specifics of AWS urls: # https://jamesd3142.wordpress.com/2018/02/28/amazon-s3-and-the-plus-symbol/ url = url.replace("+", "%2B").replace(" ", "%20") From dcbb0f5d92ed9bdd59bb53b952806e1efb91b18a Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Mon, 7 Apr 2025 12:15:10 -0400 Subject: [PATCH 09/10] Remove manual ACL setting and remove 'PRs' prefix --- tests/ci/ci_metadata.py | 4 ++-- tests/ci/s3_helper.py | 1 - tests/performance/scripts/download.sh | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ci/ci_metadata.py b/tests/ci/ci_metadata.py index ce0358f5e76e..e3b04e93c618 100644 --- a/tests/ci/ci_metadata.py +++ b/tests/ci/ci_metadata.py @@ -45,7 +45,7 @@ def __init__( self.sync_pr_run_id = 0 if self.pr_number: - self.s3_path = f"{self._S3_PREFIX}/PRs/{self.pr_number}/" + self.s3_path = f"{self._S3_PREFIX}/{self.pr_number}/" else: self.s3_path = f"{self._S3_PREFIX}/{self.git_ref}/{self.sha}/" @@ -58,7 +58,7 @@ def __init__( and GITHUB_REPOSITORY != GITHUB_UPSTREAM_REPOSITORY ): self.upstream_pr_number = int(self.git_ref.split("/pr/", maxsplit=1)[1]) - self.s3_path_upstream = f"{self._S3_PREFIX}/PRs/{self.upstream_pr_number}/" + self.s3_path_upstream = f"{self._S3_PREFIX}/{self.upstream_pr_number}/" self._updated = False diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index a1a54b6431d4..ccfc849ef479 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -103,7 +103,6 @@ def _upload_file_to_s3( logging.info("Processing file without compression") logging.info("File is too large, do not provide content type") - metadata["ACL"] = "public-read" self.client.upload_file(file_path, bucket_name, s3_path, ExtraArgs=metadata) url = self.s3_url(bucket_name, s3_path) logging.info("Upload %s to %s Meta: %s", file_path, url, metadata) diff --git a/tests/performance/scripts/download.sh b/tests/performance/scripts/download.sh index 7a740a38fd2b..b0695c1bfe00 100755 --- a/tests/performance/scripts/download.sh +++ b/tests/performance/scripts/download.sh @@ -31,6 +31,7 @@ function download # Test all of them. declare -a urls_to_try=( "$S3_URL/PRs/$left_pr/$left_sha/$BUILD_NAME/performance.tar.zst" + "$S3_URL/$left_pr/$left_sha/$BUILD_NAME/performance.tar.zst" ) for path in "${urls_to_try[@]}" From 95c7e06264f7e338631bbf80b0c16d8012da7c25 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Mon, 19 May 2025 11:33:55 -0400 Subject: [PATCH 10/10] skip docker config for docker build jobs --- .github/workflows/reusable_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml index 53a551dbc72a..09629bf5f881 100644 --- a/.github/workflows/reusable_test.yml +++ b/.github/workflows/reusable_test.yml @@ -137,6 +137,7 @@ jobs: with: job_type: test - name: Docker setup + if: ${{ !contains(inputs.name, 'Docker') }} uses: ./.github/actions/docker_setup with: test_name: ${{ inputs.test_name }}