From f32c12c22075b0d78a358d0bb87ae3705b36c0d7 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 5 Feb 2026 12:10:49 -0500 Subject: [PATCH 1/5] reimplement broken tests handling for integration jobs --- ci/jobs/integration_test_job.py | 122 ++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/ci/jobs/integration_test_job.py b/ci/jobs/integration_test_job.py index 004f8f311191..9f1d3ac4eaa9 100644 --- a/ci/jobs/integration_test_job.py +++ b/ci/jobs/integration_test_job.py @@ -5,6 +5,9 @@ from pathlib import Path from typing import List, Tuple +import yaml # NOTE (strtgbb): Used for loading broken tests rules +import re + from ci.jobs.scripts.find_tests import Targeting from ci.jobs.scripts.integration_tests_configs import IMAGES_ENV, get_optimal_test_batch from ci.praktika.info import Info @@ -22,6 +25,106 @@ MAX_MEM_PER_WORKER = 11 +def get_broken_tests_rules(broken_tests_file_path: str) -> dict: + if ( + not os.path.isfile(broken_tests_file_path) + or os.path.getsize(broken_tests_file_path) == 0 + ): + raise ValueError( + "There is something wrong with getting broken tests rules: " + f"file '{broken_tests_file_path}' is empty or does not exist." + ) + + with open(broken_tests_file_path, "r", encoding="utf-8") as broken_tests_file: + broken_tests = yaml.safe_load(broken_tests_file) + + compiled_rules = {"exact": {}, "pattern": {}} + + for test in broken_tests: + regex = test.get("regex") is True + rule = { + "reason": test["reason"], + } + + if test.get("message"): + rule["message"] = re.compile(test["message"]) if regex else test["message"] + + if test.get("not_message"): + rule["not_message"] = ( + re.compile(test["not_message"]) if regex else test["not_message"] + ) + if test.get("check_types"): + rule["check_types"] = test["check_types"] + + if regex: + rule["regex"] = True + compiled_rules["pattern"][re.compile(test["name"])] = rule + else: + compiled_rules["exact"][test["name"]] = rule + + return compiled_rules + + +def test_is_known_fail(broken_tests_rules, test_name, test_logs, job_flags): + matching_rules = [] + + def matches_substring(substring, log, is_regex): + if log is None: + return False + if is_regex: + return bool(substring.search(log)) + return substring in log + + broken_tests_log = f"{temp_path}/broken_tests_handler.log" + + with open(broken_tests_log, "a") as log_file: + + log_file.write(f"Checking known broken tests for failed test: {test_name}\n") + log_file.write("Potential matching rules:\n") + exact_rule = broken_tests_rules["exact"].get(test_name) + if exact_rule: + log_file.write(f"{test_name} - {exact_rule}\n") + matching_rules.append(exact_rule) + + for name_re, data in broken_tests_rules["pattern"].items(): + if name_re.fullmatch(test_name): + log_file.write(f"{name_re} - {data}\n") + matching_rules.append(data) + + if not matching_rules: + return False + + log_file.write(f"First line of test logs: {test_logs.splitlines()[0]}\n") + + for rule_data in matching_rules: + if rule_data.get("check_types") and not any( + ct in job_flags for ct in rule_data["check_types"] + ): + log_file.write( + f"Skip rule: Check types didn't match: '{rule_data['check_types']}' not in '{job_flags}'\n" + ) + continue # check_types didn't match → skip rule + + is_regex = rule_data.get("regex", False) + not_message = rule_data.get("not_message") + if not_message and matches_substring(not_message, test_logs, is_regex): + log_file.write( + f"Skip rule: Not message matched: '{rule_data['not_message']}'\n" + ) + continue # not_message matched → skip rule + message = rule_data.get("message") + if message and not matches_substring(message, test_logs, is_regex): + log_file.write( + f"Skip rule: Message didn't match: '{rule_data['message']}'\n" + ) + continue + + log_file.write(f"Matched rule: {rule_data}\n") + return rule_data["reason"] + + return False + + def _start_docker_in_docker(): with open("./ci/tmp/docker-in-docker.log", "w") as log_file: dockerd_proc = subprocess.Popen( @@ -533,6 +636,25 @@ def main(): ) attached_files.append("./ci/tmp/dmesg.log") + broken_tests_rules = get_broken_tests_rules("tests/broken_tests.yaml") + for result in test_results: + if result.status == Result.StatusExtended.FAIL: + last_log_path = sorted([p for p in result.files if p.endswith(".log")])[-1] + with open(last_log_path, "r") as log_file: + log_content = log_file.read() + known_fail_reason = test_is_known_fail( + broken_tests_rules, + result.name, + log_content, + job_params, + ) + if known_fail_reason: + result.status = Result.StatusExtended.BROKEN + result.info += f"\nMarked as broken: {known_fail_reason}" + + if os.path.exists(f"{temp_path}/broken_tests_handler.log"): + attached_files.append(f"{temp_path}/broken_tests_handler.log") + R = Result.create_from(results=test_results, stopwatch=sw, files=attached_files) if has_error: From 096dbedd2ddfc44ba9e2646acef2bc4c6d046742 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 5 Feb 2026 12:30:16 -0500 Subject: [PATCH 2/5] fix docker push credentials --- ci/defs/job_configs.py | 2 +- ci/jobs/docker_server.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index ecfec61952d8..c22707a774f2 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -1045,7 +1045,7 @@ class JobConfigs: docker_keeper = Job.Config( name=JobNames.DOCKER_KEEPER, runs_on=RunnerLabels.STYLE_CHECK_AMD, - command="python3 ./ci/jobs/docker_server.py --tag-type head --allow-build-reuse", + command="python3 ./ci/jobs/docker_server.py --tag-type head --allow-build-reuse --push", digest_config=Job.CacheDigestConfig( include_paths=[ "./ci/jobs/docker_server.py", diff --git a/ci/jobs/docker_server.py b/ci/jobs/docker_server.py index 6e64e708b613..45f3f5775e11 100644 --- a/ci/jobs/docker_server.py +++ b/ci/jobs/docker_server.py @@ -58,10 +58,10 @@ def docker_login(relogin: bool = True) -> None: "docker system info | grep --quiet -E 'Username|Registry'" ): Shell.check( - "docker login --username 'robotclickhouse' --password-stdin", + "docker login --username 'altinityinfra' --password-stdin", strict=True, stdin_str=Secret.Config( - "dockerhub_robot_password", type=Secret.Type.AWS_SSM_PARAMETER + "DOCKER_PASSWORD", type=Secret.Type.GH_SECRET ).get_value(), encoding="utf-8", ) @@ -348,7 +348,7 @@ def main(): push = True image = DockerImageData(image_repo, image_path) - tags = [f'{info.pr_number}-{version_dict["string"]}'] + tags = [f'{info.pr_number}-{version_dict["describe"]}'] repo_urls = {} direct_urls: Dict[str, List[str]] = {} From 62a8c898f67cfbf53b738940ac1fe5e7c3794230 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 5 Feb 2026 18:02:51 -0500 Subject: [PATCH 3/5] add debugging --- ci/jobs/integration_test_job.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/ci/jobs/integration_test_job.py b/ci/jobs/integration_test_job.py index 9f1d3ac4eaa9..98790795ed7d 100644 --- a/ci/jobs/integration_test_job.py +++ b/ci/jobs/integration_test_job.py @@ -639,15 +639,27 @@ def main(): broken_tests_rules = get_broken_tests_rules("tests/broken_tests.yaml") for result in test_results: if result.status == Result.StatusExtended.FAIL: - last_log_path = sorted([p for p in result.files if p.endswith(".log")])[-1] - with open(last_log_path, "r") as log_file: - log_content = log_file.read() - known_fail_reason = test_is_known_fail( - broken_tests_rules, - result.name, - log_content, - job_params, - ) + try: + last_log_path = sorted([p for p in result.files if p.endswith(".log")])[ + -1 + ] + with open(last_log_path, "r") as log_file: + log_content = log_file.read() + except Exception as e: + print(f"Error getting last log path for result {result.name}: {e}") + print(f"Result files: {result.files}") + continue + try: + known_fail_reason = test_is_known_fail( + broken_tests_rules, + result.name, + log_content, + job_params, + ) + except Exception as e: + print(f"Error getting known fail reason for result {result.name}: {e}") + continue + if known_fail_reason: result.status = Result.StatusExtended.BROKEN result.info += f"\nMarked as broken: {known_fail_reason}" From 4d09ecbba88076fab19ebdd1912b1481e38a453e Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 5 Feb 2026 20:33:44 -0500 Subject: [PATCH 4/5] more debugging and fixes --- ci/jobs/integration_test_job.py | 10 ++++++++++ .../test_old_client_with_replicated_columns.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ci/jobs/integration_test_job.py b/ci/jobs/integration_test_job.py index 98790795ed7d..15ce157e5288 100644 --- a/ci/jobs/integration_test_job.py +++ b/ci/jobs/integration_test_job.py @@ -647,7 +647,17 @@ def main(): log_content = log_file.read() except Exception as e: print(f"Error getting last log path for result {result.name}: {e}") + print( + [ + a + for a in dir(result) + if not a.startswith("_") and not callable(getattr(result, a)) + ] + ) print(f"Result files: {result.files}") + print(f"Result info: {result.info}") + print(f"Error info: {error_info}") + continue try: known_fail_reason = test_is_known_fail( diff --git a/tests/integration/test_backward_compatibility/test_old_client_with_replicated_columns.py b/tests/integration/test_backward_compatibility/test_old_client_with_replicated_columns.py index 5e7d8779f382..a95928a6d1ca 100644 --- a/tests/integration/test_backward_compatibility/test_old_client_with_replicated_columns.py +++ b/tests/integration/test_backward_compatibility/test_old_client_with_replicated_columns.py @@ -11,7 +11,7 @@ ) backward = cluster.add_instance( "backward", - image="clickhouse/clickhouse-server", + image="altinityinfra/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, ) From a4f646053f83128bb365c9bd073e6b3c52852476 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 5 Feb 2026 21:09:49 -0500 Subject: [PATCH 5/5] fixes --- .github/workflows/master.yml | 2 +- .github/workflows/pull_request.yml | 2 +- .github/workflows/release_builds.yml | 2 +- ci/praktika/yaml_additional_templates.py | 2 +- .../test_old_client_with_replicated_columns.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 7719b6e85625..2f79519695ce 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -4660,7 +4660,7 @@ jobs: env: COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - VERSION: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }} + VERSION: ${{ fromJson(needs.config_workflow.outputs.data).JOB_KV_DATA.version.string }} steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 2deb0d327eb2..718e4c976e49 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -4833,7 +4833,7 @@ jobs: env: COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - VERSION: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }} + VERSION: ${{ fromJson(needs.config_workflow.outputs.data).JOB_KV_DATA.version.string }} steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 diff --git a/.github/workflows/release_builds.yml b/.github/workflows/release_builds.yml index 721bfc824eb3..802d9710fb78 100644 --- a/.github/workflows/release_builds.yml +++ b/.github/workflows/release_builds.yml @@ -1288,7 +1288,7 @@ jobs: env: COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - VERSION: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }} + VERSION: ${{ fromJson(needs.config_workflow.outputs.data).JOB_KV_DATA.version.string }} steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 diff --git a/ci/praktika/yaml_additional_templates.py b/ci/praktika/yaml_additional_templates.py index 97a24da3dc72..0f38df90bf00 100644 --- a/ci/praktika/yaml_additional_templates.py +++ b/ci/praktika/yaml_additional_templates.py @@ -132,7 +132,7 @@ class AltinityWorkflowTemplates: env: COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - VERSION: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }} + VERSION: ${{ fromJson(needs.config_workflow.outputs.data).JOB_KV_DATA.version.string }} steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 diff --git a/tests/integration/test_backward_compatibility/test_old_client_with_replicated_columns.py b/tests/integration/test_backward_compatibility/test_old_client_with_replicated_columns.py index a95928a6d1ca..45796c7ceefc 100644 --- a/tests/integration/test_backward_compatibility/test_old_client_with_replicated_columns.py +++ b/tests/integration/test_backward_compatibility/test_old_client_with_replicated_columns.py @@ -11,7 +11,7 @@ ) backward = cluster.add_instance( "backward", - image="altinityinfra/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, )