diff --git a/.github/workflows/docker_publish.yml b/.github/workflows/docker_publish.yml index 1e59aa8b5b8d..b7b5fcd0d940 100644 --- a/.github/workflows/docker_publish.yml +++ b/.github/workflows/docker_publish.yml @@ -69,9 +69,42 @@ jobs: echo "Pulling the image" docker pull $IMAGE + # Create container for version and symtable checks + echo "Creating container for checks..." + CONTAINER_ID=$(docker create $IMAGE $COMPONENT --version) + + # Check if binary has symbol table (not stripped) + echo "Checking if binary has symtable..." + # Try common binary locations + BINARY_PATH="" + for path in "/usr/bin/clickhouse" "/usr/bin/$COMPONENT"; do + if docker cp "$CONTAINER_ID:$path" /tmp/clickhouse-check 2>/dev/null; then + BINARY_PATH="$path" + break + fi + done + + if [ -z "$BINARY_PATH" ]; then + echo "✗ Could not find clickhouse binary in container" + docker rm "$CONTAINER_ID" + exit 1 + fi + + echo "Found binary at $BINARY_PATH" + if readelf -S /tmp/clickhouse-check | grep -q '\.symtab'; then + echo "✓ Binary has symtable" + else + echo "✗ Binary is missing symtable" + docker rm "$CONTAINER_ID" + rm -f /tmp/clickhouse-check + exit 1 + fi + rm -f /tmp/clickhouse-check + # Get version and clean it up echo "Getting version from image..." - VERSION_OUTPUT=$(docker run --rm $IMAGE $COMPONENT --version) + VERSION_OUTPUT=$(docker start -a "$CONTAINER_ID") + docker rm "$CONTAINER_ID" echo "Raw version output: $VERSION_OUTPUT" # Extract just the version number diff --git a/.github/workflows/sign_and_release.yml b/.github/workflows/sign_and_release.yml index c06cb4cba2b8..21ecd04576fb 100644 --- a/.github/workflows/sign_and_release.yml +++ b/.github/workflows/sign_and_release.yml @@ -102,8 +102,31 @@ jobs: echo "Running clickhouse binary..." ./clickhouse -q'q' - echo "Stripping the binary..." - strip clickhouse -o clickhouse-stripped + echo "Downloading clickhouse-stripped binary..." + if ! aws s3 cp "${SRC_URL}/${ARM_PATH}/clickhouse-stripped" clickhouse-stripped; then + echo "Failed to download clickhouse-stripped binary" + exit 1 + fi + + chmod +x clickhouse-stripped + + echo "Running clickhouse-stripped binary..." + ./clickhouse-stripped -q'q' + + echo "Downloading clickhouse-common-static tar..." + if ! aws s3 cp "${SRC_URL}/${ARM_PATH}/clickhouse-common-static-${{ inputs.package_version }}-arm64.tgz" clickhouse-common-static.tgz; then + echo "Failed to download clickhouse-common-static tar" + exit 1 + fi + + tar -xvf clickhouse-common-static.tgz + + if readelf -S "clickhouse-common-static-${{ inputs.package_version }}/usr/bin/clickhouse" | grep -q '\.symtab'; then + echo "✓ Binary has symtable" + else + echo "✗ Binary is missing symtable" + exit 1 + fi echo "Uploading processed binaries..." if ! aws s3 cp clickhouse "${SRC_URL}/${ARM_PATH}/arm64-bin/non-self-extracting/"; then @@ -114,6 +137,10 @@ jobs: echo "Failed to upload stripped clickhouse binary" exit 1 fi + if ! aws s3 cp "clickhouse-common-static-${{ inputs.package_version }}/usr/bin/clickhouse" "${SRC_URL}/${ARM_PATH}/arm64-bin/non-self-extracting/clickhouse-with-symtable"; then + echo "Failed to upload clickhouse-with-symtable binary" + exit 1 + fi copy-packages: needs: extract-package-info @@ -177,8 +204,32 @@ jobs: echo "Running clickhouse binary..." ./clickhouse -q'q' - echo "Stripping the binary..." - strip clickhouse -o clickhouse-stripped + echo "Downloading clickhouse-stripped binary..." + if ! aws s3 cp "${DEST_URL}/packages/AMD_PACKAGES/amd64-bin/clickhouse-stripped" clickhouse-stripped; then + echo "Failed to download clickhouse-stripped binary" + exit 1 + fi + + chmod +x clickhouse-stripped + + echo "Running clickhouse-stripped binary..." + ./clickhouse-stripped -q'q' + + + echo "Downloading clickhouse-common-static tar..." + if ! aws s3 cp "${DEST_URL}/packages/AMD_PACKAGES/clickhouse-common-static-${{ inputs.package_version }}-amd64.tgz" clickhouse-common-static.tgz; then + echo "Failed to download clickhouse-common-static tar" + exit 1 + fi + + tar -xvf clickhouse-common-static.tgz + + if readelf -S "clickhouse-common-static-${{ inputs.package_version }}/usr/bin/clickhouse" | grep -q '\.symtab'; then + echo "✓ Binary has symtable" + else + echo "✗ Binary is missing symtable" + exit 1 + fi echo "Uploading processed binaries..." if ! aws s3 cp clickhouse "${DEST_URL}/packages/AMD_PACKAGES/amd64-bin/non-self-extracting/"; then @@ -189,6 +240,10 @@ jobs: echo "Failed to upload stripped clickhouse binary" exit 1 fi + if ! aws s3 cp clickhouse-common-static-${{ inputs.package_version }}/usr/bin/clickhouse "${DEST_URL}/packages/AMD_PACKAGES/amd64-bin/non-self-extracting/clickhouse-with-symtable"; then + echo "Failed to upload stripped clickhouse binary" + exit 1 + fi copy-test-results: needs: extract-package-info @@ -259,6 +314,20 @@ jobs: repository: Altinity/ClickHouse path: ClickHouse + - name: Install required packages + run: | + echo "Installing required packages..." + sudo add-apt-repository -y universe + sudo apt-get update + sudo apt-get install -y software-properties-common python3-pip apt-utils gnupg rpm createrepo-c file wget + wget -q http://archive.ubuntu.com/ubuntu/pool/universe/d/dpkg-sig/dpkg-sig_0.13.1+nmu4_all.deb + sudo dpkg -i dpkg-sig_0.13.1+nmu4_all.deb || sudo apt-get install -f -y + rm dpkg-sig_0.13.1+nmu4_all.deb + sudo apt-add-repository --yes --update ppa:ansible/ansible + sudo apt-get install -y ansible + sudo ln -s /usr/bin/createrepo_c /usr/bin/createrepo + pip3 install boto3 botocore natsort --break-system-packages + - name: Download packages run: | if ! aws s3 sync "${DEST_URL}/packages/ARM_PACKAGES/" /home/runner/.cache/tmp/packages --exact-timestamps; then @@ -273,16 +342,6 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - name: Install required packages - run: | - echo "Installing required packages..." - sudo apt-get update - sudo apt-get install -y software-properties-common python3-pip dpkg-sig apt-utils gnupg rpm createrepo-c file - sudo apt-add-repository --yes --update ppa:ansible/ansible - sudo apt-get install -y ansible - sudo ln -s /usr/bin/createrepo_c /usr/bin/createrepo - pip3 install boto3 botocore - - name: Set up GPG passphrase run: | if [ "${RELEASE_ENVIRONMENT}" == "production" ]; then diff --git a/tests/ci/release/packaging/ansible/roles/update_bin_repo/tasks/main.yml b/tests/ci/release/packaging/ansible/roles/update_bin_repo/tasks/main.yml index b6844cffc750..998d496e397d 100644 --- a/tests/ci/release/packaging/ansible/roles/update_bin_repo/tasks/main.yml +++ b/tests/ci/release/packaging/ansible/roles/update_bin_repo/tasks/main.yml @@ -15,6 +15,7 @@ mv /home/runner/.cache/tmp/packages/{{ item }}-bin/clickhouse-stripped {{ local_repo_path }}/{{ repo_prefix }}bin-repo/{{ item }}/v{{ pkgver }}/self-extracting/clickhouse-stripped mv /home/runner/.cache/tmp/packages/{{ item }}-bin/non-self-extracting/clickhouse {{ local_repo_path }}/{{ repo_prefix }}bin-repo/{{ item }}/v{{ pkgver }}/non-self-extracting/clickhouse mv /home/runner/.cache/tmp/packages/{{ item }}-bin/non-self-extracting/clickhouse-stripped {{ local_repo_path }}/{{ repo_prefix }}bin-repo/{{ item }}/v{{ pkgver }}/non-self-extracting/clickhouse-stripped + mv /home/runner/.cache/tmp/packages/{{ item }}-bin/non-self-extracting/clickhouse-with-symtable {{ local_repo_path }}/{{ repo_prefix }}bin-repo/{{ item }}/v{{ pkgver }}/non-self-extracting/clickhouse-with-symtable loop: - amd64 - arm64 @@ -27,10 +28,12 @@ - amd64/v{{ pkgver }}/self-extracting/clickhouse-stripped - amd64/v{{ pkgver }}/non-self-extracting/clickhouse - amd64/v{{ pkgver }}/non-self-extracting/clickhouse-stripped + - amd64/v{{ pkgver }}/non-self-extracting/clickhouse-with-symtable - arm64/v{{ pkgver }}/self-extracting/clickhouse - arm64/v{{ pkgver }}/self-extracting/clickhouse-stripped - arm64/v{{ pkgver }}/non-self-extracting/clickhouse - arm64/v{{ pkgver }}/non-self-extracting/clickhouse-stripped + - arm64/v{{ pkgver }}/non-self-extracting/clickhouse-with-symtable when: (pkgver | regex_replace('^(\\d+).*$', '\\1')) is version('24', '>=') - name: Copy new binaries diff --git a/tests/ci/release/packaging/dirindex/dirindexgen.py b/tests/ci/release/packaging/dirindex/dirindexgen.py index 6b4f3332883d..27733a9e62d4 100755 --- a/tests/ci/release/packaging/dirindex/dirindexgen.py +++ b/tests/ci/release/packaging/dirindex/dirindexgen.py @@ -1,8 +1,10 @@ #!/bin/env python3 import argparse +import re import textwrap import boto3 +from natsort import natsorted def folder_list_add(folders, key, value): @@ -39,6 +41,16 @@ def folderjoin(items, slashes=False): return result +def is_bin_repo_arch_folder(folder): + """ + Check if the folder is a bin-repo architecture folder that should use natural sorting. + Matches patterns like: bin-repo/arm64, antalya-bin-repo/amd64, fips-bin-repo/arm64, etc. + """ + # Match folders ending with (bin-repo|antalya-bin-repo|fips-bin-repo|hotfix-bin-repo)/(arm64|amd64) + pattern = r'(bin-repo|antalya-bin-repo|fips-bin-repo|hotfix-bin-repo)/(arm64|amd64)$' + return bool(re.search(pattern, folder)) + + def main(): parser = argparse.ArgumentParser() parser.add_argument(dest='fqdn', help="Name of S3 bucket and domain name") @@ -98,7 +110,13 @@ def main(): parent_folder = folderjoin(folder.split("/")[:-1], slashes=True) indexdata.append(f'{parent_dir_str}' + f'{" ":{maxlen-len(parent_dir_str)}} {"-":20} {"-":20}\n') - for sub in subs: + # Apply natural sorting (descending) for bin-repo architecture folders + # to show newest versions first (e.g., v25.3 before v24.8 before v23.8) + if is_bin_repo_arch_folder(folder): + sorted_subs = list(reversed(natsorted(subs))) + else: + sorted_subs = sorted(subs) + for sub in sorted_subs: sub_path = folderjoin([folder, sub], slashes=True) indexdata.append(f'{sub}/{" ":{maxlen-len(sub)-1}}' + f' {"-":20} {"-":20}\n')