feat(build): support Linux aarch64 builds

.asf.yaml

@@ -59,17 +59,31 @@ github:
           app_slug: -1
         - name: "rat-license-check"
           app_slug: -1
-        - name: "asan-ubsan"
+        - name: "script-tests"
           app_slug: -1
-        - name: "tsan"
+        - name: "asan-ubsan-x86_64"
           app_slug: -1
-        - name: "clang-debug"
+        - name: "tsan-x86_64"
           app_slug: -1
-        - name: "clang-release"
+        - name: "clang-debug-x86_64"
           app_slug: -1
-        - name: "gcc-debug"
+        - name: "clang-release-x86_64"
           app_slug: -1
-        - name: "gcc-release"
+        - name: "gcc-debug-x86_64"
+          app_slug: -1
+        - name: "gcc-release-x86_64"
+          app_slug: -1
+        - name: "gcc-debug-aarch64"
+          app_slug: -1
+        - name: "gcc-release-aarch64"
+          app_slug: -1
+        - name: "clang-debug-aarch64"
+          app_slug: -1
+        - name: "asan-ubsan-aarch64"
+          app_slug: -1
+        - name: "clang-release-aarch64"
+          app_slug: -1
+        - name: "tsan-aarch64"
           app_slug: -1
         - name: "gcc8-test"
           app_slug: -1

.github/workflows/build_and_test.yaml

@@ -33,31 +33,69 @@ permissions:
   contents: read
 
 jobs:
+  script-tests:
+    name: script-tests
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    steps:
+      - name: Checkout paimon-cpp
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - name: Run CMake module tests
+        shell: bash
+        run: ci/scripts/test_cmake_modules.sh
+      - name: Run packaging argument tests
+        shell: bash
+        run: ci/scripts/test_packaging_args.sh
+      - name: Run asan_symbolize tests
+        shell: bash
+        run: ci/scripts/test_asan_symbolize.sh
+
   build-and-test:
     name: ${{ matrix.name }}
-    runs-on: ubuntu-24.04
+    runs-on: ${{ matrix.runner || 'ubuntu-24.04' }}
     timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
         include:
-          - name: gcc-release
+          - name: gcc-release-x86_64
             cc: gcc-14
             cxx: g++-14
             build_args: --build_type Release
-          - name: clang-release
+          - name: clang-release-x86_64
             build_args: --build_type Release
-          - name: gcc-debug
+          - name: gcc-debug-x86_64
             cc: gcc-14
             cxx: g++-14
-          - name: clang-debug
+          - name: clang-debug-x86_64
             fetch_depth: '0' # fetch the PR target branch history for clang-tidy
             build_args: >-
               --check_clang_tidy
               --lint_git_target_commit "origin/${{ github.base_ref || github.event.repository.default_branch }}"
-          - name: asan-ubsan
+          - name: asan-ubsan-x86_64
             build_args: --enable_asan --enable_ubsan
-          - name: tsan
+          - name: tsan-x86_64
+            skip_rust: true
+            build_args: --enable_tsan
+          - name: gcc-debug-aarch64
+            runner: ubuntu-24.04-arm
+            cc: gcc-14
+            cxx: g++-14
+          - name: gcc-release-aarch64
+            runner: ubuntu-24.04-arm
+            cc: gcc-14
+            cxx: g++-14
+            build_args: --build_type Release
+          - name: clang-debug-aarch64
+            runner: ubuntu-24.04-arm
+          - name: asan-ubsan-aarch64
+            runner: ubuntu-24.04-arm
+            build_args: --enable_asan --enable_ubsan
+          - name: clang-release-aarch64
+            runner: ubuntu-24.04-arm
+            build_args: --build_type Release
+          - name: tsan-aarch64
+            runner: ubuntu-24.04-arm
             skip_rust: true
             build_args: --enable_tsan
     steps:

.github/workflows/release_candidate.yaml

@@ -99,21 +99,26 @@ jobs:
     name: Verify source archive (${{ matrix.compiler }})
     if: github.ref_type == 'tag'
     needs: archive
-    runs-on: ubuntu-24.04
+    runs-on: ${{ matrix.runner || 'ubuntu-24.04' }}
     timeout-minutes: 180
     strategy:
       fail-fast: false
       matrix:
         compiler:
           - gcc-14
           - clang
+          - gcc-14-aarch64
         include:
           - compiler: gcc-14
             cc: gcc-14
             cxx: g++-14
           - compiler: clang
             cc: clang
             cxx: clang++
+          - compiler: gcc-14-aarch64
+            cc: gcc-14
+            cxx: g++-14
+            runner: ubuntu-24.04-arm
     steps:
       - name: Checkout source
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

README.md

@@ -49,7 +49,8 @@ Paimon C++ currently provides:
 > Paimon C++ therefore currently treats the `bitmap` global index type as unsupported. The legacy
 > implementation remains in the codebase pending migration to the Java-compatible format.
 
-Note: Only Linux x86_64 builds are currently supported and verified.
+Note: Linux `x86_64` and `aarch64` builds are supported and verified in CI. See the supported
+platform matrix in [docs/source/building.rst](docs/source/building.rst) for other platforms.
 
 ## Building
 

build_and_package.sh

@@ -26,6 +26,8 @@ MAKE_CLEAN=false
 PACKAGE=false
 CMAKE_OPTIONS=()
 JOBS=""
+PACKAGE_PLATFORM=""
+PRINT_NAME=false
 
 show_help() {
     cat << EOF
@@ -37,6 +39,12 @@ Options:
   -c, --clean       Clean build directory before building
   -p, --package     Package creation
   -j, --jobs <num>  Number of parallel jobs for building (default: auto-detect)
+      --platform <label>
+                    Platform label used in the package name (default: the host,
+                    e.g. linux-aarch64). This only labels the artifact; it does
+                    not configure a cross build. Must match
+                    [A-Za-z0-9][A-Za-z0-9._-]*, i.e. a single path component.
+      --print-name  Print the resolved package name and exit
   -h, --help        Show this help message
 
 CMake Options:
@@ -46,6 +54,7 @@ CMake Options:
 Examples:
   $0 -r -p -j 8 -DPAIMON_BUILD_SHARED=ON -DPAIMON_BUILD_STATIC=OFF
   $0 --debug --clean --package --jobs 4
+  $0 -r -p --platform linux-aarch64
 
 EOF
 }
@@ -82,6 +91,29 @@ while [[ $# -gt 0 ]]; do
                 exit 1
             fi
             ;;
+        --platform=*)
+            # Normalize the equals form onto the two-argument form below, so the
+            # value goes through the same validation.
+            set -- --platform "${1#*=}" "${@:2}"
+            ;;
+        --platform)
+            shift
+            # The label becomes a path component of both the install prefix and
+            # the tarball name, so it must stay a single safe component: the
+            # pattern admits no separator and no shell metacharacter. Requiring a
+            # leading letter or digit additionally stops a following option from
+            # being swallowed as the value.
+            if [[ $# -eq 0 || ! $1 =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then
+                echo "Error: --platform requires a label matching [A-Za-z0-9][A-Za-z0-9._-]*" >&2
+                exit 1
+            fi
+            PACKAGE_PLATFORM="$1"
+            shift
+            ;;
+        --print-name)
+            PRINT_NAME=true
+            shift
+            ;;
         -h|--help)
             show_help
             exit 0
@@ -94,8 +126,27 @@ while [[ $# -gt 0 ]]; do
     esac
 done
 
+# Defaults to the host. Note this only names the artifact: the bundled third-party
+# builds receive just CMAKE_C_COMPILER / CMAKE_CXX_COMPILER (see
+# EP_COMMON_TOOLCHAIN in cmake_modules/ThirdpartyToolchain.cmake) and no toolchain
+# file, sysroot or find-root, so a cross build is not wired up end to end.
+if [ -z "$PACKAGE_PLATFORM" ]; then
+    HOST_OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+    if [ "$HOST_OS" = "darwin" ]; then
+        HOST_OS="macos"
+    fi
+    PACKAGE_PLATFORM="$HOST_OS-$(uname -m)"
+fi
+BUILD_NAME="$BUILD_NAME-$PACKAGE_PLATFORM"
+
+if [ "$PRINT_NAME" = true ]; then
+    echo "$BUILD_NAME"
+    exit 0
+fi
+
 echo "========== Build Configuration =========="
 echo "Build Type: $BUILD_TYPE"
+echo "Package Platform: $PACKAGE_PLATFORM"
 echo "Package Name: $BUILD_NAME"
 echo "Clean Build: $MAKE_CLEAN"
 echo "Package: $PACKAGE"

build_support/asan_symbolize.py

@@ -8,6 +8,7 @@
 #
 #===------------------------------------------------------------------------===#
 import bisect
+import io
 import os
 import re
 import subprocess
@@ -333,8 +334,17 @@ def process_stdin(self):
     if sys.version_info[0] == 2:
       sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
     else:
-      # Unbuffered output is not supported in Python 3
-      sys.stdout = os.fdopen(sys.stdout.fileno(), 'w')
+      # Test output is not guaranteed to be valid UTF-8, so pin both streams to the same
+      # encoding and round such bytes through surrogateescape, which passes them out
+      # unchanged instead of raising and truncating the rest of the test log. Both the
+      # encoding and the error handler have to be set on both streams: taking either from
+      # the locale or from PYTHONIOENCODING would let them disagree and re-encode the bytes.
+      # The buffers are rewrapped rather than reconfigured, which needs Python 3.7, and
+      # line buffering stands in for the unbuffered output Python 3 does not support.
+      sys.stdin = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8',
+                                   errors='surrogateescape')
+      sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8',
+                                    errors='surrogateescape', line_buffering=True)
 
     while True:
       line = sys.stdin.readline()

ci/scripts/build_paimon.sh

@@ -136,6 +136,12 @@ fi
 if [[ "${enable_tsan}" == "true" ]]; then
     ENABLE_TANTIVY="OFF" # Tantivy's Rust library is not TSAN-instrumented.
 fi
+# CI always builds natively, so the host architecture is the target architecture.
+host_arch=$(uname -m)
+if [[ "${host_arch}" != "x86_64" ]]; then
+    ENABLE_LUMINA="OFF"
+    echo "=== Lumina disabled: no prebuilt artifacts for ${host_arch} ==="
+fi
 
 CMAKE_ARGS=(
     "-G Ninja"

ci/scripts/test_asan_symbolize.sh

@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Tests that build_support/asan_symbolize.py, which every test binary pipes its
+# output through, passes bytes that are not valid UTF-8 straight through. A test
+# that fails while printing such a byte used to kill the symbolizer, and with it
+# the rest of the test log.
+
+set -euo pipefail
+
+source_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+symbolizer="${source_dir}/build_support/asan_symbolize.py"
+python=${PYTHON:-python3}
+# The vendored script emits SyntaxWarnings for its own regexes; they are unrelated noise here.
+export PYTHONWARNINGS=ignore
+
+status=0
+
+# Returns non-zero on failure as well as recording it, so that a caller running it in a
+# subshell, where the recorded status would not propagate, can still see the result.
+check() {
+    local name=$1 input=$2 expected=$3 actual_hex expected_hex result=0
+    echo "=== ${name} ==="
+    # Compare hex dumps so that a mismatch is readable and the shell does not
+    # mangle the bytes on the way. A failing symbolizer is a failed check, not a
+    # reason to abort the script, so the remaining checks still run.
+    if ! actual_hex=$(printf '%b' "${input}" | "${python}" "${symbolizer}" | od -An -tx1 |
+        tr -d ' \n'); then
+        echo "symbolizer exited non-zero"
+        status=1
+        result=1
+    fi
+    expected_hex=$(printf '%b' "${expected}" | od -An -tx1 | tr -d ' \n')
+    if [[ "${actual_hex}" != "${expected_hex}" ]]; then
+        echo "expected: ${expected_hex}"
+        echo "actual:   ${actual_hex}"
+        status=1
+        result=1
+    fi
+    return "${result}"
+}
+
+# A lone 0xff is what a failing TINYINT literal assertion prints. Every byte,
+# valid UTF-8 or not, has to come out unchanged.
+# `|| status=1` keeps `set -e` from aborting on the first failed check.
+check "invalid utf-8 round trips" 'ok\n\xff\xfe binary\nplain\n' \
+    'ok\n\xff\xfe binary\nplain\n' || status=1
+
+# Valid multi byte UTF-8 must not be damaged either: test names contain it.
+check "utf-8 round trips" '\xe4\xb8\xad\xe6\x96\x87\n' '\xe4\xb8\xad\xe6\x96\x87\n' || status=1
+
+# The bytes have to survive whatever the environment asks Python to use: an encoding taken
+# from PYTHONIOENCODING on one stream and from the locale on the other would re-encode them.
+(
+    export PYTHONIOENCODING=latin-1
+    check "invalid utf-8 round trips under PYTHONIOENCODING" 'ok\n\xff\xfe\n' \
+        'ok\n\xff\xfe\n' || status=1
+    # status is set in this subshell and does not reach the caller, so report it as the exit
+    # code, which stays correct if another check is added here.
+    exit "${status}"
+) || status=1
+
+# An invalid byte must not stop the lines that follow it from being processed:
+# the stack frame below is still rewritten by the symbolizer. Its stderr is dropped
+# because addr2line reports the fake binary path there even when the check passes.
+echo "=== keeps processing after an invalid byte ==="
+if ! output=$(printf '%b' '\xff\n    #0 0x7f6e35cf2e45  (/blah/foo.so+0x11fe45)\ntail\n' |
+    "${python}" "${symbolizer}" 2>/dev/null); then
+    echo "symbolizer exited non-zero"
+    status=1
+fi
+# "#0" and "tail" alone would also match unprocessed input, so assert that the frame was
+# actually rewritten: the symbolized form gains " in" and loses the raw binary path.
+if [[ "${output}" != *"#0"* || "${output}" != *"tail"* || "${output}" != *" in"* ||
+    "${output}" == *"/blah/foo.so"* ]]; then
+    echo "frame was not rewritten after the invalid byte, got:"
+    echo "${output}"
+    status=1
+fi
+
+exit "${status}"