diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml deleted file mode 100644 index 36e8e9b6de02..000000000000 --- a/.github/workflows/cpp.yml +++ /dev/null @@ -1,428 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: C++ - -on: - push: - branches: - - '**' - - '!dependabot/**' - tags: - - '**' - paths: - - '.dockerignore' - - '.github/workflows/cpp.yml' - - '.github/workflows/cpp_windows.yml' - - 'ci/conda_env_*' - - 'ci/docker/**' - - 'ci/scripts/ccache_setup.sh' - - 'ci/scripts/cpp_*' - - 'ci/scripts/install_azurite.sh' - - 'ci/scripts/install_gcs_testbench.sh' - - 'ci/scripts/install_minio.sh' - - 'ci/scripts/msys2_*' - - 'ci/scripts/util_*' - - 'cpp/**' - - 'compose.yaml' - - 'format/Flight.proto' - - 'testing' - pull_request: - paths: - - '.dockerignore' - - '.github/workflows/cpp.yml' - - '.github/workflows/cpp_windows.yml' - - 'ci/conda_env_*' - - 'ci/docker/**' - - 'ci/scripts/ccache_setup.sh' - - 'ci/scripts/cpp_*' - - 'ci/scripts/install_azurite.sh' - - 'ci/scripts/install_gcs_testbench.sh' - - 'ci/scripts/install_minio.sh' - - 'ci/scripts/msys2_*' - - 'ci/scripts/util_*' - - 'cpp/**' - - 'compose.yaml' - - 'format/Flight.proto' - - 'testing' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - actions: read - contents: read - -env: - ARCHERY_DEBUG: 1 - ARROW_ENABLE_TIMING_TESTS: OFF - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - docker: - name: ${{ matrix.title }} - runs-on: ${{ matrix.runs-on }} - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - include: - - arch: amd64 - clang-tools: 18 - image: conda-cpp - llvm: 18 - runs-on: ubuntu-latest - simd-level: AVX2 - title: AMD64 Conda C++ AVX2 - ubuntu: 22.04 - - arch: amd64 - clang-tools: 18 - image: ubuntu-cpp-sanitizer - llvm: 18 - runs-on: ubuntu-latest - title: AMD64 Ubuntu 24.04 C++ ASAN UBSAN - ubuntu: 24.04 - - arch: arm64/v8 - clang-tools: 18 - image: ubuntu-cpp - llvm: 18 - runs-on: ubuntu-24.04-arm - title: ARM64 Ubuntu 22.04 C++ - ubuntu: 22.04 - env: - ARCH: ${{ matrix.arch }} - ARROW_SIMD_LEVEL: ${{ matrix.simd-level }} - CLANG_TOOLS: ${{ matrix.clang-tools }} - LLVM: ${{ matrix.llvm }} - UBUNTU: ${{ matrix.ubuntu }} - steps: - - name: Show machine info - run: | - free -h - lscpu - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Restore Docker Volumes - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: .docker - key: cpp-${{ matrix.image }} - - name: Setup Python on hosted runner - if: | - matrix.runs-on == 'ubuntu-latest' - uses: actions/setup-python@v7 - with: - python-version: 3 - - name: Setup Python on self-hosted runner - if: | - contains(matrix.runs-on, 'self-hosted') - run: | - sudo apt update - sudo apt install -y --no-install-recommends python3 python3-dev python3-pip - python3 -m pip install -U pip - - name: Setup Archery - run: python3 -m pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes - sudo sysctl -w vm.mmap_rnd_bits=28 - source ci/scripts/util_enable_core_dumps.sh - archery docker run ${{ matrix.image }} - - name: Save Docker Volumes - if: ${{ !cancelled() }} - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: .docker - key: cpp-${{ matrix.image }} - include-hidden-files: true - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - build-example: - name: C++ Minimal Build Example - runs-on: ubuntu-24.04 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 45 - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Check CMake presets - run: | - cd cpp - cmake --list-presets - - name: Run minimal example - run: | - cd cpp/examples/minimal_build - docker compose run --rm minimal - - macos: - name: ${{ matrix.architecture }} macOS ${{ matrix.macos-version }} C++ - runs-on: macos-${{ matrix.macos-version }} - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - include: - - architecture: AMD64 - macos-version: "15-intel" - - architecture: ARM64 - macos-version: "14" - env: - ARROW_AZURE: ON - ARROW_BUILD_TESTS: ON - ARROW_DATASET: ON - ARROW_FLIGHT: ON - ARROW_GANDIVA: ON - ARROW_GCS: ON - ARROW_HDFS: ON - ARROW_HOME: /tmp/local - ARROW_JEMALLOC: ON - ARROW_ORC: ON - ARROW_PARQUET: ON - ARROW_S3: ON - ARROW_SUBSTRAIT: ON - ARROW_WITH_BROTLI: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_LZ4: ON - # GH-36013 disabling opentelemetry here because we can't - # get the patched version from conda - # ARROW_WITH_OPENTELEMETRY: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - steps: - - name: CPU Info - run: | - sysctl -a | grep cpu - sysctl -a | grep "hw.optional" - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Install Dependencies - run: | - brew bundle --file=cpp/Brewfile - - name: Install MinIO - run: | - $(brew --prefix bash)/bin/bash \ - ci/scripts/install_minio.sh latest ${ARROW_HOME} - - name: Set up Python - uses: actions/setup-python@v7 - with: - python-version: 3.12 - - name: Install Google Cloud Storage Testbench - run: ci/scripts/install_gcs_testbench.sh default - - name: Install Azurite Storage Emulator - run: ci/scripts/install_azurite.sh - - name: Setup ccache - run: | - ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - run: | - echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Restore ccache - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: cpp-ccache-macos-${{ matrix.macos-version }} - - name: Build - run: | - if [ "${{ matrix.macos-version }}" = "15-intel" ]; then - # This is a workaround. - # - # Homebrew uses /usr/local as prefix. So packages - # installed by Homebrew also use /usr/local/include. We - # want to include headers for packages installed by - # Homebrew as system headers to ignore warnings in them. - # But "-isystem /usr/local/include" isn't used by CMake - # because /usr/local/include is marked as the default - # include path. So we disable -Werror to avoid build error - # by warnings from packages installed by Homebrew. - export BUILD_WARNING_LEVEL=PRODUCTION - fi - ci/scripts/cpp_build.sh $(pwd) $(pwd)/build - - name: Save ccache - if: ${{ !cancelled() }} - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: cpp-ccache-macos-${{ matrix.macos-version }} - - name: Test - shell: bash - run: | - # MinIO is installed on ${ARROW_HOME}/bin - export PATH="${ARROW_HOME}/bin:${PATH}" - sudo sysctl -w kern.coredump=1 - sudo sysctl -w kern.corefile=/tmp/core.%N.%P - ulimit -c unlimited # must enable within the same shell - ci/scripts/cpp_test.sh $(pwd) $(pwd)/build - - windows: - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - name: AMD64 Windows 2022 AVX2 C++ - uses: ./.github/workflows/cpp_windows.yml - with: - arch: x64 - os: windows-2022 - simd-level: AVX2 - - windows-mingw: - name: AMD64 Windows MinGW ${{ matrix.msystem_upper }} C++ - runs-on: windows-2022 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - # Build may take 1h+ without cache. - timeout-minutes: 120 - strategy: - fail-fast: false - matrix: - include: - - msystem_lower: mingw64 - msystem_upper: MINGW64 - - msystem_lower: clang64 - msystem_upper: CLANG64 - env: - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: ON - ARROW_BUILD_TYPE: release - ARROW_DATASET: ON - ARROW_FLIGHT: ON - ARROW_FLIGHT_SQL: ON - ARROW_FLIGHT_SQL_ODBC: OFF - ARROW_GANDIVA: ON - ARROW_GCS: ON - ARROW_HDFS: OFF - ARROW_HOME: /${{ matrix.msystem_lower}} - ARROW_JEMALLOC: OFF - ARROW_PARQUET: ON - ARROW_S3: ON - ARROW_SUBSTRAIT: ON - ARROW_USE_GLOG: OFF - ARROW_VERBOSE_THIRDPARTY_BUILD: OFF - ARROW_WITH_BROTLI: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_OPENTELEMETRY: OFF - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - ARROW_CMAKE_ARGS: >- - -DARROW_PACKAGE_PREFIX=/${{ matrix.msystem_lower}} - -DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON - # We can't use unity build because we don't have enough memory on - # GitHub Actions. - # CMAKE_UNITY_BUILD: ON - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - uses: msys2/setup-msys2@v2 - with: - msystem: ${{ matrix.msystem_upper }} - update: true - - name: Setup MSYS2 - shell: msys2 {0} - run: ci/scripts/msys2_setup.sh cpp - - name: Restore ccache - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ccache - key: cpp-ccache-${{ matrix.msystem_lower}} - - name: Build - shell: msys2 {0} - run: | - export CMAKE_BUILD_PARALLEL_LEVEL=$NUMBER_OF_PROCESSORS - ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build" - - name: Save ccache - if: ${{ !cancelled() }} - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ccache - key: cpp-ccache-${{ matrix.msystem_lower}} - - name: Download Timezone Database - if: matrix.msystem_upper == 'CLANG64' - shell: bash - run: | - # TODO(GH-48593): msys2 clang64 uses libc++ and vendored date.h library - # which needs tzdata database to build Arrow with time zone support. - # https://github.com/apache/arrow/issues/48593 - ci/scripts/download_tz_database.sh - - name: Download MinIO - # Match the version pinned in ci/scripts/install_minio.sh. - shell: msys2 {0} - run: | - mkdir -p /usr/local/bin - wget \ - --output-document /usr/local/bin/minio.exe \ - https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2025-01-20T14-49-07Z - chmod +x /usr/local/bin/minio.exe - - name: Set up Python - uses: actions/setup-python@v7 - id: python-install - with: - python-version: '3.12' - - name: Install Google Cloud Storage Testbench - shell: msys2 {0} - env: - PIPX_BIN_DIR: /usr/local/bin - PIPX_BASE_PYTHON: ${{ steps.python-install.outputs.python-path }} - run: | - ci/scripts/install_gcs_testbench.sh default - - name: Test - shell: msys2 {0} - run: | - ci/scripts/cpp_test.sh "$(pwd)" "$(pwd)/build" diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml deleted file mode 100644 index f0fd53bc1a83..000000000000 --- a/.github/workflows/cpp_extra.yml +++ /dev/null @@ -1,847 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: C++ Extra - -on: - push: - branches: - - '**' - - '!dependabot/**' - paths: - - '.dockerignore' - - '.github/workflows/check_labels.yml' - - '.github/workflows/cpp_extra.yml' - - '.github/workflows/cpp_windows.yml' - - '.github/workflows/report_ci.yml' - - 'ci/conda_env_*' - - 'ci/docker/**' - - 'ci/scripts/ccache_setup.sh' - - 'ci/scripts/cpp_*' - - 'ci/scripts/install_azurite.sh' - - 'ci/scripts/install_gcs_testbench.sh' - - 'ci/scripts/install_minio.sh' - - 'ci/scripts/msys2_*' - - 'ci/scripts/util_*' - - 'cpp/**' - - 'compose.yaml' - - 'dev/archery/archery/**' - - 'format/Flight.proto' - - 'testing' - tags: - - '**' - pull_request: - # This trigger ensures that the `check_labels` workflow runs on manual label changes - types: - - labeled - - opened - - reopened - - synchronize - schedule: - - cron: | - 0 0 * * * - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - actions: read - contents: read - pull-requests: read - -env: - ARCHERY_DEBUG: 1 - ARROW_ENABLE_TIMING_TESTS: OFF - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - check-labels: - uses: ./.github/workflows/check_labels.yml - with: - parent-workflow: cpp_extra - - check-enabled: - # Check whether the CI builds in this workflow need to be run - needs: check-labels - runs-on: ubuntu-latest - outputs: - is_enabled: ${{ steps.set_enabled.outputs.is_enabled }} - steps: - - id: set_enabled - if: >- - github.ref_type == 'tag' || - (github.event_name == 'schedule' && github.repository == 'apache/arrow') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') - run: echo "is_enabled=true" >> "$GITHUB_OUTPUT" - - docker: - needs: check-enabled - if: needs.check-enabled.outputs.is_enabled == 'true' - name: ${{ matrix.title }} - runs-on: ${{ matrix.runs-on }} - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - include: - - image: alpine-linux-cpp - runs-on: ubuntu-latest - title: AMD64 Alpine Linux - - image: ubuntu-cpp - run-options: >- - -e ARROW_CTEST_TIMEOUT=2000 - -e ARROW_C_FLAGS_DEBUG="-O1" - -e ARROW_CXX_FLAGS_DEBUG="-O1" - -e ARROW_GANDIVA=OFF - -e ARROW_LARGE_MEMORY_TESTS=ON - -e BUILD_WARNING_LEVEL=PRODUCTION - runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/volume=80gb/spot=capacity-optimized" - title: AMD64 Ubuntu Large Memory Tests - - image: conda-cpp - run-options: >- - -e ARROW_USE_MESON=ON - runs-on: ubuntu-latest - title: AMD64 Ubuntu Meson - # TODO: We should remove this "continue-on-error: true" once GH-47207 is resolved - - continue-on-error: true - envs: - - DEBIAN=13 - image: debian-cpp - run-options: >- - -e CMAKE_CXX_STANDARD=23 - runs-on: ubuntu-latest - title: AMD64 Debian C++23 - - image: debian-cpp - run-options: >- - -e ARROW_SIMD_LEVEL=NONE - -e ARROW_CXXFLAGS=-march=x86-64 - runs-on: ubuntu-latest - title: AMD64 Debian SIMD Level NONE - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Restore Docker Volumes - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: .docker - key: cpp-extra-${{ matrix.image }} - - name: Setup Python - uses: actions/setup-python@v7 - with: - python-version: 3 - - name: Setup Archery - run: python3 -m pip install -e dev/archery[docker] - - name: Execute Docker Build - continue-on-error: ${{ matrix.continue-on-error || false }} - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - ENVS: ${{ toJSON(matrix.envs) }} - run: | - # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes - sudo sysctl -w vm.mmap_rnd_bits=28 - source ci/scripts/util_enable_core_dumps.sh - if [ "${ENVS}" != "null" ]; then - echo "${ENVS}" | jq -r '.[]' | while read env; do - echo "${env}" >> .env - done - fi - archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }} - - name: Save Docker Volumes - if: ${{ !cancelled() }} - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: .docker - key: cpp-extra-${{ matrix.image }} - include-hidden-files: true - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - msvc-arm64: - needs: check-enabled - if: needs.check-enabled.outputs.is_enabled == 'true' - name: ARM64 Windows 11 MSVC - uses: ./.github/workflows/cpp_windows.yml - with: - arch: arm64 - os: windows-11-arm - simd-level: NONE - - jni-linux: - needs: check-enabled - if: needs.check-enabled.outputs.is_enabled == 'true' - name: JNI ${{ matrix.platform.runs-on }} ${{ matrix.platform.arch }} - runs-on: ${{ matrix.platform.runs-on }} - timeout-minutes: 240 - permissions: - actions: read - contents: read - # This is for using GitHub Packages for vcpkg cache - packages: write - strategy: - fail-fast: false - matrix: - platform: - - arch: "amd64" - runs-on: ubuntu-latest - - arch: "arm64/v8" - runs-on: ubuntu-24.04-arm - env: - ARCH: ${{ matrix.platform.arch }} - REPO: ghcr.io/${{ github.repository }}-dev - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Free up disk space - run: | - ci/scripts/util_free_space.sh - - name: Restore Docker Volumes - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: .docker - key: jni-${{ matrix.platform.runs-on }} - - name: Setup Python - uses: actions/setup-python@v7 - with: - python-version: 3 - - name: Setup Archery - run: python3 -m pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ github.actor }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite" - run: | - source ci/scripts/util_enable_core_dumps.sh - archery docker run cpp-jni - - name: Save Docker Volumes - if: ${{ !cancelled() }} - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: .docker - key: jni-${{ matrix.platform.runs-on }} - include-hidden-files: true - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ github.actor }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - continue-on-error: true - run: archery docker push cpp-jni - - jni-macos: - needs: check-enabled - if: needs.check-enabled.outputs.is_enabled == 'true' - name: JNI macOS - runs-on: macos-14 - timeout-minutes: 45 - env: - MACOSX_DEPLOYMENT_TARGET: "14.0" - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Install dependencies - run: | - brew bundle --file=cpp/Brewfile - # We want to link aws-sdk-cpp statically but Homebrew's - # aws-sdk-cpp provides only shared library. If we have - # Homebrew's aws-sdk-cpp, our build mix Homebrew's - # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's - # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. - brew uninstall aws-sdk-cpp - # We want to use bundled RE2 for static linking. If - # Homebrew's RE2 is installed, its header file may be used. - # We uninstall Homebrew's RE2 to ensure using bundled RE2. - brew uninstall grpc || : # gRPC depends on RE2 - brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too - brew uninstall re2 - # We want to use bundled Protobuf for static linking. If - # Homebrew's Protobuf is installed, its library file may be - # used on test We uninstall Homebrew's Protobuf to ensure using - # bundled Protobuf. - brew uninstall protobuf - - name: Prepare ccache - run: | - echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - - name: Restore ccache - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ccache - key: jni-macos - - name: CMake - run: | - cmake \ - -S cpp \ - -B cpp.build \ - --preset=ninja-release-jni-macos \ - -DARROW_BUILD_TESTS=ON \ - -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install - - name: Build - run: | - cmake --build cpp.build - - name: Install - run: | - cmake --install cpp.build - - name: Test - env: - ARROW_TEST_DATA: ${{ github.workspace }}/testing/data - PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data - run: | - # unstable - exclude_tests="arrow-acero-asof-join-node-test" - exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" - ctest \ - --exclude-regex "${exclude_tests}" \ - --label-regex unittest \ - --output-on-failure \ - --parallel "$(sysctl -n hw.ncpu)" \ - --test-dir "cpp.build" \ - --timeout 300 - - name: Build example - run: | - cmake \ - -S cpp/examples/minimal_build/ \ - -B cpp/examples/minimal_build.build \ - -GNinja \ - -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install - cmake --build cpp/examples/minimal_build.build - cd cpp/examples/minimal_build - ../minimal_build.build/arrow-example - - name: Save ccache - if: ${{ !cancelled() }} - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ccache - key: jni-macos - - jni-windows: - needs: check-enabled - if: needs.check-enabled.outputs.is_enabled == 'true' - name: JNI Windows - runs-on: windows-2022 - timeout-minutes: 240 - - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - - name: Install msys2 (for tzdata for ORC tests) - uses: msys2/setup-msys2@v2 - id: setup-msys2 - - - name: Install cmake - shell: bash - run: | - ci/scripts/install_cmake.sh 4.1.2 /usr - - - name: Install ccache - shell: bash - run: | - ci/scripts/install_ccache.sh 4.13.6 /usr - - - name: Setup ccache - shell: bash - run: | - ci/scripts/ccache_setup.sh - - - name: ccache info - id: ccache-info - shell: bash - run: | - echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - - name: Restore ccache - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: jni-windows - - - name: CMake - shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - cmake ^ - -S cpp ^ - -B cpp.build ^ - --preset=ninja-release-jni-windows ^ - -DARROW_BUILD_TESTS=ON ^ - -DCMAKE_INSTALL_PREFIX=%CD%\cpp.install - - - name: Build - shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - cmake --build cpp.build - - - name: Install - shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - cmake --install cpp.build - - - name: Test - shell: cmd - env: - MSYS2_LOCATION: ${{ steps.setup-msys2.outputs.msys2-location }} - ARROW_TEST_DATA: ${{ github.workspace }}\testing\data - PARQUET_TEST_DATA: ${{ github.workspace }}\cpp\submodules\parquet-testing\data - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - set TZDIR=%MSYS2_LOCATION%\usr\share\zoneinfo - - set exclude_tests=arrow-acero-asof-join-node-test - set exclude_tests=%exclude_tests%^|arrow-acero-hash-join-node-test - - ctest ^ - --exclude-regex "%exclude_tests%" ^ - --label-regex unittest ^ - --output-on-failure ^ - --parallel %NUMBER_OF_PROCESSORS% ^ - --test-dir cpp.build ^ - --timeout 300 - - - name: Build example - shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - - cmake ^ - -S cpp/examples/minimal_build ^ - -B cpp/examples/minimal_build.build ^ - -GNinja ^ - -DCMAKE_BUILD_TYPE=Release ^ - -DCMAKE_INSTALL_PREFIX=%CD%\cpp.install - - cmake --build cpp/examples/minimal_build.build - - cd cpp\examples\minimal_build - call ..\minimal_build.build\arrow-example.exe - - - name: Save ccache - if: ${{ !cancelled() }} - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: jni-windows - - odbc-linux: - needs: check-enabled - if: needs.check-enabled.outputs.is_enabled == 'true' - name: ODBC Linux - runs-on: ubuntu-latest - timeout-minutes: 75 - strategy: - fail-fast: false - env: - ARCH: amd64 - UBUNTU: 24.04 - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Setup Python on hosted runner - uses: actions/setup-python@v7 - with: - python-version: 3 - - name: Setup Archery - run: python3 -m pip install -e dev/archery[docker] - - name: Set Up Dremio Instance - run: | - docker compose up -d dremio - cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh - - name: Restore Docker Volumes - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: .docker - key: ubuntu-cpp-odbc - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes - sudo sysctl -w vm.mmap_rnd_bits=28 - source ci/scripts/util_enable_core_dumps.sh - archery docker run ubuntu-cpp-odbc - - name: Save Docker Volumes - if: ${{ !cancelled() }} - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: .docker - key: ubuntu-cpp-odbc - include-hidden-files: true - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ubuntu-cpp-odbc - - odbc-macos: - needs: check-enabled - if: needs.check-enabled.outputs.is_enabled == 'true' - name: ODBC ${{ matrix.build-type }} ${{ matrix.architecture }} macOS ${{ matrix.macos-version }} - runs-on: macos-${{ matrix.macos-version }} - timeout-minutes: 120 - strategy: - fail-fast: false - matrix: - include: - - { architecture: AMD64, macos-version: "15-intel", build-type : debug } - - { architecture: AMD64, macos-version: "15-intel", build-type : release } - - { architecture: ARM64, macos-version: "14", build-type : debug } - - { architecture: ARM64, macos-version: "14", build-type : release } - env: - ARROW_BUILD_TESTS: ON - ARROW_BUILD_TYPE: ${{ matrix.build-type }} - ARROW_DEPENDENCY_SOURCE: BUNDLED - ARROW_DEPENDENCY_USE_SHARED: OFF - ARROW_FLIGHT_SQL_ODBC: ON - ARROW_FLIGHT_SQL_ODBC_INSTALLER: ON - ARROW_HOME: /tmp/local - ARROW_MIMALLOC: OFF - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Install Dependencies - run: | - brew bundle --file=cpp/Brewfile - - # We want to use bundled RE2 for static linking. If - # Homebrew's RE2 is installed, its header file may be used. - # We uninstall Homebrew's RE2 to ensure using bundled RE2. - brew uninstall grpc || : # gRPC depends on RE2 - brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too - brew uninstall re2 - - # We want to use bundled Protobuf for static linking. If - # Homebrew's Protobuf is installed, its library file may be - # used on test We uninstall Homebrew's Protobuf to ensure using - # bundled Protobuf. - brew uninstall protobuf - - # We want to use bundled Boost for static linking. - brew uninstall boost - - name: Setup ccache - run: | - ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - run: | - echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Restore ccache - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: cpp-odbc-ccache-macos-${{ matrix.macos-version }}-${{ matrix.build-type }} - - name: Build - run: | - # Homebrew uses /usr/local as prefix. So packages - # installed by Homebrew also use /usr/local/include. We - # want to include headers for packages installed by - # Homebrew as system headers to ignore warnings in them. - # But "-isystem /usr/local/include" isn't used by CMake - # because /usr/local/include is marked as the default - # include path. So we disable -Werror to avoid build error - # by warnings from packages installed by Homebrew. - export BUILD_WARNING_LEVEL=PRODUCTION - LIBIODBC_DIR="$(brew --cellar libiodbc)/$(brew list --versions libiodbc | awk '{print $2}')" - ODBC_INCLUDE_DIR=$LIBIODBC_DIR/include - export ARROW_CMAKE_ARGS="-DODBC_INCLUDE_DIR=$ODBC_INCLUDE_DIR" - export CXXFLAGS="$CXXFLAGS -I$ODBC_INCLUDE_DIR" - ci/scripts/cpp_build.sh $(pwd) $(pwd)/build - - name: Save ccache - if: ${{ !cancelled() }} - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: cpp-odbc-ccache-macos-${{ matrix.macos-version }}-${{ matrix.build-type }} - - name: Setup Python - uses: actions/setup-python@v7 - with: - python-version: 3 - - name: Setup Archery - run: python3 -m pip install -e dev/archery - - name: Check ODBC Dependency - run: | - # ODBC dependency should not include other Arrow or third party libraries - archery linking check-dependencies \ - --allow CoreFoundation \ - --allow libSystem \ - --allow libarrow_flight_sql_odbc \ - --allow libc++ \ - --allow libiconv \ - --allow libresolv \ - --allow libz \ - "$(pwd)/build/cpp/${{ matrix.build-type }}/libarrow_flight_sql_odbc.dylib" - - name: Generate macOS Installer - if: matrix.build-type == 'release' - shell: bash - run: | - cd $(pwd)/build/cpp - cpack - - name: Upload ODBC PKG to the job - if: matrix.build-type == 'release' - uses: actions/upload-artifact@v7 - with: - name: flight-sql-odbc-pkg-installer-${{ matrix.architecture }} - path: build/cpp/ArrowFlightSQLODBC-*.pkg - if-no-files-found: error - - name: Register Flight SQL ODBC Driver - run: | - sudo cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh $(pwd)/build/cpp/${{ matrix.build-type }}/libarrow_flight_sql_odbc.dylib - - name: Test - shell: bash - run: | - sudo sysctl -w kern.coredump=1 - sudo sysctl -w kern.corefile=/tmp/core.%N.%P - ulimit -c unlimited # must enable within the same shell - # Give CI write access for system DSN - sudo mkdir -p /Library/ODBC - sudo chown -R $USER /Library/ODBC - ci/scripts/cpp_test.sh $(pwd) $(pwd)/build - - odbc-msvc: - needs: check-enabled - if: needs.check-enabled.outputs.is_enabled == 'true' - name: ODBC Windows - runs-on: windows-2022 - timeout-minutes: 240 - permissions: - actions: read - contents: read - packages: write - env: - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: ON - ARROW_BUILD_TYPE: release - # Turn Arrow CSV off to disable `find_package(Arrow)` check on MSVC CI. - # GH-49050 TODO: enable `find_package(Arrow)` check on MSVC CI. - ARROW_CSV: OFF - ARROW_DEPENDENCY_SOURCE: VCPKG - ARROW_FLIGHT_SQL_ODBC: ON - ARROW_FLIGHT_SQL_ODBC_INSTALLER: ON - ARROW_HOME: /usr - # GH-49465: work around the gRPC/Abseil exit hang on Windows - # https://github.com/grpc/grpc/issues/39321 - # https://github.com/abseil/abseil-cpp/issues/1877 - # Build Arrow with GPR_DISABLE_ABSEIL_SYNC so grpcpp's Mutex ABI - # matches the gRPC rebuilt by the overlay triplet. Remove once fixed upstream. - ARROW_CXXFLAGS: -DGPR_DISABLE_ABSEIL_SYNC - CMAKE_GENERATOR: Ninja - CMAKE_INSTALL_PREFIX: /usr - VCPKG_BINARY_SOURCES: 'clear;nugettimeout,600;nuget,GitHub,readwrite' - # GH-49465: custom triplet that rebuilds gRPC with GPR_DISABLE_ABSEIL_SYNC - # (native sync instead of absl::Mutex). See ci/vcpkg/amd64-windows-no-absl-sync-release.cmake - VCPKG_DEFAULT_TRIPLET: amd64-windows-no-absl-sync-release - VCPKG_OVERLAY_TRIPLETS: ${{ github.workspace }}/ci/vcpkg - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Build ODBC Windows - uses: ./.github/actions/odbc-windows - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Test - shell: cmd - run: | - set VCPKG_ROOT_KEEP=%VCPKG_ROOT% - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - set VCPKG_ROOT=%VCPKG_ROOT_KEEP% - # Convert VCPKG Windows path to MSYS path - for /f "usebackq delims=" %%I in (`bash -c "cygpath -u \"$VCPKG_ROOT_KEEP\""` ) do set VCPKG_ROOT=%%I - bash -c "ci/scripts/cpp_test.sh $(pwd) $(pwd)/build" - - name: Install WiX Toolset - shell: pwsh - run: | - Invoke-WebRequest -Uri https://github.com/wixtoolset/wix/releases/download/v6.0.0/wix-cli-x64.msi -OutFile wix-cli-x64.msi - Start-Process -FilePath wix-cli-x64.msi -ArgumentList '/quiet', 'Include_freethreaded=1' -Wait - echo "C:\Program Files\WiX Toolset v6.0\bin\" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - name: Build MSI ODBC installer - shell: pwsh - run: | - # Verify WiX version - wix --version - cd build/cpp - cpack - - name: Upload ODBC MSI to the job - uses: actions/upload-artifact@v7 - with: - name: flight-sql-odbc-msi-installer - path: build/cpp/Apache-Arrow-Flight-SQL-ODBC-*-win64.msi - if-no-files-found: error - - name: Install ODBC MSI - run: | - cd build/cpp - $odbc_msi = Get-ChildItem -Filter "Apache-Arrow-Flight-SQL-ODBC-*-win64.msi" - if (-not $odbc_msi) { - Write-Error "ODBC MSI not found" - exit 1 - } - - foreach ($msi in $odbc_msi) { - Write-Host "Installing $($msi.Name) with logs" - $log = "odbc-install.log" - Start-Process msiexec.exe -Wait -ArgumentList "/i `"$msi`"", "/qn", "/L*V `"$log`"" - Get-Content $log - } - - name: Check ODBC DLL installation - run: | - $dirs = Get-ChildItem "C:\Program Files" -Directory -Filter "Apache-Arrow-Flight-SQL-ODBC*" - - foreach ($dir in $dirs) { - $bin = Join-Path $dir.FullName "bin" - - if (Test-Path $bin) { - tree $bin /f - - $dll = Join-Path $bin "arrow_flight_sql_odbc.dll" - if (Test-Path $dll) { - Write-Host "Found ODBC DLL: $dll" - exit 0 - } - } - } - - Write-Error "ODBC DLL not found" - exit 1 - - odbc-nightly: - needs: odbc-msvc - name: ODBC nightly - runs-on: ubuntu-latest - if: github.event_name == 'schedule' && github.repository == 'apache/arrow' - steps: - - name: Download the artifacts - uses: actions/download-artifact@v8 - with: - name: flight-sql-odbc-msi-installer - - name: Prepare ODBC installer for sync - run: | - mkdir odbc-installer - mv *.msi odbc-installer/ - - # Add `dev-yyyy-mm-dd` to ODBC MSI before `win64.msi`: - # Apache Arrow Flight SQL ODBC-24.0.0-win64.msi -> - # Apache Arrow Flight SQL ODBC-24.0.0-dev-2026-02-06-win64.msi - cd odbc-installer - msi_name=$(ls *.msi) - dev_msi_name=$(echo ${msi_name} | sed -e "s/win64\.msi$/dev-$(date +%Y-%m-%d)-win64.msi/") - mv "${msi_name}" "${dev_msi_name}" - cd .. - - tree odbc-installer - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 1 - path: arrow - repository: apache/arrow - ref: main - submodules: recursive - - name: Sync to Remote - uses: ./arrow/.github/actions/sync-nightlies - with: - upload: true - switches: -avzh --update --delete --progress - local_path: odbc-installer - remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/odbc - remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} - remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} - remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} - remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} - remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} - - report-extra-cpp: - if: github.event_name == 'schedule' && always() - needs: - - docker - - jni-linux - - jni-macos - - msvc-arm64 - - odbc-linux - - odbc-macos - - odbc-msvc - - odbc-nightly - uses: ./.github/workflows/report_ci.yml - secrets: - ARROW_SMTP_PASSWORD: ${{ secrets.ARROW_SMTP_PASSWORD }} - ARROW_ZULIP_WEBHOOK: ${{ secrets.ARROW_ZULIP_WEBHOOK }} diff --git a/.github/workflows/cpp_windows.yml b/.github/workflows/cpp_windows.yml deleted file mode 100644 index 3e3cb44aa76b..000000000000 --- a/.github/workflows/cpp_windows.yml +++ /dev/null @@ -1,141 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: C++ Windows MSVC Reusable - -on: - workflow_call: - inputs: - arch: - description: "The architecture to build for" - required: true - type: string - os: - description: "The runner label to run the job on" - required: true - type: string - simd-level: - description: "The Arrow SIMD level to build with" - required: true - type: string - -permissions: - actions: read - contents: read - -jobs: - windows: - runs-on: ${{ inputs.os }} - timeout-minutes: 60 - env: - ARROW_AZURE: ON - ARROW_BOOST_USE_SHARED: OFF - ARROW_BUILD_BENCHMARKS: ON - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: ON - ARROW_DATASET: ON - ARROW_FILESYSTEM: ON - ARROW_FLIGHT: OFF - ARROW_HDFS: ON - ARROW_HOME: /usr - ARROW_JEMALLOC: OFF - ARROW_MIMALLOC: ON - ARROW_ORC: ON - ARROW_PARQUET: ON - ARROW_SIMD_LEVEL: ${{ inputs.simd-level }} - ARROW_SUBSTRAIT: ON - ARROW_USE_GLOG: OFF - ARROW_VERBOSE_THIRDPARTY_BUILD: OFF - ARROW_WITH_BROTLI: OFF - ARROW_WITH_BZ2: OFF - ARROW_WITH_LZ4: OFF - ARROW_WITH_OPENTELEMETRY: OFF - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - BOOST_SOURCE: BUNDLED - CMAKE_CXX_STANDARD: "20" - CMAKE_GENERATOR: Ninja - CMAKE_INSTALL_PREFIX: /usr - # ccache does not support PDB files (https://github.com/ccache/ccache/issues/1040) - CMAKE_MSVC_DEBUG_INFORMATION_FORMAT: Embedded - CMAKE_UNITY_BUILD: ON - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Install msys2 (for tzdata for ORC tests) - uses: msys2/setup-msys2@v2 - id: setup-msys2 - - name: Install cmake - shell: bash - run: | - ci/scripts/install_cmake.sh 4.1.2 /usr - - name: Install ccache - shell: bash - run: | - ci/scripts/install_ccache.sh 4.13.6 /usr - - name: Setup ccache - shell: bash - run: | - ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: | - echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Restore ccache - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: cpp-ccache-windows-${{ inputs.arch }} - - name: Build - shell: cmd - env: - VCVARS_ARCH: ${{ inputs.arch }} - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" %VCVARS_ARCH% - bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build" - - name: Save ccache - if: ${{ !cancelled() }} - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: cpp-ccache-windows-${{ inputs.arch }} - - name: Test - shell: cmd - env: - VCVARS_ARCH: ${{ inputs.arch }} - MSYS2_LOCATION: ${{ steps.setup-msys2.outputs.msys2-location }} - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" %VCVARS_ARCH% - # For ORC - set TZDIR=%MSYS2_LOCATION%\usr\share\zoneinfo - bash -c "ci/scripts/cpp_test.sh $(pwd) $(pwd)/build" diff --git a/.github/workflows/cuda_extra.yml b/.github/workflows/cuda_extra.yml deleted file mode 100644 index fedf672fb1bd..000000000000 --- a/.github/workflows/cuda_extra.yml +++ /dev/null @@ -1,156 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: CUDA Extra - -on: - push: - branches: - - '**' - - '!dependabot/**' - paths: - - '**/*cuda*' - - '**/*gpu*' - tags: - - '**' - pull_request: - # This trigger ensures that the `check_labels` workflow runs on manual label changes - types: - - labeled - - opened - - reopened - - synchronize - schedule: - - cron: | - 0 6 * * * - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - actions: read - contents: read - pull-requests: read - -jobs: - check-labels: - uses: ./.github/workflows/check_labels.yml - with: - parent-workflow: cuda_extra - - check-enabled: - # Check whether the CI builds in this workflow need to be run - needs: check-labels - runs-on: ubuntu-latest - outputs: - is_enabled: ${{ steps.set_enabled.outputs.is_enabled }} - steps: - - id: set_enabled - if: >- - github.ref_type == 'tag' || - (github.event_name == 'schedule' && github.repository == 'apache/arrow') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: CUDA') - run: echo "is_enabled=true" >> "$GITHUB_OUTPUT" - - docker: - needs: check-enabled - if: needs.check-enabled.outputs.is_enabled == 'true' - name: ${{ matrix.title }} - runs-on: "runs-on=${{ github.run_id }}/family=g4dn.xlarge/image=ubuntu24-gpu-x64/spot=capacity-optimized" - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - include: - - cuda: 12.9.0 - ubuntu: 24.04 - image: ubuntu-cuda-cpp - title: AMD64 Ubuntu 24 CUDA 12.9.0 - - cuda: 11.7.1 - ubuntu: 22.04 - image: ubuntu-cuda-cpp - title: AMD64 Ubuntu 22 CUDA 11.7.1 - - cuda: 12.9.0 - ubuntu: 24.04 - image: ubuntu-cuda-python - title: AMD64 Ubuntu 24 CUDA 12.9.0 Python - - cuda: 11.7.1 - ubuntu: 22.04 - image: ubuntu-cuda-python - title: AMD64 Ubuntu 22 CUDA 11.7.1 Python - env: - ARCHERY_DEBUG: 1 - ARROW_ENABLE_TIMING_TESTS: OFF - DOCKER_VOLUME_PREFIX: ".docker/" - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Cache Docker Volumes - uses: actions/cache@v6 - with: - path: .docker - key: extra-${{ matrix.image }}-${{ hashFiles('cpp/**') }} - restore-keys: extra-${{ matrix.image }}- - - name: Setup Python - uses: actions/setup-python@v7 - with: - python-version: 3 - - name: Setup Archery - run: python3 -m pip install -e dev/archery[docker] - - name: Display NVIDIA SMI details - run: | - nvidia-smi - nvidia-smi -L - nvidia-smi -q -d Memory - - name: Execute Docker Build - continue-on-error: ${{ matrix.continue-on-error || false }} - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - CUDA: ${{ matrix.cuda }} - UBUNTU: ${{ matrix.ubuntu }} - run: | - # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes - sudo sysctl -w vm.mmap_rnd_bits=28 - source ci/scripts/util_enable_core_dumps.sh - archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }} - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - report-extra-cpp: - if: github.event_name == 'schedule' && always() - needs: - - docker - uses: ./.github/workflows/report_ci.yml - secrets: - ARROW_SMTP_PASSWORD: ${{ secrets.ARROW_SMTP_PASSWORD }} - ARROW_ZULIP_WEBHOOK: ${{ secrets.ARROW_ZULIP_WEBHOOK }} diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml deleted file mode 100644 index 595ab9a933c5..000000000000 --- a/.github/workflows/dev.yml +++ /dev/null @@ -1,113 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Dev - -on: - # always trigger except Dependabot "push" - push: - branches: - - '**' - - '!dependabot/**' - tags: - - '**' - pull_request: - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - -jobs: - - lint: - name: Lint C++, Python, R, Docker, RAT - # Use Ubuntu 24.04 to ensure working pre-commit on Ubuntu 24.04. - runs-on: ubuntu-24.04 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - - name: Install pre-commit - run: | - sudo apt update - sudo apt install -y -V \ - pre-commit \ - ruby-dev \ - libuv1-dev - - name: Cache pre-commit - uses: actions/cache@v6 - with: - path: | - ~/.cache/pre-commit - key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} - - name: Run pre-commit - run: | - pre-commit run --all-files --color=always --show-diff-on-failure - - release: - name: Source Release and Merge Script on ${{ matrix.runs-on }} - runs-on: ${{ matrix.runs-on }} - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - runs-on: - - macos-latest - - ubuntu-latest - env: - GIT_AUTHOR_NAME: "github-actions[bot]" - GIT_AUTHOR_EMAIL: "github-actions[bot]@users.noreply.github.com" - GIT_COMMITTER_NAME: "github-actions[bot]" - GIT_COMMITTER_EMAIL: "github-actions[bot]@users.noreply.github.com" - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - - name: Install Python - uses: actions/setup-python@v7 - with: - python-version: '3.12' - - name: Install Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: ruby - - name: Install Dependencies - shell: bash - run: | - gem install test-unit openssl - pip install build "cython>=3.1" pytest requests scikit-build-core setuptools-scm - - name: Run Release Test - shell: bash - run: | - echo "GH_TOKEN=${{ secrets.GITHUB_TOKEN }}" > dev/release/.env - ci/scripts/release_test.sh $(pwd) - - name: Run Merge Script Test - shell: bash - run: pytest -v dev/test_merge_arrow_pr.py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml deleted file mode 100644 index 197eb19b1510..000000000000 --- a/.github/workflows/integration.yml +++ /dev/null @@ -1,152 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Integration - -on: - push: - branches: - - '**' - - '!dependabot/**' - tags: - - '**' - paths: - - '.dockerignore' - - '.github/workflows/integration.yml' - - 'ci/**' - - 'dev/archery/**' - - 'compose.yaml' - - 'integration/**' - - 'cpp/**' - - 'format/**' - - 'ruby/red-arrow-format/**' - pull_request: - paths: - - '.dockerignore' - - '.github/workflows/integration.yml' - - 'ci/**' - - 'dev/archery/**' - - 'compose.yaml' - - 'integration/**' - - 'cpp/**' - - 'format/**' - - 'ruby/red-arrow-format/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - - docker: - name: AMD64 Conda Integration Test - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Checkout Arrow Rust - uses: actions/checkout@v7 - with: - persist-credentials: false - repository: apache/arrow-rs - path: rust - - name: Checkout Arrow nanoarrow - uses: actions/checkout@v7 - with: - persist-credentials: false - repository: apache/arrow-nanoarrow - path: nanoarrow - - name: Checkout Arrow Go - uses: actions/checkout@v7 - with: - persist-credentials: false - repository: apache/arrow-go - path: go - - name: Checkout Arrow Java - uses: actions/checkout@v7 - with: - persist-credentials: false - repository: apache/arrow-java - path: java - - name: Checkout Arrow JS - uses: actions/checkout@v7 - with: - persist-credentials: false - repository: apache/arrow-js - path: js - - name: Checkout Arrow .NET - uses: actions/checkout@v7 - with: - persist-credentials: false - repository: apache/arrow-dotnet - path: dotnet - - name: Free up disk space - run: | - ci/scripts/util_free_space.sh - - name: Cache Docker Volumes - uses: actions/cache@v6 - with: - path: .docker - key: conda-${{ hashFiles('cpp/**') }} - restore-keys: conda- - - name: Setup Python - uses: actions/setup-python@v7 - with: - python-version: 3.12 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - run: | - source ci/scripts/util_enable_core_dumps.sh - archery docker run \ - -e ARCHERY_DEFAULT_BRANCH="${DEFAULT_BRANCH}" \ - -e ARCHERY_INTEGRATION_WITH_DOTNET=1 \ - -e ARCHERY_INTEGRATION_WITH_GO=1 \ - -e ARCHERY_INTEGRATION_WITH_JAVA=1 \ - -e ARCHERY_INTEGRATION_WITH_JS=1 \ - -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \ - -e ARCHERY_INTEGRATION_WITH_RUST=1 \ - -e RUST_BACKTRACE=1 \ - conda-integration - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push conda-integration diff --git a/.github/workflows/package_linux.yml b/.github/workflows/package_linux.yml deleted file mode 100644 index a70ae8586154..000000000000 --- a/.github/workflows/package_linux.yml +++ /dev/null @@ -1,367 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Package Linux - -on: - push: - branches: - - '**' - - '!dependabot/**' - - '!release-*' - paths: - - '.github/workflows/check_labels.yml' - - '.github/workflows/package_linux.yml' - - '.github/workflows/report_ci.yml' - - 'cpp/**' - - 'c_glib/**' - - 'dev/archery/archery/**' - - 'dev/release/binary-task.rb' - - 'dev/release/verify-apt.sh' - - 'dev/release/verify-yum.sh' - - 'dev/tasks/linux-packages/**' - - 'format/Flight.proto' - tags: - - "apache-arrow-*-rc*" - pull_request: - # This trigger ensures that the `check_labels` workflow runs on manual label changes - types: - - labeled - - opened - - reopened - - synchronize - schedule: - - cron: "0 2 * * *" - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - actions: read - contents: read - pull-requests: read - -jobs: - check-labels: - uses: ./.github/workflows/check_labels.yml - with: - parent-workflow: package_linux - - check-enabled: - # Check whether the CI builds in this workflow need to be run - needs: check-labels - runs-on: ubuntu-latest - outputs: - is_enabled: ${{ steps.set_enabled.outputs.is_enabled }} - steps: - - id: set_enabled - if: >- - github.ref_type == 'tag' || - (github.event_name == 'schedule' && github.repository == 'apache/arrow') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: Package: Linux') - run: echo "is_enabled=true" >> "$GITHUB_OUTPUT" - - package: - needs: check-enabled - if: needs.check-enabled.outputs.is_enabled == 'true' - permissions: - # Upload artifacts to GitHub Release - contents: write - # Upload cached Docker images to GitHub Packages - packages: write - name: ${{ matrix.id }} - runs-on: ${{ contains(matrix.id, 'amd64') && 'ubuntu-latest' || 'ubuntu-24.04-arm' }} - timeout-minutes: 160 - strategy: - fail-fast: false - matrix: - id: - - almalinux-8-amd64 - - almalinux-8-arm64 - - almalinux-9-amd64 - - almalinux-9-arm64 - - almalinux-10-amd64 - - almalinux-10-arm64 - - amazon-linux-2023-amd64 - - amazon-linux-2023-arm64 - - centos-9-stream-amd64 - - centos-9-stream-arm64 - - debian-trixie-amd64 - - debian-trixie-arm64 - - debian-forky-amd64 - - debian-forky-arm64 - - ubuntu-jammy-amd64 - - ubuntu-jammy-arm64 - - ubuntu-noble-amd64 - - ubuntu-noble-arm64 - - ubuntu-resolute-amd64 - - ubuntu-resolute-arm64 - env: - BUILD_DIR: "${{ github.workspace }}/packages.build" - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Free up disk space - run: | - ci/scripts/util_free_space.sh - - name: Prepare environment variables - env: - ID: ${{ matrix.id }} - run: | - set -x - - case "${ID}" in - centos-*) - # Example: centos-9-stream-amd64 -> centos - distribution="${ID%%-*}" - ;; - *) - # Example: almalinux-8-amd64 -> almalinux - # Example: amazon-linux-2023-amd64 -> amazon-linux - distribution="${ID%-*-*}" - ;; - esac - echo "DISTRIBUTION=${distribution}" >> "${GITHUB_ENV}" - - # Example: almalinux-8-amd64 -> amd64 - architecture="${ID##*-}" - echo "ARCHITECTURE=${architecture}" >> "${GITHUB_ENV}" - - # Example: almalinux-8-amd64 -> almalinux-8 - target="${ID%-*}" - case "${target}" in - almalinux-*|amazon-linux-*|centos-*) - echo "TASK_NAMESPACE=yum" >> "${GITHUB_ENV}" - # Example: centos-9-stream-amd64 -> centos-9-stream - # Example: amazon-linux-2023-amd64 -> amazon-linux-2023 - version="${ID%-*}" - # Example: centos-9-stream -> 9-stream - # Example: amazon-linux-2023 -> 2023 - version="${version##${distribution}-}" - echo "DISTRIBUTION_VERSION=${version}" >> "${GITHUB_ENV}" - if [ "${architecture}" = "arm64" ]; then - # Example: almalinux-8 -> almalinux-8-aarch64 - target="${target}-aarch64" - fi - echo "YUM_TARGETS=${target}" >> "${GITHUB_ENV}" - ;; - *) - echo "TASK_NAMESPACE=apt" >> "${GITHUB_ENV}" - # Example: debian-bookworm-amd64 -> debian-bookworm - code_name="${ID%-*}" - # Example: debian-bookworm -> bookworm - code_name="${code_name#*-}" - echo "DISTRIBUTION_CODE_NAME=${code_name}" >> "${GITHUB_ENV}" - if [ "${architecture}" = "arm64" ]; then - # Example: ubuntu-noble -> ubuntu-noble-arm64 - target="${target}-arm64" - fi - echo "APT_TARGETS=${target}" >> "${GITHUB_ENV}" - ;; - esac - echo "TARGET=${target}" >> "${GITHUB_ENV}" - - if [ "${GITHUB_REF_TYPE}" = "tag" ]; then - # Example: apache-arrow-21.0.0-rc0 -> 21.0.0-rc0 - version="${GITHUB_REF_NAME#apache-arrow-}" - echo "ARROW_VERSION=${version}" >> "${GITHUB_ENV}" - fi - - name: Restore ccache - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ env.BUILD_DIR }}/${{ env.TARGET }} - key: package-linux-${{ matrix.id }} - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y \ - rake \ - reprotest \ - ruby \ - ruby-dev - - name: Allow unprivileged user namespaces - if: env.TASK_NAMESPACE == 'apt' - # Reprotest's domain_host variation uses `unshare -r --uts`, - # which writes to /proc/self/uid_map. Ubuntu 24.04 restricts - # unprivileged user namespaces via AppArmor by default, so - # the call fails. Try to lift the restriction so reprotest - # can run the full set of variations. - run: | - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - - name: Prepare apache-arrow-apt-source for arm64 - if: env.ARCHITECTURE == 'arm64' - run: | - pushd dev/tasks/linux-packages/apache-arrow-apt-source/apt - for target in *-*; do - cp -a ${target} ${target}-arm64 - done - popd - - name: Prepare apache-arrow-release for arm64 - if: env.ARCHITECTURE == 'arm64' - run: | - pushd dev/tasks/linux-packages/apache-arrow-release/yum - for target in *-*; do - cp -a ${target} ${target}-aarch64 - done - popd - - name: Update version - if: github.ref_type != 'tag' - run: | - pushd dev/tasks/linux-packages - rake version:update - popd - - name: Login to GitHub Container registry - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Wait for creating GitHub Release - if: github.ref_type == 'tag' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - dev/release/utils-watch-gh-workflow.sh \ - ${GITHUB_REF_NAME} \ - release_candidate.yml - - name: Build - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - rake -C dev/tasks/linux-packages/apache-arrow docker:pull:${TARGET} || : - rake -C dev/tasks/linux-packages ${TASK_NAMESPACE}:build - - name: Save ccache - if: >- - !cancelled() - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ env.BUILD_DIR }}/${{ env.TARGET }} - key: package-linux-${{ matrix.id }} - include-hidden-files: true - - name: Docker Push - continue-on-error: true - if: >- - success() && - github.event_name == 'push' && - github.ref_name == 'main' - run: | - rake -C dev/tasks/linux-packages/apache-arrow docker:push:${TARGET} - - name: Build artifact tarball - run: | - mkdir -p "${DISTRIBUTION}" - cp -a \ - dev/tasks/linux-packages/*/${TASK_NAMESPACE}/repositories/${DISTRIBUTION}/* \ - "${DISTRIBUTION}/" - set -x - # We use latest .deb/.rpm of - # apache-arrow-apt-source/apache-arrow-release built for - # amd64 because they are architecture independent. - if [ "${ARCHITECTURE}" = "amd64" ]; then - if [ "${TASK_NAMESPACE}" = "apt" ]; then - # Create - # https://packages.apache.org/artifactory/arrow/${DISTRIBUTION}/apache-arrow-apt-source-latest-${DISTRIBUTION_CODE_NAME}.deb - # for easy to install. - cp -a \ - ${DISTRIBUTION}/pool/${DISTRIBUTION_CODE_NAME}/*/a/apache-arrow-apt-source/*.deb \ - ${DISTRIBUTION}/apache-arrow-apt-source-latest-${DISTRIBUTION_CODE_NAME}.deb - else - # Create - # https://packages.apache.org/artifactory/arrow/${DISTRIBUTION}/${DISTRIBUTION_VERSION}/apache-arrow-release-latest.rpm - # for easy to install. - cp -a \ - ${DISTRIBUTION}/${DISTRIBUTION_VERSION}/x86_64/Packages/apache-arrow-release-*.rpm \ - ${DISTRIBUTION}/${DISTRIBUTION_VERSION}/apache-arrow-release-latest.rpm - fi - fi - tar cvzf ${{ matrix.id }}.tar.gz ${DISTRIBUTION} - dev/release/utils-generate-checksum.sh ${{ matrix.id }}.tar.gz - - name: Upload the artifacts to the job - uses: actions/upload-artifact@v7 - with: - name: ${{ matrix.id }} - path: ${{ matrix.id }}.tar.gz* - - name: Upload the artifacts to GitHub Release - if: github.ref_type == 'tag' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release upload ${GITHUB_REF_NAME} \ - --clobber \ - ${{ matrix.id }}.tar.gz* - - name: Set up test - run: | - sudo apt install -y \ - apt-utils \ - cpio \ - createrepo-c \ - devscripts \ - gpg \ - rpm \ - rsync - gem install --user-install apt-dists-merge - { - echo "Key-Type: RSA" - echo "Key-Length: 4096" - echo "Name-Real: Test" - echo "Name-Email: test@example.com" - echo "%no-protection" - } | gpg --full-generate-key --batch - GPG_KEY_ID=$(gpg --list-keys --with-colon test@example.com | grep fpr | cut -d: -f10) - echo "GPG_KEY_ID=${GPG_KEY_ID}" >> ${GITHUB_ENV} - if [ "${TASK_NAMESPACE}" = "yum" ]; then - repositories_dir=dev/tasks/linux-packages/apache-arrow-release/yum/repositories - rpm2cpio ${repositories_dir}/*/*/*/Packages/apache-arrow-release-*.noarch.rpm | - cpio -id - mv etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow \ - dev/tasks/linux-packages/KEYS - fi - gpg --export --armor test@example.com >> dev/tasks/linux-packages/KEYS - - name: Test - run: | - pushd dev/tasks/linux-packages - rake --trace ${TASK_NAMESPACE}:test - popd - - name: Verify Reproducibility - if: env.TASK_NAMESPACE == 'apt' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Validate reproducibility. Reprotest runs the build twice - # inside its own tempdir and doesn't copy artifacts back, - # so this is purely a verification step. - reprotest \ - --vary=-fileordering \ - --build-command \ - "rake -C dev/tasks/linux-packages ${TASK_NAMESPACE}:build" \ - "${PWD}" \ - "dev/tasks/linux-packages/*/apt/repositories/${DISTRIBUTION}/pool/${DISTRIBUTION_CODE_NAME}/*/*/*/*.deb" - - report-package-linux: - if: github.event_name == 'schedule' && always() - needs: - - package - uses: ./.github/workflows/report_ci.yml - secrets: - ARROW_SMTP_PASSWORD: ${{ secrets.ARROW_SMTP_PASSWORD }} - ARROW_ZULIP_WEBHOOK: ${{ secrets.ARROW_ZULIP_WEBHOOK }} diff --git a/.github/workflows/package_odbc.yml b/.github/workflows/package_odbc.yml deleted file mode 100644 index cc2aad00aca7..000000000000 --- a/.github/workflows/package_odbc.yml +++ /dev/null @@ -1,178 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Package Apache Arrow Flight SQL ODBC Driver - -on: - push: - tags: - - "apache-arrow-*-rc*" - workflow_dispatch: - inputs: - odbc_release_step: - description: 'ODBC MSI release step' - required: false - default: false - type: boolean - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - actions: read - contents: read - pull-requests: read - -jobs: - # GH-49537 CPack only packages from a single build directory, so the build - # cannot be reused and we need to rebuild the MSI in a separate workflow. This - # means both the odbc-msvc-upload-dll and odbc-msvc-upload-msi jobs do a full build. - odbc-msvc-upload-dll: - name: ODBC Windows Upload Unsigned DLL - runs-on: windows-2022 - if: github.event_name == 'push' - timeout-minutes: 240 - permissions: - packages: write - env: &odbc_msvc_env - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: OFF - ARROW_BUILD_TYPE: release - # Turn Arrow CSV off to disable `find_package(Arrow)` check on MSVC CI. - # GH-49050 TODO: enable `find_package(Arrow)` check on MSVC CI. - ARROW_CSV: OFF - ARROW_DEPENDENCY_SOURCE: VCPKG - ARROW_FLIGHT_SQL_ODBC: ON - ARROW_FLIGHT_SQL_ODBC_INSTALLER: ON - ARROW_HOME: /usr - CMAKE_GENERATOR: Ninja - CMAKE_INSTALL_PREFIX: /usr - VCPKG_BINARY_SOURCES: 'clear;nugettimeout,600;nuget,GitHub,readwrite' - VCPKG_DEFAULT_TRIPLET: x64-windows - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Build ODBC Windows - uses: ./.github/actions/odbc-windows - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Rename Unsigned ODBC DLL - run: | - Rename-Item ` - -Path build/cpp/${{ env.ARROW_BUILD_TYPE }}/arrow_flight_sql_odbc.dll ` - -NewName arrow_flight_sql_odbc_unsigned.dll - - name: Upload ODBC DLL to the job - uses: actions/upload-artifact@v7 - with: - name: flight-sql-odbc-dll - path: build/cpp/${{ env.ARROW_BUILD_TYPE }}/arrow_flight_sql_odbc_unsigned.dll - if-no-files-found: error - - odbc-dll-release: - needs: odbc-msvc-upload-dll - name: Upload Unsigned ODBC DLL to Release - runs-on: ubuntu-latest - permissions: - # Upload to GitHub Release - contents: write - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Download the artifacts - uses: actions/download-artifact@v8 - with: - name: flight-sql-odbc-dll - - name: Wait for creating GitHub Release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - dev/release/utils-watch-gh-workflow.sh \ - ${GITHUB_REF_NAME} \ - release_candidate.yml - - name: Upload the artifacts to GitHub Release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release upload ${GITHUB_REF_NAME} \ - --clobber \ - arrow_flight_sql_odbc_unsigned.dll - - odbc-msvc-upload-msi: - name: ODBC Windows Build & Upload Unsigned MSI - runs-on: windows-2022 - if: inputs.odbc_release_step - timeout-minutes: 240 - permissions: - # Upload to GitHub Release - contents: write - packages: write - env: *odbc_msvc_env - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Download signed ODBC DLL - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release download $env:GITHUB_REF_NAME ` - --pattern arrow_flight_sql_odbc.dll ` - --clobber - - name: Build ODBC Windows - uses: ./.github/actions/odbc-windows - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Replace unsigned DLL with signed DLL - run: | - Move-Item ` - -Path ./arrow_flight_sql_odbc.dll ` - -Destination build/cpp/${{ env.ARROW_BUILD_TYPE }}/arrow_flight_sql_odbc.dll ` - -Force - - name: Install WiX Toolset - shell: pwsh - run: | - Invoke-WebRequest -Uri https://github.com/wixtoolset/wix/releases/download/v6.0.0/wix-cli-x64.msi -OutFile wix-cli-x64.msi - Start-Process -FilePath wix-cli-x64.msi -ArgumentList '/quiet', 'Include_freethreaded=1' -Wait - echo "C:\Program Files\WiX Toolset v6.0\bin\" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - name: Build MSI ODBC installer - shell: pwsh - run: | - # Verify WiX version - wix --version - cd build/cpp - cpack - - name: Upload the artifacts to GitHub Release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd build/cpp - gh release upload $env:GITHUB_REF_NAME ` - --clobber ` - Apache-Arrow-Flight-SQL-ODBC-*-win64.msi diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml deleted file mode 100644 index acad4c2dd8c6..000000000000 --- a/.github/workflows/python.yml +++ /dev/null @@ -1,315 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Python - -on: - push: - branches: - - '**' - - '!dependabot/**' - tags: - - '**' - paths: - - '.dockerignore' - - '.github/workflows/python.yml' - - 'ci/**' - - 'cpp/**' - - 'compose.yaml' - - 'docs/source/python/**' - - 'python/**' - pull_request: - paths: - - '.dockerignore' - - '.github/workflows/python.yml' - - 'ci/**' - - 'cpp/**' - - 'compose.yaml' - - 'docs/source/python/**' - - 'python/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - - docker: - name: ${{ matrix.title }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - max-parallel: 20 - matrix: - name: - - conda-python-docs - - conda-python-3.12-nopandas - - conda-python-3.11-pandas-2.2.2-numpy-2.0.2 - - conda-python-3.14-pandas-latest - - conda-python-3.13-no-numpy - include: - - name: conda-python-docs - image: conda-python-docs - title: AMD64 Conda Python 3.12 Sphinx & Numpydoc - python: "3.12" - - name: conda-python-3.12-nopandas - image: conda-python - title: AMD64 Conda Python 3.12 Without Pandas - python: "3.12" - - name: conda-python-3.11-pandas-2.2.2-numpy-2.0.2 - image: conda-python-pandas - title: AMD64 Conda Python 3.11 Pandas 2.2.2 NumPy 2.0.2 - python: "3.11" - pandas: "2.2.2" - numpy: "2.0.2" - - name: conda-python-3.14-pandas-latest - image: conda-python-pandas - title: AMD64 Conda Python 3.14 Pandas latest - python: "3.14" - pandas: latest - - name: conda-python-3.13-no-numpy - image: conda-python-no-numpy - title: AMD64 Conda Python 3.13 without NumPy - python: "3.13" - env: - PYTHON: ${{ matrix.python || 3.11 }} - UBUNTU: ${{ matrix.ubuntu || 22.04 }} - PANDAS: ${{ matrix.pandas || 'latest' }} - NUMPY: ${{ matrix.numpy || 'latest' }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Restore Docker Volumes - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: .docker - key: python-${{ matrix.name }} - - name: Setup Python - uses: actions/setup-python@v7 - with: - python-version: 3.12 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - source ci/scripts/util_enable_core_dumps.sh - archery docker run ${{ matrix.image }} - - name: Save Docker Volumes - if: ${{ !cancelled() }} - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: .docker - key: python-${{ matrix.name }} - include-hidden-files: true - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - macos: - name: ${{ matrix.architecture }} macOS ${{ matrix.macos-version }} Python 3 - runs-on: macos-${{ matrix.macos-version }} - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - max-parallel: 20 - matrix: - include: - - architecture: AMD64 - macos-version: "15-intel" - large-memory-tests: "OFF" - - architecture: ARM64 - macos-version: "14" - large-memory-tests: "ON" - env: - ARROW_HOME: /tmp/local - ARROW_AZURE: ON - ARROW_DATASET: ON - ARROW_FLIGHT: ON - ARROW_GANDIVA: ON - ARROW_GCS: OFF - ARROW_HDFS: ON - ARROW_JEMALLOC: ON - ARROW_ORC: ON - ARROW_PARQUET: ON - PARQUET_REQUIRE_ENCRYPTION: ON - ARROW_PYTHON: ON - ARROW_S3: ON - ARROW_SUBSTRAIT: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_ZSTD: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_BROTLI: ON - ARROW_BUILD_TESTS: OFF - PYARROW_TEST_LARGE_MEMORY: ${{ matrix.large-memory-tests }} - PYTEST_ARGS: "-n auto --durations=40" - # Current oldest supported version according to https://endoflife.date/macos - MACOSX_DEPLOYMENT_TARGET: 12.0 - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Setup Python - uses: actions/setup-python@v7 - with: - python-version: '3.11' - - name: Install Dependencies - shell: bash - run: | - brew bundle --file=cpp/Brewfile - python -m pip install \ - -r python/requirements-build.txt \ - -r python/requirements-test.txt - - name: Install MinIO - run: | - $(brew --prefix bash)/bin/bash \ - ci/scripts/install_minio.sh latest /usr/local - - name: Setup ccache - shell: bash - run: ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Restore ccache - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: python-ccache-macos-${{ matrix.macos-version }} - - name: Build - shell: bash - run: | - if [ "${{ matrix.macos-version }}" = "15-intel" ]; then - # This is a workaround. - # - # Homebrew uses /usr/local as prefix. So packages - # installed by Homebrew also use /usr/local/include. We - # want to include headers for packages installed by - # Homebrew as system headers to ignore warnings in them. - # But "-isystem /usr/local/include" isn't used by CMake - # because /usr/local/include is marked as the default - # include path. So we disable -Werror to avoid build error - # by warnings from packages installed by Homebrew. - export BUILD_WARNING_LEVEL=PRODUCTION - fi - - python -m pip install wheel - ci/scripts/cpp_build.sh $(pwd) $(pwd)/build - ci/scripts/python_build.sh $(pwd) $(pwd)/build - - name: Save ccache - if: ${{ !cancelled() }} - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: python-ccache-macos-${{ matrix.macos-version }} - - name: Test - shell: bash - run: ci/scripts/python_test.sh $(pwd) $(pwd)/build - - windows: - name: AMD64 Windows 2022 Python 3.13 - runs-on: windows-2022 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - env: - CCACHE_DIR: C:/ccache - PYTHON_CMD: "py -3.13" - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Setup Python - uses: actions/setup-python@v7 - with: - python-version: 3.13 - - name: Install ccache - shell: bash - run: | - ci/scripts/install_ccache.sh 4.13.6 /usr - - name: Setup ccache - shell: bash - run: | - ci/scripts/ccache_setup.sh - - name: Generate path variables - id: path-info - shell: bash - run: | - echo "usr-windows-dir="$(cygpath --absolute --windows /usr)"" >> $GITHUB_OUTPUT - - name: Restore ccache - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ env.CCACHE_DIR }} - key: python-ccache-windows - - name: Build Arrow C++ and PyArrow - shell: cmd - env: - USR_WINDOWS_DIR: ${{ steps.path-info.outputs.usr-windows-dir }} - run: | - call "ci\scripts\python_build.bat" %cd% "%USR_WINDOWS_DIR%" - - name: Save ccache - if: ${{ !cancelled() }} - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ env.CCACHE_DIR }} - key: python-ccache-windows - - name: Test PyArrow - shell: cmd - run: | - call "ci\scripts\python_test.bat" %cd% diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml deleted file mode 100644 index 98b301d095dc..000000000000 --- a/.github/workflows/r.yml +++ /dev/null @@ -1,352 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: R - -on: - push: - branches: - - '**' - - '!dependabot/**' - tags: - - '**' - paths: - - '.dockerignore' - - ".github/workflows/r.yml" - - "ci/docker/**" - - "ci/etc/rprofile" - - "ci/scripts/PKGBUILD" - - "ci/scripts/cpp_*.sh" - - "ci/scripts/install_minio.sh" - - "ci/scripts/r_*.sh" - - "cpp/**" - - "compose.yaml" - - "r/**" - pull_request: - paths: - - '.dockerignore' - - ".github/workflows/r.yml" - - "ci/docker/**" - - "ci/etc/rprofile" - - "ci/scripts/PKGBUILD" - - "ci/scripts/cpp_*.sh" - - "ci/scripts/install_minio.sh" - - "ci/scripts/r_*.sh" - - "cpp/**" - - "compose.yaml" - - "r/**" - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - ubuntu: - name: AMD64 Ubuntu ${{ matrix.ubuntu }} R ${{ matrix.r }} Force-Tests ${{ matrix.force-tests }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - r: ["4.6"] - ubuntu: [24.04] - force-tests: ["true"] - env: - R: ${{ matrix.r }} - UBUNTU: ${{ matrix.ubuntu }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Free up disk space - run: | - ci/scripts/util_free_space.sh - - name: Cache Docker Volumes - uses: actions/cache@v6 - with: - path: .docker - # As this key is identical on both matrix builds only one will be able to successfully cache, - # this is fine as there are no differences in the build - key: ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}-${{ github.run_id }} - restore-keys: | - ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- - ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}- - - name: Setup Python - uses: actions/setup-python@v7 - with: - python-version: 3.12 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - source ci/scripts/util_enable_core_dumps.sh - # Setting a non-default and non-probable Marquesas French Polynesia time - # it has both with a .45 offset and very very few people who live there. - archery docker run -e TZ=MART -e ARROW_R_FORCE_TESTS=${{ matrix.force-tests }} ubuntu-r - - name: Dump install logs - run: cat r/check/arrow.Rcheck/00install.out - if: always() - - name: Dump test logs - run: cat r/check/arrow.Rcheck/tests/testthat.Rout* - if: always() - - name: Save the test output - if: always() - uses: actions/upload-artifact@v7 - with: - name: test-output-${{ matrix.ubuntu }}-${{ matrix.r }} - path: r/check/arrow.Rcheck/tests/testthat.Rout* - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ubuntu-r - - bundled: - name: "${{ matrix.config.org }}/${{ matrix.config.image }}:${{ matrix.config.tag }}" - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - config: - - { org: "rhub", image: "ubuntu-gcc12", tag: "latest" } - env: - R_ORG: ${{ matrix.config.org }} - R_IMAGE: ${{ matrix.config.image }} - R_TAG: ${{ matrix.config.tag }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Setup Python - uses: actions/setup-python@v7 - with: - python-version: 3.12 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - source ci/scripts/util_enable_core_dumps.sh - # Don't set a TZ here to test that case. These builds will have the following warning in them: - # System has not been booted with systemd as init system (PID 1). Can't operate. - # Failed to connect to bus: Host is down - archery docker run -e TZ="" r - - name: Dump install logs - run: cat r/check/arrow.Rcheck/00install.out - if: always() - - name: Dump test logs - run: cat r/check/arrow.Rcheck/tests/testthat.Rout* - if: always() - - name: Save the test output - if: always() - uses: actions/upload-artifact@v7 - with: - name: test-output-bundled - path: r/check/arrow.Rcheck/tests/testthat.Rout* - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push r - - windows-cpp: - name: AMD64 Windows C++ RTools ${{ matrix.config.rtools }} ${{ matrix.config.arch }} - runs-on: windows-2022 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - config: - - { rtools: 40, arch: 'ucrt64' } - steps: - - run: git config --global core.autocrlf false - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - - name: Setup ccache - shell: bash - run: | - ci/scripts/ccache_setup.sh - echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV - - name: Cache ccache - uses: actions/cache@v6 - with: - path: ccache - key: r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}-${{ github.run_id }} - restore-keys: | - r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- - r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}- - - uses: r-lib/actions/setup-r@v2 - with: - # Note: RTools must be 40 here because RTools40 + ucrt is how we build the Arrow C++ - # static library. The R is not used here but R 4.1 was the last R to use - # Rtools40. - r-version: "4.1" - rtools-version: ${{ matrix.config.rtools }} - Ncpus: 2 - - name: Build Arrow C++ - shell: bash - env: - MINGW_ARCH: ${{ matrix.config.arch }} - RTOOLS_VERSION: ${{ matrix.config.rtools }} - run: ci/scripts/r_windows_build.sh - - name: Rename libarrow.zip - # So that they're unique when multiple are downloaded in the next step - shell: bash - run: mv libarrow.zip libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip - - uses: actions/upload-artifact@v7 - with: - name: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip - path: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip - - windows-r: - needs: [windows-cpp] - name: AMD64 Windows R ${{ matrix.config.rversion }} - runs-on: windows-2022 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - config: - - { rversion: "release" } - - env: - ARROW_R_CXXFLAGS: "-Werror" - _R_CHECK_TESTS_NLINES_: 0 - steps: - - run: git config --global core.autocrlf false - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - # This must be done before r-lib/actions/setup-r because curl in - # Rtools doesn't work on non Rtools' MSYS2 environment. If we - # use "shell: bash" after r-lib/actions/setup-r, bash in Rtools - # is used on non Rtools' MSYS2 environment. - - name: Install MinIO - shell: bash - run: | - mkdir -p "$HOME/.local/bin" - ci/scripts/install_minio.sh latest "$HOME/.local" - echo "$HOME/.local/bin" >> $GITHUB_PATH - - name: Download Timezone Database - shell: bash - run: | - # RTools 40 uses GCC 8.x which does not support C++20 chrono timezones, - # so Arrow uses the vendored date library which requires tzdata - ci/scripts/download_tz_database.sh - - run: mkdir r/windows - - name: Download artifacts - uses: actions/download-artifact@v8 - with: - name: libarrow-rtools40-ucrt64.zip - path: r/windows - - name: Unzip and rezip libarrows - shell: bash - run: | - cd r/windows - ls *.zip | xargs -n 1 unzip -uo - rm -rf *.zip - - uses: r-lib/actions/setup-r@v2 - with: - r-version: ${{ matrix.config.rversion }} - Ncpus: 2 - - uses: r-lib/actions/setup-r-dependencies@v2 - env: - GITHUB_PAT: "${{ github.token }}" - with: - # For some arcane reason caching does not work on the windows runners - # most likely due to https://github.com/actions/cache/issues/815 - cache: false - working-directory: 'r' - extra-packages: | - any::rcmdcheck - any::cyclocomp - # TODO(ARROW-17149): figure out why the GCS tests are hanging on Windows - # - name: Install Google Cloud Storage Testbench - # shell: bash - # run: ci/scripts/install_gcs_testbench.sh default - - name: Check - shell: Rscript {0} - run: | - # Because we do R CMD build and r/windows is in .Rbuildignore, - # assemble the libarrow.zip file and pass it as an env var - setwd("r/windows") - zip("libarrow.zip", ".") - setwd("..") - - Sys.setenv( - RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"), "r", "windows", "libarrow.zip"), - MAKEFLAGS = paste0("-j", parallel::detectCores()), - ARROW_R_DEV = TRUE, - "_R_CHECK_FORCE_SUGGESTS_" = FALSE, - "_R_CHECK_STOP_ON_INVALID_NUMERIC_VERSION_INPUTS_" = TRUE, - "_R_CHECK_DONTTEST_EXAMPLES_" = TRUE - ) - rcmdcheck::rcmdcheck(".", - build_args = '--no-build-vignettes', - args = c('--no-manual', '--as-cran', '--ignore-vignettes'), - error_on = 'warning', - check_dir = 'check', - timeout = 3600 - ) - - name: Dump install logs - shell: cmd - run: cat r/check/arrow.Rcheck/00install.out - if: always() - - name: Dump test logs - shell: bash - run: find r/check -name 'testthat.Rout*' -exec cat '{}' \; || true - if: always() diff --git a/.github/workflows/r_extra.yml b/.github/workflows/r_extra.yml deleted file mode 100644 index 3209e08bc4d5..000000000000 --- a/.github/workflows/r_extra.yml +++ /dev/null @@ -1,215 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: R Extra - -on: - push: - branches: - - '**' - - '!dependabot/**' - paths: - - '.dockerignore' - - '.github/workflows/check_labels.yml' - - '.github/workflows/r_extra.yml' - - '.github/workflows/report_ci.yml' - - 'ci/docker/**' - - 'ci/etc/rprofile' - - 'ci/scripts/PKGBUILD' - - 'ci/scripts/cpp_*.sh' - - 'ci/scripts/install_minio.sh' - - 'ci/scripts/r_*.sh' - - 'cpp/**' - - 'compose.yaml' - - 'dev/archery/archery/**' - - 'r/**' - tags: - - '**' - pull_request: - # This trigger ensures that the `check_labels` workflow runs on manual label changes - types: - - labeled - - opened - - reopened - - synchronize - schedule: - - cron: | - 0 4 * * * - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - actions: read - contents: read - pull-requests: read - -jobs: - check-labels: - uses: ./.github/workflows/check_labels.yml - with: - parent-workflow: r_extra - - check-enabled: - # Check whether the CI builds in this workflow need to be run - needs: check-labels - runs-on: ubuntu-latest - outputs: - is_enabled: ${{ steps.set_enabled.outputs.is_enabled }} - steps: - - id: set_enabled - if: >- - github.ref_type == 'tag' || - (github.event_name == 'schedule' && github.repository == 'apache/arrow') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: R') - run: echo "is_enabled=true" >> "$GITHUB_OUTPUT" - - docker: - needs: check-enabled - if: needs.check-enabled.outputs.is_enabled == 'true' - name: ${{ matrix.title }} - runs-on: ${{ matrix.runs-on }} - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - include: - - envs: - - R_IMAGE=r-ver - - R_ORG=rocker - image: r - runs-on: ubuntu-latest - title: Rocker R-ver - - envs: - - R_IMAGE=r-ver - - R_ORG=rocker - image: r - runs-on: ubuntu-latest - run-options: >- - -e ARROW_DEPENDENCY_SOURCE=BUNDLED - title: Rocker R-ver - Bundled Dependencies - - envs: - - R_IMAGE=r-ver - - R_ORG=rocker - image: r - runs-on: ubuntu-latest - run-options: >- - -e LIBARROW_MINIMAL=TRUE - title: Rocker R-ver - Minimal - - envs: - - R_IMAGE=r-ver - - R_ORG=rocker - image: r - runs-on: ubuntu-latest - run-options: >- - -e ARROW_OFFLINE_BUILD=true - title: Rocker R-ver - Offline build - - envs: - - R_IMAGE=ubuntu-release - - R_ORG=rhub - image: r - runs-on: ubuntu-latest - title: Rhub - - envs: - - R_IMAGE=ubuntu-gcc15 - - R_ORG=rhub - image: r - runs-on: ubuntu-latest - run-options: >- - -e INSTALL_ARGS=--use-LTO - title: Rhub GCC 15 with LTO - - envs: - - R_IMAGE=r-base - - R_ORG=rstudio - - R_TAG=4.5-jammy - image: r - runs-on: ubuntu-latest - title: RStudio R-Base 4.5 - env: - ARCHERY_DEBUG: 1 - ARROW_ENABLE_TIMING_TESTS: OFF - DOCKER_VOLUME_PREFIX: ".docker/" - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Cache Docker Volumes - uses: actions/cache@v6 - with: - path: .docker - key: extra-${{ matrix.image }}-${{ hashFiles('cpp/**') }} - restore-keys: extra-${{ matrix.image }}- - - name: Setup Python - uses: actions/setup-python@v7 - with: - python-version: 3 - - name: Setup Archery - run: python3 -m pip install -e dev/archery[docker] - - name: Execute Docker Build - continue-on-error: ${{ matrix.continue-on-error || false }} - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - ENVS: ${{ toJSON(matrix.envs) }} - run: | - source ci/scripts/util_enable_core_dumps.sh - if [ "${ENVS}" != "null" ]; then - echo "${ENVS}" | jq -r '.[]' | while read env; do - echo "${env}" >> .env - done - fi - archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }} - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - name: Dump install logs - run: cat r/check/arrow.Rcheck/00install.out - if: always() - continue-on-error: true - - name: Dump test logs - run: cat r/check/arrow.Rcheck/tests/testthat.Rout* - if: always() - continue-on-error: true - - name: Save the test output - if: always() - uses: actions/upload-artifact@v7 - with: - name: test-output-{{ "${{ matrix.config.r_image }}" }} - path: r/check/arrow.Rcheck/tests/testthat.Rout* - continue-on-error: true - - report-extra-cpp: - if: github.event_name == 'schedule' && always() - needs: - - docker - uses: ./.github/workflows/report_ci.yml - secrets: - ARROW_SMTP_PASSWORD: ${{ secrets.ARROW_SMTP_PASSWORD }} - ARROW_ZULIP_WEBHOOK: ${{ secrets.ARROW_ZULIP_WEBHOOK }} diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 639315c0c9b4..194ca94d7a23 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -68,71 +68,6 @@ env: jobs: - ubuntu: - name: AMD64 Ubuntu ${{ matrix.ubuntu }} GLib & Ruby - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - ubuntu: - - 22.04 - env: - UBUNTU: ${{ matrix.ubuntu }} - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Restore Docker Volumes - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: .docker - key: ruby-ubuntu-${{ matrix.ubuntu }} - - name: Setup Python - uses: actions/setup-python@v7 - with: - python-version: 3.12 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - source ci/scripts/util_enable_core_dumps.sh - archery docker run \ - -e ARROW_FLIGHT=ON \ - -e ARROW_FLIGHT_SQL=ON \ - -e ARROW_GCS=ON \ - -e nlohmann_json_SOURCE=BUNDLED \ - -e Protobuf_SOURCE=BUNDLED \ - -e gRPC_SOURCE=BUNDLED \ - ubuntu-ruby - - name: Save Docker Volumes - if: ${{ !cancelled() }} - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: .docker - key: ruby-ubuntu-${{ matrix.ubuntu }} - include-hidden-files: true - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - shell: bash - run: archery docker push ubuntu-ruby - macos: name: ARM64 macOS GLib & Ruby runs-on: macos-latest @@ -216,258 +151,3 @@ jobs: shell: bash run: ci/scripts/ruby_test.sh $(pwd) $(pwd)/build - windows-mingw: - name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} GLib & Ruby - runs-on: windows-2022 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - mingw-n-bits: - - 64 - ruby-version: - - ruby - env: - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: OFF - ARROW_BUILD_UTILITIES: OFF - ARROW_BUILD_TYPE: release - ARROW_DATASET: ON - ARROW_FLIGHT: ON - ARROW_FLIGHT_SQL: ON - ARROW_GANDIVA: ON - ARROW_GCS: ON - ARROW_HDFS: OFF - ARROW_HOME: /ucrt${{ matrix.mingw-n-bits }} - ARROW_JEMALLOC: OFF - ARROW_PARQUET: ON - ARROW_PYTHON: OFF - ARROW_S3: ON - ARROW_USE_GLOG: OFF - ARROW_WITH_BROTLI: ON - ARROW_WITH_BZ2: ON - ARROW_WITH_LZ4: ON - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - ARROW_CMAKE_ARGS: >- - -DARROW_PACKAGE_PREFIX=/ucrt${{ matrix.mingw-n-bits }} - -DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON - CMAKE_UNITY_BUILD: ON - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Setup Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.ruby-version }} - - name: Setup MSYS2 - run: | - ridk exec bash ci\scripts\msys2_setup.sh ruby - - name: Restore ccache - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - # CCACHE_DIR is set by `msys2_setup.sh` - path: ${{ env.CCACHE_DIR }} - key: ruby-ccache-ucrt${{ matrix.mingw-n-bits }} - - name: Show ccache config - run: | - ccache --show-config - - name: Build C++ - run: | - $Env:CMAKE_BUILD_PARALLEL_LEVEL = $Env:NUMBER_OF_PROCESSORS - $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" - $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" - $ErrorActionPreference = "Continue" - ridk exec bash ci\scripts\cpp_build.sh "${source_dir}" "${build_dir}" - - name: Build GLib - run: | - $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" - $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" - $ErrorActionPreference = "Continue" - ridk exec bash ci\scripts\c_glib_build.sh "${source_dir}" "${build_dir}" - - name: Save ccache - if: ${{ !cancelled() }} - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ env.CCACHE_DIR }} - key: ruby-ccache-ucrt${{ matrix.mingw-n-bits }} - - name: RubyGems info - id: rubygems-info - run: | - Write-Output "gem-dir=$(ridk exec gem env gemdir)" | ` - Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - - name: Cache RubyGems - uses: actions/cache@v6 - with: - path: ${{ steps.rubygems-info.outputs.gem-dir }} - key: ruby-rubygems-ucrt${{ matrix.mingw-n-bits }}-${{ hashFiles('**/Gemfile', 'ruby/*/*.gemspec') }} - restore-keys: ruby-rubygems-ucrt${{ matrix.mingw-n-bits }}- - - name: Install test dependencies - run: | - bundle install --gemfile c_glib\Gemfile - bundle install --gemfile ruby\Gemfile - Get-ChildItem ruby\*\Gemfile | ` - ForEach-Object {bundle install --gemfile $_} - - name: Test GLib - run: | - $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" - $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" - $ErrorActionPreference = "Continue" - ridk exec bash ci\scripts\c_glib_test.sh "${source_dir}" "${build_dir}" - - name: Test Ruby - run: | - $Env:PKG_CONFIG_PATH = ` - "$(ridk exec cygpath --absolute --windows "${Env:ARROW_HOME}/lib/pkgconfig")" - $Env:GI_TYPELIB_PATH = ` - "$(ridk exec cygpath --absolute --windows "${Env:ARROW_HOME}/lib/girepository-1.0")" - $Env:RUBYOPTS = "-rdevkit" - $Env:MAKE = "ridk exec make" - $ErrorActionPreference = "Continue" - rake -f ruby\Rakefile - - windows-msvc: - name: AMD64 Windows MSVC GLib - runs-on: windows-2022 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 240 - strategy: - fail-fast: false - env: - ARROW_ACERO: ON - ARROW_BOOST_USE_SHARED: OFF - ARROW_BUILD_BENCHMARKS: OFF - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: OFF - ARROW_DATASET: ON - ARROW_DEPENDENCY_SOURCE: VCPKG - ARROW_DEPENDENCY_USE_SHARED: OFF - ARROW_FLIGHT: ON - ARROW_FLIGHT_SQL: ON - ARROW_GANDIVA: OFF - ARROW_GLIB_VAPI: "false" - ARROW_HDFS: OFF - ARROW_HOME: "${{ github.workspace }}/dist" - ARROW_JEMALLOC: OFF - ARROW_MIMALLOC: ON - ARROW_ORC: OFF - ARROW_PARQUET: ON - ARROW_SUBSTRAIT: OFF - ARROW_USE_GLOG: OFF - ARROW_VERBOSE_THIRDPARTY_BUILD: OFF - ARROW_WITH_BROTLI: OFF - ARROW_WITH_BZ2: OFF - ARROW_WITH_LZ4: OFF - ARROW_WITH_OPENTELEMETRY: OFF - ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: ON - ARROW_WITH_ZSTD: ON - CCACHE_DIR: C:/ccache - CMAKE_CXX_STANDARD: "20" - CMAKE_GENERATOR: Ninja - CMAKE_INSTALL_PREFIX: "${{ github.workspace }}/dist" - # ccache does not support PDB files (https://github.com/ccache/ccache/issues/1040) - CMAKE_MSVC_DEBUG_INFORMATION_FORMAT: Embedded - CMAKE_UNITY_BUILD: ON - VCPKG_BINARY_SOURCES: 'clear;nugettimeout,600;nuget,GitHub,readwrite' - VCPKG_DEFAULT_TRIPLET: x64-windows - permissions: - packages: write - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - submodules: recursive - - name: Install meson - run: | - python -m pip install meson - - name: Install ccache - shell: bash - run: | - ci/scripts/install_ccache.sh 4.13.6 /usr - - name: Setup ccache - shell: bash - run: | - ci/scripts/ccache_setup.sh - - name: Restore ccache - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ env.CCACHE_DIR }} - key: ruby-ccache-msvc - - name: Show ccache config - run: | - ccache --show-config - - name: Checkout vcpkg - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 0 - path: vcpkg - repository: microsoft/vcpkg - - name: Bootstrap vcpkg - run: | - vcpkg\bootstrap-vcpkg.bat - $VCPKG_ROOT = $(Resolve-Path -LiteralPath "vcpkg").ToString() - Write-Output ${VCPKG_ROOT} | ` - Out-File -FilePath ${Env:GITHUB_PATH} -Encoding utf8 -Append - Write-Output "VCPKG_ROOT=${VCPKG_ROOT}" | ` - Out-File -FilePath ${Env:GITHUB_ENV} -Encoding utf8 -Append - - name: Setup NuGet credentials for vcpkg caching - shell: bash - run: | - $(vcpkg fetch nuget | tail -n 1) \ - sources add \ - -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json" \ - -storepasswordincleartext \ - -name "GitHub" \ - -username "$GITHUB_REPOSITORY_OWNER" \ - -password "${{ secrets.GITHUB_TOKEN }}" - $(vcpkg fetch nuget | tail -n 1) \ - setapikey "${{ secrets.GITHUB_TOKEN }}" \ - -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json" - - name: Build C++ - shell: cmd - run: | - set VCPKG_ROOT_KEEP=%VCPKG_ROOT% - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - set VCPKG_ROOT=%VCPKG_ROOT_KEEP% - bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build" - - name: Build GLib - shell: cmd - run: | - set VCPKG_ROOT_KEEP=%VCPKG_ROOT% - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - set VCPKG_ROOT=%VCPKG_ROOT_KEEP% - bash -c "ci/scripts/c_glib_build.sh $(pwd) $(pwd)/build" - - name: Save ccache - if: ${{ !cancelled() }} - continue-on-error: true - uses: apache/infrastructure-actions/stash/save@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ env.CCACHE_DIR }} - key: ruby-ccache-msvc diff --git a/c_glib/test/run-test.sh b/c_glib/test/run-test.sh index 8b1868942073..977710454ae2 100755 --- a/c_glib/test/run-test.sh +++ b/c_glib/test/run-test.sh @@ -71,6 +71,7 @@ case "${DEBUGGER}" in ;; "lldb") DEBUGGER_ARGS+=(--one-line "env DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}") + DEBUGGER_ARGS+=(--batch -o run -o bt) DEBUGGER_ARGS+=(--) ;; esac diff --git a/ci/scripts/c_glib_test.sh b/ci/scripts/c_glib_test.sh index dd30ebb2f403..867a3ff403d8 100755 --- a/ci/scripts/c_glib_test.sh +++ b/ci/scripts/c_glib_test.sh @@ -40,7 +40,13 @@ run_test_args=() if [ -n "${RUNNER_DEBUG}" ]; then run_test_args+=(-v) fi -ruby test/run-test.rb "${run_test_args[@]}" +#ruby test/run-test.rb "${run_test_args[@]}" +pushd "${build_dir}" +for i in {1..10}; do echo "*** TRY $i ***" +DEBUGGER="lldb" ${source_dir}/test/run-test.sh -t TestFlightSQLClient || true +done +popd + if [[ "$(uname -s)" == "Linux" ]]; then # TODO(kszucs): on osx it fails to load 'lgi.corelgilua51' despite that lgi diff --git a/ruby/red-arrow-flight/Rakefile b/ruby/red-arrow-flight/Rakefile index 77c9f81e995c..24aa27d7b974 100644 --- a/ruby/red-arrow-flight/Rakefile +++ b/ruby/red-arrow-flight/Rakefile @@ -35,7 +35,7 @@ task :test do ruby("-S", "rake") end args = [] - args << "-v" if ENV["RUNNER_DEBUG"] + args << "-v" ruby("test/run-test.rb", *args) end end diff --git a/ruby/red-arrow-flight/test/helper/server.rb b/ruby/red-arrow-flight/test/helper/server.rb index 1ea4855897b0..c8bc4d1b620a 100644 --- a/ruby/red-arrow-flight/test/helper/server.rb +++ b/ruby/red-arrow-flight/test/helper/server.rb @@ -25,25 +25,35 @@ class Server < ArrowFlight::Server private def virtual_do_list_flights(context, criteria) + puts "#{__FILE__}:#{__LINE__}" generator = InfoGenerator.new + puts "#{__FILE__}:#{__LINE__}" [generator.page_view] end def virtual_do_do_get(context, ticket) + puts "#{__FILE__}:#{__LINE__}" generator = InfoGenerator.new + puts "#{__FILE__}:#{__LINE__}" if ticket.data.to_s != generator.page_view_ticket raise Arrow::Error::Invalid.new("invalid ticket") end + puts "#{__FILE__}:#{__LINE__}" table = generator.page_view_table + puts "#{__FILE__}:#{__LINE__}" ArrowFlight::RecordBatchStream.new(table) end def virtual_do_do_put(context, reader, writer) + puts "#{__FILE__}:#{__LINE__}" @uploaded_table = reader.read_all + puts "#{__FILE__}:#{__LINE__}" writer.write(Arrow::Buffer.new("done")) + puts "#{__FILE__}:#{__LINE__}" if @uploaded_table.n_rows.zero? raise Arrow::Error::Invalid.new("empty table") end + puts "#{__FILE__}:#{__LINE__}" true end end