Implement a C API for metatomic

.github/workflows/build-wheels.yml

@@ -32,9 +32,9 @@ jobs:
     strategy:
       matrix:
         include:
-          - image: gcc11-manylinux_2_28_x86_64
+          - image: rustc-manylinux_2_28_x86_64
             os: ubuntu-24.04
-          - image: gcc11-manylinux_2_28_aarch64
+          - image: rustc-manylinux_2_28_aarch64
             os: ubuntu-24.04-arm
     steps:
       - uses: actions/checkout@v7
@@ -56,6 +56,84 @@ jobs:
           retention-days: 1
           compression-level: 0
 
+  build-core-wheels:
+    needs: build-docker-images
+    runs-on: ${{ matrix.os }}
+    name: ${{ matrix.name }}
+    strategy:
+      matrix:
+        include:
+          - name: x86_64 Linux
+            os: ubuntu-24.04
+            rust-target: x86_64-unknown-linux-gnu
+            cibw-arch: x86_64
+          - name: arm64 Linux
+            os: ubuntu-24.04-arm
+            rust-target: aarch64-unknown-linux-gnu
+            cibw-arch: aarch64
+          - name: arm64 macOS
+            os: macos-15
+            rust-target: aarch64-apple-darwin
+            cibw-arch: arm64
+          - name: x86_64 Windows
+            os: windows-2022
+            rust-target: x86_64-pc-windows-msvc
+            cibw-arch: AMD64
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - name: setup rust
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+          target: ${{ matrix.rust-target }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.14"
+
+      - name: install dependencies
+        run: python -m pip install cibuildwheel twine
+
+      - name: download custom manylinux docker image
+        if: startsWith(matrix.os, 'ubuntu')
+        uses: actions/download-artifact@v8
+        with:
+          name: docker-image-rustc-manylinux_2_28_${{ matrix.cibw-arch }}
+          path: ${{ runner.temp }}
+
+      - name: load custom manylinux docker image
+        if: startsWith(matrix.os, 'ubuntu')
+        run: docker load --input ${{ runner.temp }}/rustc-manylinux_2_28_${{ matrix.cibw-arch }}.tar
+
+      - name: build metatomic-core wheel
+        run: python -m cibuildwheel python/metatomic_core
+        env:
+          CIBW_BUILD: cp311-*
+          CIBW_SKIP: "*musllinux*"
+          CIBW_ARCHS: ${{ matrix.cibw-arch }}
+          CIBW_BUILD_VERBOSITY: 1
+          CIBW_MANYLINUX_X86_64_IMAGE: rustc-manylinux_2_28_x86_64
+          CIBW_MANYLINUX_AARCH64_IMAGE: rustc-manylinux_2_28_aarch64
+          CIBW_ENVIRONMENT: >
+            MACOSX_DEPLOYMENT_TARGET=11
+          # do not complain for missing shared libraries provided by other packages
+          CIBW_REPAIR_WHEEL_COMMAND_MACOS: |
+            delocate-wheel --ignore-missing-dependencies --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: |
+            auditwheel repair --exclude libmetatensor.so -w {dest_dir} {wheel}
+
+      - name: check wheels with twine
+        run: twine check wheelhouse/*.whl
+
+      - uses: actions/upload-artifact@v7
+        with:
+          name: core-wheel-${{ matrix.os }}-${{ matrix.cibw-arch }}
+          path: ./wheelhouse/*.whl
+
   build-torch-wheels:
     needs: build-docker-images
     runs-on: ${{ matrix.os }}
@@ -122,12 +200,12 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
         uses: actions/download-artifact@v8
         with:
-          name: docker-image-gcc11-manylinux_2_28_${{ matrix.cibw-arch }}
+          name: docker-image-rustc-manylinux_2_28_${{ matrix.cibw-arch }}
           path: ${{ runner.temp }}
 
       - name: load custom manylinux docker image
         if: startsWith(matrix.os, 'ubuntu')
-        run: docker load --input ${{ runner.temp }}/gcc11-manylinux_2_28_${{ matrix.cibw-arch }}.tar
+        run: docker load --input ${{ runner.temp }}/rustc-manylinux_2_28_${{ matrix.cibw-arch }}.tar
 
       - name: build metatomic-torch wheel
         run: python -m cibuildwheel python/metatomic_torch
@@ -136,10 +214,19 @@ jobs:
           CIBW_SKIP: "*musllinux*"
           CIBW_ARCHS: ${{ matrix.cibw-arch }}
           CIBW_BUILD_VERBOSITY: 1
-          CIBW_MANYLINUX_X86_64_IMAGE: gcc11-manylinux_2_28_x86_64
-          CIBW_MANYLINUX_AARCH64_IMAGE: gcc11-manylinux_2_28_aarch64
+          CIBW_MANYLINUX_X86_64_IMAGE: rustc-manylinux_2_28_x86_64
+          CIBW_MANYLINUX_AARCH64_IMAGE: rustc-manylinux_2_28_aarch64
+          # METATOMIC_NO_LOCAL_DEPS is set to 1 when building a tag of
+          # metatomic-torch, which will force to use the version of
+          # metatomic-core already released on PyPI. Otherwise, this will use
+          # the version of metatomic-core from git checkout (in case there are
+          # unreleased breaking changes).
+          #
+          # This means that when releasing a breaking change in metatomic-core,
+          # the full release should be available on PyPI before pushing the new
+          # metatomic-torch tag.
           CIBW_ENVIRONMENT: >
-            METATOMIC_NO_LOCAL_DEPS=1
+            METATOMIC_NO_LOCAL_DEPS=${{ startsWith(github.ref, 'refs/tags/metatomic-torch-v') && '1' || '0' }}
             METATOMIC_TORCH_BUILD_WITH_TORCH_VERSION=${{ matrix.torch-version }}.*
             PIP_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cpu
             MACOSX_DEPLOYMENT_TARGET=11
@@ -242,6 +329,9 @@ jobs:
       - name: install dependencies
         run: python -m pip install wheel build twine
 
+      - name: build metatomic-core sdist
+        run: python -m build python/metatomic_core --sdist --outdir=dist/
+
       - name: build metatomic-torch sdist
         run: python -m build python/metatomic_torch --sdist --outdir=dist/
 
@@ -259,6 +349,7 @@ jobs:
 
       - name: create C++ tarballs
         run: |
+          ./scripts/package-core.sh dist/cxx/
           ./scripts/package-torch.sh dist/cxx/
 
       - uses: actions/upload-artifact@v7
@@ -271,12 +362,19 @@ jobs:
 
   merge-and-release:
     name: Merge and release wheels/sdists
-    needs: [merge-torch-wheels, build-others]
+    needs: [build-core-wheels, merge-torch-wheels, build-others]
     runs-on: ubuntu-24.04
     permissions:
       contents: write
       pull-requests: write
     steps:
+      - name: Download metatomic-core wheels
+        uses: actions/download-artifact@v8
+        with:
+          path: wheels
+          pattern: core-wheel-*
+          merge-multiple: true
+
       - name: Download metatomic-torch wheels
         uses: actions/download-artifact@v8
         with:
@@ -305,6 +403,17 @@ jobs:
           name: wheels
           description: ⚙️ Download Python wheels for this pull-request (you can install these with pip)
 
+      - name: upload to GitHub release (metatomic-core)
+        if: startsWith(github.ref, 'refs/tags/metatomic-core-v')
+        uses: softprops/action-gh-release@v3
+        with:
+          files: |
+            wheels/cxx/metatomic-core-cxx-*.tar.gz
+            wheels/metatomic_core-*
+          prerelease: ${{ contains(github.ref, '-rc') }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
       - name: upload to GitHub release (metatomic)
         if: startsWith(github.ref, 'refs/tags/metatomic-v')
         uses: softprops/action-gh-release@v3
@@ -348,8 +457,8 @@ jobs:
 
   test-build-external:
     # This checks building the wheels with external libraries. This setup is
-    # mainly used for the conda packages metatensor-*-python, which use the
-    # libmetatensor-* conda packages to provide the native code.
+    # mainly used for the conda packages metatomic-*-python, which use the
+    # libmetatomic-* conda packages to provide the native code.
     runs-on: ${{ matrix.os }}
     name: External libraries / ${{ matrix.os }}
     defaults:
@@ -375,6 +484,12 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: setup rust
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+          target: ${{ matrix.rust-target }}
+
       - name: setup Python
         uses: actions/setup-python@v7
         with:
@@ -393,9 +508,9 @@ jobs:
 
       - name: setup libmetatensor
         run: |
-          curl --location -O https://github.com/metatensor/metatensor/releases/download/metatensor-core-v0.2.3/metatensor-core-cxx-0.2.3.tar.gz
-          tar xf metatensor-core-cxx-0.2.3.tar.gz
-          cmake -B build-metatensor -S metatensor-core-cxx-0.2.3 \
+          curl --location -O https://github.com/metatensor/metatensor/releases/download/metatensor-core-v0.2.4/metatensor-core-cxx-0.2.4.tar.gz
+          tar xf metatensor-core-cxx-0.2.4.tar.gz
+          cmake -B build-metatensor -S metatensor-core-cxx-0.2.4 \
                 -DMETATENSOR_INSTALL_BOTH_STATIC_SHARED=OFF \
                 -DCMAKE_INSTALL_PREFIX=$CMAKE_PREFIX_PATH \
                 -DCMAKE_BUILD_TYPE=Debug
@@ -412,6 +527,15 @@ jobs:
           cmake --build build-metatensor-torch --config Debug
           cmake --install build-metatensor-torch --config Debug
 
+      - name: build libmetatomic
+        run: |
+          cmake -B build-metatomic-core -S metatomic-core \
+                -DMETATOMIC_INSTALL_BOTH_STATIC_SHARED=OFF \
+                -DCMAKE_INSTALL_PREFIX=$CMAKE_PREFIX_PATH \
+                -DCMAKE_BUILD_TYPE=Debug
+          cmake --build build-metatomic-core --config Debug
+          cmake --install build-metatomic-core --config Debug
+
       - name: build libmetatomic-torch
         run: |
           cmake -B build-metatomic-torch -S metatomic-torch \
@@ -420,6 +544,14 @@ jobs:
           cmake --build build-metatomic-torch --config Debug
           cmake --install build-metatomic-torch --config Debug
 
+      - name: build metatomic-core wheels
+        run: |
+          python -m build python/metatomic_core --wheel --outdir=dist/
+          # check that the wheel is using an external library
+          unzip -l dist/metatomic_core*.whl | grep "_external.py"
+        env:
+          METATOMIC_CORE_PYTHON_USE_EXTERNAL_LIB: "ON"
+
       - name: build metatomic-torch wheels
         run: |
           pip install build

.github/workflows/python-tests.yml

@@ -0,0 +1,95 @@
+name: Python tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    # Check all PR
+
+concurrency:
+  group: python-tests-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+jobs:
+  python-tests:
+    runs-on: ${{ matrix.os }}
+    name: ${{ matrix.os }} / Python ${{ matrix.python-version }} / Torch ${{ matrix.torch-version }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-24.04
+            python-version: "3.11"
+            torch-version: "2.3"
+            numpy-version-pin: "<2.0"
+          - os: ubuntu-24.04
+            python-version: "3.11"
+            torch-version: "2.13"
+          - os: ubuntu-24.04
+            # TorchScript is no longer supported in Python 3.14
+            # so we keep a test with 3.13 to make sure this doesn't break
+            python-version: "3.13"
+            torch-version: "2.13"
+          - os: ubuntu-24.04
+            python-version: "3.14"
+            torch-version: "2.13"
+          - os: macos-15
+            python-version: "3.14"
+            torch-version: "2.13"
+          - os: windows-2022
+            python-version: "3.14"
+            torch-version: "2.13"
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - name: setup Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: setup rust
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+
+      - name: Setup sccache
+        if: ${{ !env.ACT }}
+        uses: mozilla-actions/sccache-action@v0.0.10
+        with:
+          version: "v0.10.0"
+
+      - name: setup MSVC command prompt
+        uses: ilammy/msvc-dev-cmd@v1
+
+      - name: Setup sccache environnement variables
+        if: ${{ !env.ACT }}
+        run: |
+          echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
+          echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+          echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> $GITHUB_ENV
+          echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> $GITHUB_ENV
+
+      - name: install tests dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install tox coverage
+
+      - name: run tests
+        run: tox -e lint,core-tests,torch-tests,docs-tests
+        env:
+          PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu
+          METATOMIC_TESTS_TORCH_VERSION: ${{ matrix.torch-version }}
+
+      - name: combine Python coverage files
+        shell: bash
+        run: |
+            coverage combine .tox/*/.coverage
+            coverage xml
+
+      - name: upload to codecov.io
+        uses: codecov/codecov-action@v7
+        with:
+          fail_ci_if_error: true
+          files: coverage.xml
+          token: ${{ secrets.CODECOV_TOKEN }}