Real python package

.github/workflows/tests.yml

@@ -2,60 +2,75 @@ name: tests
 
 on: [push, pull_request]
 
-env:
-  DOCKER_REGISTRY: ghcr.io/commaai
-  RUN: docker run -e PYTHONWARNINGS=error --shm-size 1G --name msgq msgq /bin/sh -c
-  RUN_NAMED: docker run -e PYTHONWARNINGS=error --shm-size 1G --rm msgq /bin/sh -c
-  CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm msgqci /bin/bash -c
-  BUILD: docker buildx build --pull --load --cache-to type=inline --cache-from $DOCKER_REGISTRY/msgq:latest -t msgq -f Dockerfile .
-  PYTHONWARNINGS: error
-
 jobs:
-  build:
-    name: build
-    runs-on: ubuntu-latest
+
+  static_analysis:
+    runs-on: ubuntu-24.04
     steps:
-    - uses: actions/checkout@v3
-    - name: Build docker image
-      run: eval "$BUILD"
-    - name: Push to dockerhub
-      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/msgq'
-      run: |
-        docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-        docker tag msgq $DOCKER_REGISTRY/msgq:latest
-        docker push $DOCKER_REGISTRY/msgq:latest
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+    - name: Installing requirements
+      run:
+        python -m pip install -r requirements.txt
+    - name: Installing ubuntu requirements
+      run: "sudo apt-get install -y cppcheck"
+    - name: Static analysis
+      run: pre-commit run --all
+
+  scons_test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: ["ubuntu-20.04", "ubuntu-24.04", "macos-14"]
+        python: ["3.11", "3.12"]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Installing python requirements
+      run:
+        python -m pip install -r requirements.txt
+    - name: Installing ubuntu requirements
+      if: ${{startsWith(matrix.os, 'ubuntu')}}
+      run:
+        scripts/ubuntu_dependencies.sh
+    - name: Installing macos requirements
+      if: ${{startsWith(matrix.os, 'macos')}}
+      run:
+        scripts/macos_dependencies.sh
+    - name: Building
+      run: scons -j$(nproc || sysctl -n hw.logicalcpu)
+
 
-  unit_tests:
-    name: unit tests
-    runs-on: ubuntu-latest
+  full_tests:
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        os: ["ubuntu-20.04", "ubuntu-24.04"]
+        python: ["3.11", "3.12"]
         flags: ['', '--asan', '--ubsan']
         backend: ['MSGQ', 'ZMQ']
     steps:
-    - uses: actions/checkout@v3
-    - name: Build docker image
-      run: eval "$BUILD"
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Installing ubuntu requirements
+      run:
+        scripts/ubuntu_dependencies.sh
+    - name: Building msgq
+      run: |
+        pip3 install -e .[dev]
+    - name: Python tests
+      run: ${{ matrix.backend }}=1 pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append
     - name: C++ tests
       run: |
-        $RUN "export ${{ matrix.backend }}=1 && \
-              scons ${{ matrix.flags }} -j$(nproc) && \
-              msgq/test_runner && \
-              msgq/visionipc/test_runner"
-    - name: python tests
-      run: $RUN_NAMED "${{ matrix.backend }}=1 coverage run -m pytest"
+        export ${{ matrix.backend }}=1
+        scons ${{ matrix.flags }} -j$(nproc)
+        msgq/test_runner
+        msgq/visionipc/test_runner
     - name: Upload coverage
-      run: |
-        docker commit msgq msgqci
-        $CI_RUN "cd /project/msgq && bash <(curl -s https://codecov.io/bash) -v -F unit_tests_${{ matrix.backend }}"
-
-  static_analysis:
-    name: static analysis
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-    - name: Build docker image
-      run: eval "$BUILD"
-    - name: Static analysis
-      # TODO: a package pre-commit installs has a warning, remove the unset once that's fixed
-      run: $RUN "git init && git add -A && unset PYTHONWARNINGS && pre-commit run --all"
+      run: "bash <(curl -s https://codecov.io/bash) -v -F unit_tests_${{ matrix.backend }}"

.github/workflows/wheels.yml

@@ -0,0 +1,50 @@
+name: wheels
+
+on: [push, pull_request]
+
+jobs:
+  wheels:
+    runs-on: ${{ matrix.platform.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        platform: [ {os: "ubuntu-latest", target: "manylinux_x86_64",  arch: "x86_64"},
+                    {os: "ubuntu-latest", target: "manylinux_aarch64", arch: "aarch64"},
+                    {os: "macos-14",      target: "macosx_arm64",      arch: "arm64"} ]
+        python:   [ {cp: "cp311", py: "3.11"}, {cp: "cp312", py: "3.12"} ]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up QEMU
+      if: runner.os == 'Linux'
+      uses: docker/setup-qemu-action@v3
+      with:
+        platforms: all
+
+    - name: Building wheel
+      uses: pypa/[email protected]
+      env:
+        CIBW_BUILD: "${{ matrix.python.cp }}-${{ matrix.platform.target }}"
+        CIBW_ARCHS: "${{ matrix.platform.arch }}"
+        CIBW_BEFORE_ALL_LINUX: "bash {project}/scripts/manylinux_dependencies.sh"
+        CIBW_BEFORE_BUILD_MACOS: "bash {project}/scripts/macos_dependencies.sh"
+        CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
+        CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"
+        MACOSX_DEPLOYMENT_TARGET: "14.0"
+
+    - uses: actions/setup-python@v5
+      if: ${{ matrix.platform.arch != 'aarch64' }}
+      with:
+        python-version: ${{ matrix.python.py }}
+
+    - name: Installing the wheel
+      if: ${{ matrix.platform.arch != 'aarch64' }}
+      run: |
+        pip install --break-system-packages ./wheelhouse/*.whl
+
+    - name: Saving wheel
+      uses: actions/upload-artifact@v4
+      with:
+        name: cibw-wheels-${{ matrix.platform.target }}-${{ matrix.python.cp }}
+        path: ./wheelhouse/*.whl

.gitignore

@@ -17,3 +17,7 @@ services.h
 .sconsign.dblite
 libcereal_shared.*
 .mypy_cache/
+msgq.egg-info
+build/
+dist/
+catch2/

MANIFEST.in

@@ -0,0 +1,4 @@
+exclude msgq/**/*.o
+include SConscript
+include SConstruct
+recursive-include site_scons *

SConstruct

@@ -61,6 +61,7 @@ env = Environment(
     "-Werror",
     "-Wshadow",
     "-Wno-vla-cxx-extension",
+    "-Wno-unknown-warning-option",
   ] + ccflags,
   LDFLAGS=ldflags,
   LINKFLAGS=ldflags,

pyproject.toml

@@ -1,3 +1,30 @@
+[project]
+name = "msgq"
+requires-python = ">= 3.11"
+version = "0.1.0"
+
+dependencies = [
+  "numpy",
+  "Cython",
+  "pycapnp",
+  "setuptools",
+]
+
+[project.optional-dependencies]
+dev = [
+  "scons",
+  "pre-commit",
+  "ruff",
+  "parameterized",
+  "coverage",
+  "pytest",
+  "pytest-cov",
+]
+
+[build-system]
+requires = ["setuptools","scons","Cython","numpy"]
+build-backend = "setuptools.build_meta"
+
 # https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml
 [tool.ruff]
 lint.select = ["E", "F", "W", "PIE", "C4", "ISC", "RUF100", "A"]

requirements.txt

@@ -0,0 +1,50 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile --all-extras -o requirements.txt pyproject.toml
+cfgv==3.4.0
+    # via pre-commit
+coverage==7.6.0
+    # via
+    #   msgq (pyproject.toml)
+    #   pytest-cov
+cython==3.0.10
+    # via msgq (pyproject.toml)
+distlib==0.3.8
+    # via virtualenv
+filelock==3.15.4
+    # via virtualenv
+identify==2.6.0
+    # via pre-commit
+iniconfig==2.0.0
+    # via pytest
+nodeenv==1.9.1
+    # via pre-commit
+numpy==2.0.0
+    # via msgq (pyproject.toml)
+packaging==24.1
+    # via pytest
+parameterized==0.9.0
+    # via msgq (pyproject.toml)
+platformdirs==4.2.2
+    # via virtualenv
+pluggy==1.5.0
+    # via pytest
+pre-commit==3.7.1
+    # via msgq (pyproject.toml)
+pycapnp==2.0.0
+    # via msgq (pyproject.toml)
+pytest==8.2.2
+    # via
+    #   msgq (pyproject.toml)
+    #   pytest-cov
+pytest-cov==5.0.0
+    # via msgq (pyproject.toml)
+pyyaml==6.0.1
+    # via pre-commit
+ruff==0.5.3
+    # via msgq (pyproject.toml)
+scons==4.8.0
+    # via msgq (pyproject.toml)
+setuptools==71.0.4
+    # via msgq (pyproject.toml)
+virtualenv==20.26.3
+    # via pre-commit

scripts/macos_dependencies.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+ROOT="$(cd $DIR/../ && pwd)"
+ARCH=$(uname -m)
+
+if [[ $SHELL == "/bin/zsh" ]]; then
+  RC_FILE="$HOME/.zshrc"
+elif [[ $SHELL == "/bin/bash" ]]; then
+  RC_FILE="$HOME/.bash_profile"
+fi
+
+# Install brew if required
+if [[ $(command -v brew) == "" ]]; then
+  /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
+
+  if [[ $ARCH == "x86_64" ]]; then
+      echo 'eval "$(/usr/local/homebrew/bin/brew shellenv)"' >> $RC_FILE
+      eval "$(/usr/local/homebrew/bin/brew shellenv)"
+  else
+      echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> $RC_FILE
+      eval "$(/opt/homebrew/bin/brew shellenv)"
+  fi
+fi
+
+brew bundle --file=- <<-EOS
+brew "zeromq"
+cask "gcc-arm-embedded"
+brew "gcc@13"
+EOS
+
+if [[ -z "$NO_CATCH2" ]]; then
+  cd /tmp
+  git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git
+  cd Catch2
+  mv single_include/* "$ROOT"
+  rm -rf Catch2
+fi

scripts/manylinux_dependencies.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+dnf install -y clang opencl-headers ocl-icd-devel cppcheck wget
+
+wget https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz
+tar -xvf zeromq-4.3.5.tar.gz
+cd zeromq-4.3.5
+./configure
+make -j$(nproc)
+make install