Merge branch 'master' into 24.04

.github/workflows/build.yaml

@@ -0,0 +1,84 @@
+name: build
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: build kernel and system
+    runs-on: namespace-profile-arm64-8x16-2004-caching
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        lfs: true
+
+    - name: Get commit message
+      run: |
+        {
+          echo 'LAST_COMMIT_MESSAGE<<EOF'
+          if [ "${{ github.event_name }}" == "push" ]; then
+            echo "${{ github.event.head_commit.message }}"
+          elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            COMMIT_URL=$(echo "${{ github.event.repository.commits_url }}" | sed "s|{/sha}|/${{ github.sha }}|")
+            echo "$(curl -s "$COMMIT_URL" | jq -r '.commit.message')"
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            PR_COMMIT_URL=$(echo "${{ github.event.repository.commits_url }}" | sed "s|{/sha}|/${{ github.event.pull_request.head.sha }}|")
+            echo "$(curl -s "$PR_COMMIT_URL" | jq -r '.commit.message')"
+          fi
+          echo EOF
+        } | tee -a $GITHUB_ENV
+
+    - name: Get kernel submodule ref
+      id: kernel-submodule
+      run: echo "ref=$(git ls-tree HEAD | awk '$4 == "agnos-kernel-sdm845"' | awk '{print $3}')" | tee -a $GITHUB_OUTPUT
+
+    - name: Checkout agnos-kernel-sdm845
+      uses: actions/checkout@v4
+      with:
+        repository: commaai/agnos-kernel-sdm845
+        ref: ${{ steps.kernel-submodule.outputs.ref }}
+        path: agnos-kernel-sdm845
+
+    - name: ccache
+      uses: hendrikmuhs/ccache-action@c92f40bee50034e84c763e33b317c77adaa81c92
+      with:
+        key: kernel-${{ steps.kernel-submodule.outputs.ref }}
+        restore-keys: kernel-
+
+    - name: Build kernel
+      run: ./build_kernel.sh
+
+    - name: Upload artifact boot.img
+      uses: actions/upload-artifact@v4
+      if: "contains(env.LAST_COMMIT_MESSAGE, '[upload]')"
+      with:
+        name: boot.img
+        path: output/boot.img
+
+    - name: Upload artifact kernel modules
+      uses: actions/upload-artifact@v4
+      if: "contains(env.LAST_COMMIT_MESSAGE, '[upload]')"
+      with:
+        name: kernel-modules
+        path: output/*.ko
+
+    - name: Build system
+      run: ./build_system.sh
+
+    - name: Upload artifact system.img
+      uses: actions/upload-artifact@v4
+      if: "contains(env.LAST_COMMIT_MESSAGE, '[upload]')"
+      with:
+        name: system.img
+        path: output/system.img

.github/workflows/tests.yaml

@@ -1,17 +1,17 @@
 name: tests
 on:
   push:
+    branches: [ master ]
   pull_request:
 
 jobs:
   precommit:
     name: pre-commit
     runs-on: ubuntu-latest
-    timeout-minutes: 8
+    timeout-minutes: 1
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: '3.11.4'
-    - run: 'pip install pre-commit'
-    - run: 'pre-commit run --all'
+        python-version: '3.11'
+    - uses: pre-commit/action@v3.0.1

.gitignore

@@ -7,6 +7,7 @@ output/
 build/
 tmp/
 qdl/
+.ccache/
 .qemu_registered
 
 .vscode

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
     rev: v2.2.4
     hooks:
     -   id: codespell
-        exclude: '^(userspace/files/serviceproviders.xml)|(tools/)|(userspace/poetry.lock)'
+        exclude: '^(userspace/files/serviceproviders.xml)|(tools/)|(userspace/uv.lock)'
         args:
         # if you've got a short variable name that's getting flagged, add it here
         - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup
@@ -36,11 +36,3 @@ repos:
         - --select=F,E112,E113,E304,E502,E701,E702,E703,E71,E72,E731,W191,W6
         - --statistics
         - -j4
--   repo: https://github.com/python-poetry/poetry
-    rev: '1.6.0'
-    hooks:
-    -   id: poetry-check
-        entry: bash -c "cd userspace && poetry check"
-        name: validate poetry lock
-        args:
-        - --lock

Dockerfile.agnos

@@ -52,10 +52,6 @@ FROM agnos-compiler as agnos-compiler-ffmpeg
 COPY ./userspace/compile-ffmpeg.sh /tmp/agnos/
 RUN /tmp/agnos/compile-ffmpeg.sh
 
-FROM agnos-compiler as agnos-compiler-mapbox-gl-native
-COPY ./userspace/compile-mapbox-gl-native.sh /tmp/agnos/
-RUN /tmp/agnos/compile-mapbox-gl-native.sh
-
 FROM agnos-compiler as agnos-compiler-qtwayland5
 COPY ./userspace/compile-qtwayland5.sh /tmp/agnos/
 COPY ./userspace/qtwayland/patch* /tmp/agnos/
@@ -87,7 +83,6 @@ RUN /tmp/agnos/openpilot_python_dependencies.sh
 # Use other pre-compiled packages
 COPY --from=agnos-compiler-ffmpeg /tmp/ffmpeg.deb /tmp/ffmpeg.deb
 RUN cd /tmp && apt-get -o Dpkg::Options::="--force-overwrite" install -yq ./ffmpeg.deb
-COPY --from=agnos-compiler-mapbox-gl-native /tmp/libqmapboxgl.so /lib/aarch64-linux-gnu/libqmapboxgl.so
 
 RUN export PATH="/usr/local/pyenv/bin:/usr/local/pyenv/shims:$PATH" && \
     export PYENV_ROOT="/usr/local/pyenv" && \
@@ -99,18 +94,15 @@ RUN export PATH="/usr/local/pyenv/bin:/usr/local/pyenv/shims:$PATH" && \
     pyenv rehash
 
 # Install openpilot python packages
-COPY ./userspace/pyproject.toml ./userspace/poetry.lock /tmp/agnos/
+RUN mkdir -p /tmp/agnos/uv
+COPY ./userspace/pyproject.toml ./userspace/uv.lock /tmp/agnos/uv
 RUN export PATH="/usr/local/pyenv/bin:/usr/local/pyenv/shims:$PATH" && \
     export PYENV_ROOT="/usr/local/pyenv" && \
     eval "$(pyenv init -)" && \
     eval "$(pyenv virtualenv-init -)" && \
-    cd /tmp/agnos && \
-    poetry config virtualenvs.prefer-active-python true --local && \
-    poetry config virtualenvs.create false --local && \
+    cd /tmp/agnos/uv && \
     export PYOPENCL_CL_PRETEND_VERSION="2.0" && \
-    export POETRY_VIRTUALENVS_CREATE=false && \
-    MAKEFLAGS="-j$(nproc)" poetry install --no-cache --no-root && \
-    pyenv rehash
+    MAKEFLAGS="-j$(nproc)" UV_NO_CACHE=1 uv pip install --no-cache-dir --system .
 
 # Install nice to haves
 COPY ./userspace/install_extras.sh /tmp/agnos/

Dockerfile.builder

@@ -0,0 +1,35 @@
+FROM ubuntu:20.04
+
+ARG UNAME
+ARG UID
+ARG GID
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    python2 \
+    build-essential \
+    libssl-dev \
+    bc \
+    python-is-python2 \
+    openssl \
+    ccache \
+    android-sdk-libsparse-utils \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN if [ ${UID:-0} -ne 0 ] && [ ${GID:-0} -ne 0 ]; then \
+    userdel -r `getent passwd ${UID} | cut -d : -f 1` > /dev/null 2>&1; \
+    groupdel -f `getent group ${GID} | cut -d : -f 1` > /dev/null 2>&1; \
+    groupadd -g ${GID} -o ${UNAME} && \
+    useradd -u $UID -g $GID ${UNAME} \
+;fi
+
+RUN CCACHE_PATH=$(which ccache) && \
+    ln -s $CCACHE_PATH /usr/local/bin/gcc && \
+    ln -s $CCACHE_PATH /usr/local/bin/g++ && \
+    ln -s $CCACHE_PATH /usr/local/bin/cc && \
+    ln -s $CCACHE_PATH /usr/local/bin/c++ && \
+    ln -s $CCACHE_PATH /usr/local/bin/clang
+
+ENTRYPOINT ["tail", "-f", "/dev/null"]

README.md

@@ -1,45 +1,77 @@
 # agnos-builder
 
-This is the tool to build AGNOS, our Ubuntu based OS. AGNOS runs on the [comma three devkit](https://comma.ai/shop/products/three).
+This is the buidler repo for AGNOS, our Ubuntu based OS. AGNOS runs on the comma three devkit and [comma 3X](https://comma.ai/shop/comma-3x).
 
 NOTE: the `edk2_tici` and `agnos-firmware` submodules are internal, private repos.
 
-## Setup
+## Development
 
-These tools are developed on and targeted for Ubuntu 24.04.
+Any system that supports a recent [Docker](https://docs.docker.com/get-docker/) should work.
 
 Run once to set things up:
 ```sh
+git clone https://github.com/commaai/agnos-builder.git
+
+cd agnos-builder
 git submodule update --init agnos-kernel-sdm845
 ./tools/extract_tools.sh
 sudo apt install img2simg
 ```
 
-## Build the userspace
-
-build:
-```sh
+Building
+```
+./build_kernel.sh
 ./build_system.sh
 ```
 
-load:
-```sh
+Flashing to a comma 3/3X:
+```
+./flash_kernel.sh
 ./flash_system.sh
 ```
 
-## Build the kernel
+Validating changes:
+* Running openpilot is a good smoketest for general AGNOS functionality
+* [CI](https://github.com/commaai/agnos-builder/blob/master/.github/workflows/build.yaml) ensures the kernel and system builds work (and pushes the images for you to download)
+* [this](https://github.com/commaai/agnos-builder/blob/master/internal/README.md) is the checklist we go through before shipping new AGNOS releases to openpilot
 
-build:
-```sh
-./build_kernel.sh
-```
+### macOS
 
-load:
-```sh
-# flash over fastboot
-./flash_kernel.sh
+Building the kernel on macOS requires `agnos-builder` to be in a [Case-sensitive](https://support.apple.com/lv-lv/guide/disk-utility/dsku19ed921c/mac) filesystem.
 
-# or load into running system via ssh
-# ssh config needs host named 'tici'
-./load_kernel.sh
+Use this to set it up:
+```
+diskutil apfs addVolume <disk> "Case-sensitive APFS" agnos
 ```
+* replace `<disk>` with the `synthesized` disk in `diskutil list` that includes your main volume (e.g. Macintosh HD) - usually `disk3` on M-series Macs, `disk1` on Intel Macs
+* `agnos` is the name of the new volume - can be replaced with an arbitrary name
+* the volume is mounted automatically in `/Volumes/agnos`
+
+## Contributing
+
+Join us in the `#dev-agnos` channel on [Discord](https://discord.comma.ai).
+
+A comma 3/3X is helpful for working on AGNOS, but there's still lots of interesting work to do without one.
+
+* [Discord](https://discord.comma.ai)
+* [openpilot Docs](https://docs.comma.ai)
+* [Bounties](https://comma.ai/bounties)
+
+### Roadmap
+
+AGNOS's main priority is to provide a stable platform to [openpilot](https://github.com/commaai/openpilot).
+The second priority is to be a good platform for all sorts of [robotics development](https://blog.comma.ai/a-drive-in-the-office/).
+
+Now that AGNOS is good at running openpilot, the next things up are:
+* speedups: build system, CI, boot time, etc.
+* cleanups: Android kernel to mainline kernel, simplify the build system, etc.
+* open source: AGNOS started with a bunch blobs for various things, like the bootloaders and weston. We want to move everything we can to open source versions built in this repo.
+
+Some concrete things on the roadmap:
+- [ ] <10s boot time https://github.com/commaai/agnos-builder/issues/110, https://github.com/commaai/openpilot/issues/30894
+- [ ] make the image tiny, for fast updating and flashing https://github.com/commaai/agnos-builder/issues/225
+- [ ] update to Ubuntu 24.04 https://github.com/commaai/openpilot/issues/32386
+- [ ] mainline Linux kernel https://github.com/commaai/openpilot/issues/32386
+- [ ] fully open source
+  - [ ] anything from `agnos-firmware`: XBL, ABL, etc.
+  - [ ] open source Weston https://github.com/commaai/agnos-builder/issues/16