more WIP

Signed-off-by: greg pereira <grpereir@redhat.com>

.env

@@ -1,8 +1,8 @@
 # to define environment variables available to docker-compose.yml
 
 IMAGE_REPO=milvusdb
-IMAGE_ARCH=amd64
-OS_NAME=ubuntu20.04
+IMAGE_ARCH=arm64
+OS_NAME=rhel9
 
 # for services.builder.image in docker-compose.yml
 DATE_VERSION=20240520-d27db99

.github/workflows/rhel9-milvus.yaml

@@ -0,0 +1,90 @@
+name: Publish Builder
+# TODO: do not trigger action for some document file update
+
+# This workflow is triggered on pushes or pull request to the repository.
+on:
+  push:
+    # file paths to consider in the event. Optional; defaults to all.
+    paths:
+      - 'build/docker/builder/cpu/**'
+      - '.github/workflows/publish-builder.yaml'
+      - '!**.md'
+  pull_request:
+    # file paths to consider in the event. Optional; defaults to all.
+    paths:
+      - 'build/docker/builder/cpu/**'
+      - '.github/workflows/publish-builder.yaml'
+      - '!**.md'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  publish-builder:
+    name: ${{ matrix.arch }} ${{ matrix.os }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 500
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [rockylinux8]
+    env:
+      OS_NAME: ${{ matrix.os }}
+      IMAGE_ARCH: ${{ matrix.arch }}
+    steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        if: ${{ ! startsWith(runner.name, 'self') }} # skip this step if it is self-hosted runner
+        with:
+          root-reserve-mb: 20480
+          # overprovision-lvm: 'true'
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get version from system time after release step
+        id: extracter
+        run: |
+          echo "::set-output name=version::$(date +%Y%m%d)"
+          echo "::set-output name=sha_short::$(git rev-parse --short=7 HEAD)"
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            milvusdb/milvus-env
+          tags: |
+            type=raw,enable=true,value=${{ matrix.os }}-{{date 'YYYYMMDD'}}-{{sha}}
+            type=raw,enable=true,value=${{ matrix.os }}-latest
+      # - name: Setup upterm session
+      #   uses: lhotari/action-upterm@v1
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          file: build/docker/builder/cpu/${{ matrix.os }}/Dockerfile
+      - name: Bump Builder Version
+        uses: ./.github/actions/bump-builder-version
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
+        with:
+          tag: "${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}"
+          type: cpu
+          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}

build/docker/builder/cpu/rhel9/Dockerfile

@@ -0,0 +1,61 @@
+FROM registry.access.redhat.com/ubi9/ubi:9.4-947.1717074712 as vcpkg-installer
+USER 0
+RUN dnf install -y wget zip git \
+    gcc gcc-c++ cmake \
+    perl-IPC-Cmd perl-Digest-SHA \
+    perl-FindBin perl-File-Compare \
+    dnf-plugins-core ninja-build
+
+ENV VCPKG_FORCE_SYSTEM_BINARIES 1
+
+RUN mkdir /opt/vcpkg &&  \
+    wget -qO- vcpkg.tar.gz https://github.com/microsoft/vcpkg/archive/master.tar.gz | tar --strip-components=1 -xz -C /opt/vcpkg && \
+    rm -rf vcpkg.tar.gz
+
+# empty the vscpkg toolchains linux.cmake file to avoid the error
+RUN echo "" > /opt/vcpkg/scripts/toolchains/linux.cmake
+
+# install azure-identity-cpp azure-storage-blobs-cpp gtest via vcpkg
+RUN /opt/vcpkg/bootstrap-vcpkg.sh -disableMetrics && \
+    ln -s /opt/vcpkg/vcpkg /usr/local/bin/vcpkg && \
+    vcpkg version && \
+    vcpkg install azure-identity-cpp azure-storage-blobs-cpp gtest 
+
+########################################################################################
+FROM registry.redhat.io/rhel9/python-311:1-62.1717085982
+
+ARG TARGETARCH
+USER 0
+# basic deps
+RUN dnf install -y libaio libuuid-devel \
+        perl-IPC-Cmd perl-Digest-SHA \
+        libtool ninja-build
+
+COPY build/docker/builder/cpu/rhel9/install-rpms.sh /opt/app-root/src/install-rpms.sh
+RUN chmod +x /opt/app-root/src/install-rpms.sh
+RUN /opt/app-root/src/install-rpms.sh
+
+# Install conan and Go
+RUN pip3 install conan==1.61.0
+ENV GOLANG_VERSION=1.21.10
+
+# Install Go
+ENV GOPATH /go
+ENV GOROOT /usr/local/go
+ENV GO111MODULE on
+ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
+RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go$GOLANG_VERSION.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
+    mkdir -p "$GOPATH/src" "$GOPATH/bin" && \
+    go clean --modcache && \
+    chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR)
+
+
+# add go bin to path
+
+ENV VCPKG_FORCE_SYSTEM_BINARIES 1
+
+COPY --from=vcpkg-installer /opt/vcpkg/ /opt/vcpkg/
+
+USER 1001
+ENTRYPOINT [ "/entrypoint.sh" ]
+CMD ["tail", "-f", "/dev/null"]

build/docker/builder/cpu/rhel9/install-rpms.sh

@@ -0,0 +1,49 @@
+#/bin/bash
+if [[ "$TARGETARCH" == "arm64" ]] || [[ "$TARGETARCH" == "aarch64" ]]; then
+    dnf install -y https://www.rpmfind.net/linux/centos-stream/9-stream/CRB/aarch64/os/Packages/texinfo-6.7-15.el9.aarch64.rpm
+elif [[ "$TARGETARCH" == "amd64" ]] || [[ "$TARGETARCH" == "x86_64" ]]; then
+    dnf install -y https://www.rpmfind.net/linux/centos-stream/9-stream/CRB/x86_64/os/Packages/texinfo-6.7-15.el9.x86_64.rpm
+else
+    echo "\$TARGETARCH not specified. Looking up parent system arch".
+    TARGETARCH=$(uname -m)
+    if [[ "$TARGETARCH" == "arm64" ]] || [[ "$TARGETARCH" == "aarch64" ]]; then
+        dnf install -y https://www.rpmfind.net/linux/centos-stream/9-stream/CRB/aarch64/os/Packages/texinfo-6.7-15.el9.aarch64.rpm
+    elif [[ "$TARGETARCH" == "amd64" ]] || [[ "$TARGETARCH" == "x86_64" ]]; then
+        dnf install -y https://www.rpmfind.net/linux/centos-stream/9-stream/CRB/x86_64/os/Packages/texinfo-6.7-15.el9.x86_64.rpm
+    else
+        echo "failed to lookup host system OS and \$TARGETARCH was not specified. Build failure."
+        exit 1
+    fi
+fi
+RUN_MODE=""
+if [[ -z "$TARGETARCH" ]]; then
+    TARGETARCH=$(uname -m)
+    echo "\$TARGETARCH not set - defaulting to host system"
+fi
+
+if [[ "$TARGETARCH" == "arm64" ]] || [[ "$TARGETARCH" == "aarch64" ]]; then
+    RUN_MODE="aarch64"
+elif [[ "$TARGETARCH" == "amd64" ]] || [[ "$TARGETARCH" == "x86_64" ]]; then
+    RUN_MODE="x86_64"
+else
+    echo "invalid \$TARGETARCH, build failure.
+    Supported options: ['arm64', 'aarch64', 'amd64', 'x86_64']"
+    exit 1
+fi
+
+if [[ "$RUN_MODE" == "aarch64" ]]; then
+    dnf install -y \
+        https://www.rpmfind.net/linux/centos-stream/9-stream/AppStream/aarch64/os/Packages/perl-Unicode-EastAsianWidth-12.0-7.el9.noarch.rpm \
+        https://www.rpmfind.net/linux/centos-stream/9-stream/AppStream/aarch64/os/Packages/perl-Text-Unidecode-1.30-16.el9.noarch.rpm \
+        https://www.rpmfind.net/linux/centos-stream/9-stream/AppStream/aarch64/os/Packages/perl-libintl-perl-1.32-4.el9.aarch64.rpm \
+        https://www.rpmfind.net/linux/centos-stream/9-stream/CRB/aarch64/os/Packages/texinfo-6.7-15.el9.aarch64.rpm
+elif [[ "$RUN_MODE" == "x86_64" ]]; then
+    dnf install -y \
+        https://www.rpmfind.net/linux/centos-stream/9-stream/AppStream/x86_64/os/Packages/perl-Unicode-EastAsianWidth-12.0-7.el9.noarch.rpm \
+        https://www.rpmfind.net/linux/centos-stream/9-stream/AppStream/x86_64/os/Packages/perl-Text-Unidecode-1.30-16.el9.noarch.rpm \
+        https://www.rpmfind.net/linux/centos-stream/9-stream/AppStream/x86_64/os/Packages/perl-libintl-perl-1.32-4.el9.x86_64.rpm \
+        https://www.rpmfind.net/linux/centos-stream/9-stream/CRB/x86_64/os/Packages/texinfo-6.7-15.el9.x86_64.rpm
+else 
+    echo "uncaught runmode based on invalid \$TARGETARCH."
+    exit 1
+fi

build/docker/milvus/rhel9/Dockerfile

@@ -14,15 +14,9 @@ FROM registry.access.redhat.com/ubi9/ubi:9.4-947.1717074712
 
 ARG TARGETARCH
 
-RUN dnf install -y wget libgomp libaio libatomic
-
-# RUN subscription-manager register --auto-attach
-# install openblas-devel
-RUN dnf copr enable @copr/PyPI epel-9
-RUN 
-RUN dnf -y install dnf-plugins-core && \
-    # Could not find a version of openblas-devel on RHEL
-    dnf -y install openblas
+# assumes repo of codeready-builder-for-rhel-9-<arch> 
+RUN dnf install -y wget libgomp libaio libatomic openblas-devel && \
+    rm -rf /var/cache/yum/*
 
 COPY ./bin/ /milvus/bin/