diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml index dc92773450..e74ac4b29b 100644 --- a/.github/workflows/images.yaml +++ b/.github/workflows/images.yaml @@ -7,15 +7,14 @@ on: branches: [ main ] jobs: - images: - name: Build Images + retina-images: + name: Build Linux Images runs-on: ubuntu-latest strategy: matrix: platform: ["linux"] arch: ["amd64", "arm64"] - component: ["agent", "operator"] # required for AZ login/SP permissions: @@ -45,17 +44,133 @@ jobs: set -euo pipefail az acr login -n ${{ secrets.ACR_NAME }} echo "TAG=$(make version)" >> $GITHUB_ENV - make build PLATFORMS=${{ matrix.platform }}/${{ matrix.arch }} COMPONENT=${{ matrix.component }} + make retina-image PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} + retina-win-images: + name: Build Windows Images + runs-on: ubuntu-latest + + strategy: + matrix: + platform: ["windows"] + arch: ["amd64"] + + # required for AZ login/SP + permissions: + id-token: write + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - uses: actions/setup-go@v4 + with: + go-version: ">=1.21.0" + - run: go version + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Az CLI login + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Build/Push Images + shell: bash + run: | + set -euo pipefail + az acr login -n ${{ secrets.ACR_NAME }} + echo "TAG=$(make version)" >> $GITHUB_ENV + make retina-image-win PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} + + operator-images: + name: Build Operator Images + runs-on: ubuntu-latest + + strategy: + matrix: + platform: ["linux"] + arch: ["amd64", "arm64"] + + # required for AZ login/SP + permissions: + id-token: write + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - uses: actions/setup-go@v4 + with: + go-version: ">=1.21.0" + - run: go version + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Az CLI login + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Build/Push Images + shell: bash + run: | + set -euo pipefail + az acr login -n ${{ secrets.ACR_NAME }} + echo "TAG=$(make version)" >> $GITHUB_ENV + make retina-operator-image PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} + + kubectl-retina-images: + name: Build kubectl-retina Images + runs-on: ubuntu-latest + + strategy: + matrix: + platform: ["linux"] + arch: ["amd64", "arm64"] + + # required for AZ login/SP + permissions: + id-token: write + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - uses: actions/setup-go@v4 + with: + go-version: ">=1.21.0" + - run: go version + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Az CLI login + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Build/Push Images + shell: bash + run: | + set -euo pipefail + az acr login -n ${{ secrets.ACR_NAME }} + echo "TAG=$(make version)" >> $GITHUB_ENV + make kubectl-retina-image PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} manifests: name: Generate Manifests runs-on: ubuntu-latest - needs: images + needs: [retina-images, retina-win-images, operator-images, kubectl-retina-images] strategy: matrix: - component: ["agent", "operator"] + component: ["retina", "operator", "kubectl-retina"] # required for AZ login/SP permissions: diff --git a/Makefile b/Makefile index a3cfd77147..45d4d70db7 100644 --- a/Makefile +++ b/Makefile @@ -9,8 +9,6 @@ REPO_ROOT = $(shell git rev-parse --show-toplevel) ifndef TAG TAG ?= $(shell git describe --tags --always) endif -GOOS ?= $(shell go env GOOS) -GOARCH ?= $(shell go env GOARCH) OUTPUT_DIR = $(REPO_ROOT)/output BUILD_DIR = $(OUTPUT_DIR)/$(GOOS)_$(GOARCH) RETINA_BUILD_DIR = $(BUILD_DIR)/retina @@ -27,38 +25,29 @@ APP_INSIGHTS_ID ?= "" GENERATE_TARGET_DIRS = \ ./pkg/plugin/linuxutil +# Default platform is linux/amd64 +GOOS ?= linux +GOARCH ?= amd64 IMAGE_REGISTRY ?= acnpublic.azurecr.io OS ?= $(GOOS) ARCH ?= $(GOARCH) PLATFORM ?= $(OS)/$(ARCH) +PLATFORMS ?= linux/amd64 linux/arm64 windows/amd64 -CONTAINER_BUILDER ?= buildah -CONTAINER_RUNTIME ?= podman +CONTAINER_BUILDER ?= docker +CONTAINER_RUNTIME ?= docker YEAR ?=2022 ALL_ARCH.linux = amd64 arm64 ALL_ARCH.windows = amd64 -# prefer buildah, if available, but fall back to docker if that binary is not in the path. -ifeq (, $(shell which $(CONTAINER_BUILDER))) -CONTAINER_BUILDER = docker -endif -# use docker if platform is windows -ifeq ($(OS),windows) -CONTAINER_BUILDER = docker -endif -# prefer podman, if available, but fall back to docker if that binary is not in the path. -ifeq (, $(shell which $(CONTAINER_RUNTIME))) -CONTAINER_RUNTIME = docker -endif - # TAG is OS and platform agonstic, which can be used for binary version and image manifest tag, # while RETINA_PLATFORM_TAG is platform specific, which can be used for image built for specific platforms. -RETINA_PLATFORM_TAG ?= $(subst /,-,$(PLATFORM))-$(TAG) +RETINA_PLATFORM_TAG ?= $(TAG)-$(subst /,-,$(PLATFORM)) # for windows os, add year to the platform tag ifeq ($(OS),windows) -RETINA_PLATFORM_TAG = windows-ltsc$(YEAR)-amd64-$(TAG) +RETINA_PLATFORM_TAG = $(TAG)-windows-ltsc$(YEAR)-amd64 endif qemu-user-static: ## Set up the host to run qemu multiplatform container builds. @@ -156,10 +145,9 @@ retina: ## builds both retina and kapctl binaries $(MAKE) retina-binary kubectl-retina retina-binary: ## build the Retina binary - go generate ./... &&\ - cd $(RETINA_DIR) &&\ - CGO_ENABLED=0 &&\ - go build -v -o $(RETINA_BUILD_DIR)/retina$(EXE_EXT) -gcflags="-dwarflocationlists=true" -ldflags "-X main.version=$(TAG) -X main.applicationInsightsID=$(APP_INSIGHTS_ID)" + go generate ./... + export CGO_ENABLED=0 + go build -v -o $(RETINA_BUILD_DIR)/retina$(EXE_EXT) -gcflags="-dwarflocationlists=true" -ldflags "-X main.version=$(TAG) -X main.applicationInsightsID=$(APP_INSIGHTS_ID)" $(RETINA_DIR)/main.go all-kubectl-retina: $(addprefix kubectl-retina-linux-,${ALL_ARCH.linux}) $(addprefix kubectl-retina-windows-,${ALL_ARCH.windows}) ## build kubectl plugin for all platforms @@ -231,93 +219,82 @@ retina-skopeo-export: IMG=$(RETINA_IMAGE) TAG=$(RETINA_PLATFORM_TAG) -# VERSION vs TAG: VERSION is the version of the binary, TAG is the version of the container image -# which may contain OS and ARCH information. -container-buildah: # util target to build container images using buildah. do not invoke directly. - buildah bud \ - --jobs 16 \ - --platform $(PLATFORM) \ - -f $(DOCKERFILE) \ - --build-arg VERSION=$(VERSION) $(EXTRA_BUILD_ARGS) \ - --build-arg GOOS=$(GOOS) \ - --build-arg GOARCH=$(GOARCH) \ - --build-arg APP_INSIGHTS_ID=$(APP_INSIGHTS_ID) \ - --build-arg builderImage=$(IMAGE_REGISTRY)/$(RETINA_BUILDER_IMAGE):$(TAG) \ - --build-arg toolsImage=$(IMAGE_REGISTRY)/$(RETINA_TOOLS_IMAGE):$(TAG) \ - -t $(IMAGE_REGISTRY)/$(IMAGE):$(TAG) \ - $(CONTEXT_DIR) +buildx: + if docker buildx inspect retina > /dev/null 2>&1; then \ + echo "Buildx instance retina already exists."; \ + else \ + echo "Creating buildx instance retina..."; \ + docker buildx create --name retina --use --platform $$(echo "$(PLATFORMS)" | tr ' ' ','); \ + docker buildx use retina; \ + echo "Buildx instance retina created."; \ + fi; -container-docker: # util target to build container images using docker buildx. do not invoke directly. +container-docker: buildx # util target to build container images using docker buildx. do not invoke directly. + echo "Building for platform $(PLATFORM)" + os=$$(echo $(PLATFORM) | cut -d'/' -f1); \ + arch=$$(echo $(PLATFORM) | cut -d'/' -f2); \ + echo "Building for $$os/$$arch"; \ docker buildx build \ $(ACTION) \ --platform $(PLATFORM) \ -f $(DOCKERFILE) \ --build-arg VERSION=$(VERSION) $(EXTRA_BUILD_ARGS) \ - --build-arg GOOS=$(GOOS) \ - --build-arg GOARCH=$(GOARCH) \ + --build-arg GOOS=$$os \ + --build-arg GOARCH=$$arch \ --build-arg APP_INSIGHTS_ID=$(APP_INSIGHTS_ID) \ - --build-arg builderImage=$(IMAGE_REGISTRY)/$(RETINA_BUILDER_IMAGE):$(TAG) \ - --build-arg toolsImage=$(IMAGE_REGISTRY)/$(RETINA_TOOLS_IMAGE):$(TAG) \ + --target=$(TARGET) \ -t $(IMAGE_REGISTRY)/$(IMAGE):$(TAG) \ $(CONTEXT_DIR) -retina-builder-image: ## build the retina builder container image. +retina-image: ## build the retina linux container image. echo "Building for $(PLATFORM)" - $(MAKE) container-$(CONTAINER_BUILDER) \ - PLATFORM=$(PLATFORM) \ - DOCKERFILE=controller/Dockerfile.builder \ - REGISTRY=$(IMAGE_REGISTRY) \ - IMAGE=$(RETINA_BUILDER_IMAGE) \ - VERSION=$(TAG) \ - TAG=$(RETINA_PLATFORM_TAG) \ - APP_INSIGHTS_ID=$(APP_INSIGHTS_ID) \ - CONTEXT_DIR=$(REPO_ROOT) \ - ACTION=--load - -retina-builder-image-remove: - $(CONTAINER_BUILDER) rmi $(IMAGE_REGISTRY)/$(RETINA_BUILDER_IMAGE):$(RETINA_PLATFORM_TAG) - -retina-tools-image: ## build the retina container image. - echo "Building for $(PLATFORM)" - $(MAKE) container-$(CONTAINER_BUILDER) \ - PLATFORM=$(PLATFORM) \ - DOCKERFILE=controller/Dockerfile.tools \ - REGISTRY=$(IMAGE_REGISTRY) \ - IMAGE=$(RETINA_TOOLS_IMAGE) \ - VERSION=$(TAG) \ - TAG=$(RETINA_PLATFORM_TAG) \ - APP_INSIGHTS_ID=$(APP_INSIGHTS_ID) \ - CONTEXT_DIR=$(REPO_ROOT) \ - ACTION=--load - -retina-tools-image-remove: - $(CONTAINER_BUILDER) rmi $(IMAGE_REGISTRY)/$(RETINA_TOOLS_IMAGE):$(RETINA_PLATFORM_TAG) + for target in init agent; do \ + echo "Building for $$target"; \ + if [ "$$target" = "init" ]; then \ + image_name=$(RETINA_INIT_IMAGE); \ + else \ + image_name=$(RETINA_IMAGE); \ + fi; \ + $(MAKE) container-$(CONTAINER_BUILDER) \ + PLATFORM=$(PLATFORM) \ + DOCKERFILE=controller/Dockerfile.controller \ + REGISTRY=$(IMAGE_REGISTRY) \ + IMAGE=$$image_name \ + VERSION=$(TAG) \ + TAG=$(RETINA_PLATFORM_TAG) \ + APP_INSIGHTS_ID=$(APP_INSIGHTS_ID) \ + CONTEXT_DIR=$(REPO_ROOT) \ + TARGET=$$target \ + ACTION=--push; \ + done -retina-image: ## This pulls dependecies from registry. Use retina-image-local to build dependencies first locally. - echo "Building for $(PLATFORM)" - $(MAKE) container-$(CONTAINER_BUILDER) \ - PLATFORM=$(PLATFORM) \ - DOCKERFILE=controller/Dockerfile.controller \ - REGISTRY=$(IMAGE_REGISTRY) \ - IMAGE=$(RETINA_IMAGE) \ - VERSION=$(TAG) \ - TAG=$(RETINA_PLATFORM_TAG) \ - APP_INSIGHTS_ID=$(APP_INSIGHTS_ID) \ - CONTEXT_DIR=$(REPO_ROOT) \ - ACTION=--load +retina-image-win: ## build the retina Windows container image. + for year in 2019 2022; do \ + tag=$(TAG)-windows-ltsc$$year-amd64; \ + echo "Building $(RETINA_PLATFORM_TAG)"; \ + $(MAKE) container-$(CONTAINER_BUILDER) \ + PLATFORM=windows/amd64 \ + DOCKERFILE=controller/Dockerfile.windows-$$year \ + REGISTRY=$(IMAGE_REGISTRY) \ + IMAGE=$(RETINA_IMAGE) \ + VERSION=$(TAG) \ + TAG=$$tag \ + CONTEXT_DIR=$(REPO_ROOT) \ + ACTION=--push; \ + done -retina-init-image: ## build the retina container image. +retina-operator-image: ## build the retina linux operator image. echo "Building for $(PLATFORM)" $(MAKE) container-$(CONTAINER_BUILDER) \ PLATFORM=$(PLATFORM) \ - DOCKERFILE=controller/Dockerfile.init \ + DOCKERFILE=operator/Dockerfile \ REGISTRY=$(IMAGE_REGISTRY) \ - IMAGE=$(RETINA_INIT_IMAGE) \ + IMAGE=$(RETINA_OPERATOR_IMAGE) \ VERSION=$(TAG) \ TAG=$(RETINA_PLATFORM_TAG) \ APP_INSIGHTS_ID=$(APP_INSIGHTS_ID) \ CONTEXT_DIR=$(REPO_ROOT) \ - ACTION=--load + ACTION=--push kubectl-retina-image: ## build the kubectl-retina image. echo "Building for $(PLATFORM)" @@ -330,71 +307,6 @@ kubectl-retina-image: ## build the kubectl-retina image. TAG=$(RETINA_PLATFORM_TAG) \ APP_INSIGHTS_ID=$(APP_INSIGHTS_ID) \ CONTEXT_DIR=$(REPO_ROOT) \ - ACTION=--load - -retina-operator-image: ## build the retina operator image. - echo "Building for $(PLATFORM)" - $(MAKE) container-$(CONTAINER_BUILDER) \ - PLATFORM=$(PLATFORM) \ - DOCKERFILE=operator/Dockerfile \ - REGISTRY=$(IMAGE_REGISTRY) \ - IMAGE=$(RETINA_OPERATOR_IMAGE) \ - VERSION=$(TAG) \ - TAG=$(RETINA_PLATFORM_TAG) \ - APP_INSIGHTS_ID=$(APP_INSIGHTS_ID) \ - CONTEXT_DIR=$(REPO_ROOT) \ - ACTION=--load - -kubectl-retina-image-push: ## push kubectl-retina container image. - $(MAKE) container-push \ - IMAGE=$(KUBECTL_RETINA_IMAGE) \ - TAG=$(RETINA_PLATFORM_TAG) - -retina-image-push: ## push the retina container image. - $(MAKE) container-push \ - IMAGE=$(RETINA_IMAGE) \ - TAG=$(RETINA_PLATFORM_TAG) - -retina-builder-image-push: ## push the retina builder container image. - $(MAKE) container-push \ - IMAGE=$(RETINA_BUILDER_IMAGE) \ - TAG=$(RETINA_PLATFORM_TAG) - -retina-tools-image-push: ## push the retina tools container image. - $(MAKE) container-push \ - IMAGE=$(RETINA_TOOLS_IMAGE) \ - TAG=$(RETINA_PLATFORM_TAG) - -retina-init-image-push: ## push the retina container image. - $(MAKE) container-push \ - IMAGE=$(RETINA_INIT_IMAGE) \ - TAG=$(RETINA_PLATFORM_TAG) - -retina-operator-image-push: ## push the retina container image. - $(MAKE) container-push \ - IMAGE=$(RETINA_OPERATOR_IMAGE) \ - TAG=$(RETINA_PLATFORM_TAG) - -retina-image-win: ## build the retina Windows container image. - $(MAKE) container-$(CONTAINER_BUILDER) \ - PLATFORM=windows/amd64 \ - DOCKERFILE=controller/Dockerfile.windows-$(YEAR) \ - REGISTRY=$(IMAGE_REGISTRY) \ - IMAGE=$(RETINA_IMAGE) \ - VERSION=$(TAG) \ - TAG=$(RETINA_PLATFORM_TAG) \ - CONTEXT_DIR=$(REPO_ROOT) \ - ACTION=--load - -retina-image-win-push: ## push the retina Windows container image. - $(MAKE) container-$(CONTAINER_BUILDER) \ - PLATFORM=windows/amd64 \ - DOCKERFILE=controller/Dockerfile.windows-$(YEAR) \ - REGISTRY=$(IMAGE_REGISTRY) \ - IMAGE=$(RETINA_IMAGE) \ - VERSION=$(TAG) \ - TAG=$(RETINA_PLATFORM_TAG) \ - CONTEXT_DIR=$(REPO_ROOT) \ ACTION=--push proto-gen: ## generate protobuf code @@ -416,26 +328,31 @@ all-gen: ## generate all code $(MAKE) proto-gen $(MAKE) go-gen -all-images-local: - $(MAKE) -j4 retina-builder-image retina-tools-image retina-operator-image kubectl-retina-image - $(MAKE) -j2 retina-image retina-init-image - -all-images-local-push: - $(MAKE) -j3 retina-builder-image-push retina-tools-image-push retina-operator-image-push - $(MAKE) -j3 retina-image-push retina-init-image-push kubectl-retina-image-push - -base-images-remove: - $(MAKE) -j2 retina-builder-image-remove retina-tools-image-remove - -# Build images locally. -# Don't use this in pipeline, we want to pull images from registry. -retina-image-local: - $(MAKE) -j2 retina-builder-image retina-tools-image - $(MAKE) retina-image - -retina-init-image-local: - $(MAKE) -j2 retina-builder-image retina-tools-image - $(MAKE) retina-init-image +##@ Multiplatform + +manifest-retina-image: ## create a multiplatform manifest for the retina image + $(eval FULL_IMAGE_NAME=$(IMAGE_REGISTRY)/$(RETINA_IMAGE):$(TAG)) + $(eval FULL_INIT_IMAGE_NAME=$(IMAGE_REGISTRY)/$(RETINA_INIT_IMAGE):$(TAG)) + docker buildx imagetools create -t $(FULL_IMAGE_NAME) $(foreach platform,linux/amd64 linux/arm64 windows-ltsc2019-amd64 windows-ltsc2022-amd64, $(FULL_IMAGE_NAME)-$(subst /,-,$(platform))) + docker buildx imagetools create -t $(FULL_INIT_IMAGE_NAME) $(foreach platform,linux/amd64 linux/arm64, $(FULL_IMAGE_NAME)-$(subst /,-,$(platform))) + +manifest-operator-image: ## create a multiplatform manifest for the operator image + $(eval FULL_IMAGE_NAME=$(IMAGE_REGISTRY)/$(RETINA_OPERATOR_IMAGE):$(TAG)) + docker buildx imagetools create -t $(FULL_IMAGE_NAME) $(foreach platform,linux/amd64 linux/arm64, $(FULL_IMAGE_NAME)-$(subst /,-,$(platform))) + +manifest-kubectl-retina-image: ## create a multiplatform manifest for the kubectl-retina image + $(eval FULL_IMAGE_NAME=$(IMAGE_REGISTRY)/$(KUBECTL_RETINA_IMAGE):$(TAG)) + docker buildx imagetools create -t $(FULL_IMAGE_NAME) $(foreach platform,linux/amd64 linux/arm64, $(FULL_IMAGE_NAME)-$(subst /,-,$(platform))) + +manifest: + echo "Building for $(COMPONENT)" + if [ "$(COMPONENT)" = "retina" ]; then \ + $(MAKE) manifest-retina-image; \ + elif [ "$(COMPONENT)" = "operator" ]; then \ + $(MAKE) manifest-operator-image; \ + elif [ "$(COMPONENT)" = "kubectl-retina" ]; then \ + $(MAKE) manifest-kubectl-retina-image; \ + fi ##@ Tests # Make sure the layer has only one directory. @@ -464,25 +381,6 @@ retina-test-image: ## build the retina container image for testing. COVER_PKG ?= . -retina-integration-test-image: # Build the retina container image for integration testing. - docker build \ - -t $(IMAGE_REGISTRY)/$(RETINA_INTEGRATION_TEST_IMAGE):$(RETINA_PLATFORM_TAG) \ - -f test/integration/Dockerfile.integration \ - --build-arg kubeconfig=$(HOME)/.kube/config \ - --build-arg ENABLE_POD_LEVEL=$(ENABLE_POD_LEVEL) \ - . - -retina-integration-docker-deploy: - docker rm -f retina-integ-container 2> /dev/null - docker run \ - -e RETINA_AGENT_IMAGE=$(IMAGE_REGISTRY)/$(RETINA_IMAGE):$(TAG) \ - -e ENABLE_POD_LEVEL=$(ENABLE_POD_LEVEL) \ - -v $(HOME)/.kube/config:/root/.kube/config \ - --name retina-integ-container \ - $(IMAGE_REGISTRY)/$(RETINA_INTEGRATION_TEST_IMAGE):$(RETINA_PLATFORM_TAG) \ - || true - docker cp retina-integ-container:/tmp/retina-integration-logs . - retina-ut: $(ENVTEST) # Run unit tests. go build -o test-summary ./test/utsummary/main.go CGO_ENABLED=0 KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use -p path)" go test -tags=unit -coverprofile=coverage.out -v -json ./... | ./test-summary --progress --verbose @@ -501,179 +399,8 @@ retina-cc: # Code coverage. python3 scripts/coverage/compare_cov.py; \ fi; -retina-integration: $(GINKGO) # Integration tests. - export ACK_GINKGO_RC=true && $(GINKGO) -keepGoing -tags=integration ./test/integration/... -v -progress -trace -cover -coverprofile=coverage.out - -retina-export-logs: # Export worker node logs. - mkdir kubernetes-logs - kubectl get pods -A -o wide > kubernetes-logs/pods.txt - docker cp retina-cluster-worker:/var/log kubernetes-logs - -##@ kind - -kind-setup: ## Deploy kind cluster. - $(KIND) create cluster --name $(KIND_CLUSTER) --config ./test/kind/kind.yaml -# Install NPM - kubectl apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/azure-npm.yaml - sleep 5s - kubectl -n kube-system wait --for=condition=ready --timeout=120s pod -l k8s-app=azure-npm - -kind-clean: ## Delete kind cluster. - $(KIND) delete cluster --name $(KIND_CLUSTER) - -kind-load-image: ## Load local image to kind nodes. -# $(MAKE) retina-image - $(KIND) load docker-image --name $(KIND_CLUSTER) $(IMAGE_REGISTRY)/$(RETINA_IMAGE):$(RETINA_PLATFORM_TAG) - $(KIND) load docker-image --name $(KIND_CLUSTER) $(IMAGE_REGISTRY)/$(RETINA_OPERATOR_IMAGE):$(RETINA_PLATFORM_TAG) - $(KIND) load docker-image --name $(KIND_CLUSTER) $(IMAGE_REGISTRY)/$(RETINA_INIT_IMAGE):$(RETINA_PLATFORM_TAG) - -kind-install: kind-load-image # Install Retina in kind cluster. - helm install retina ./deploy/manifests/controller/helm/retina/ \ - --set image.repository=$(IMAGE_REGISTRY)/$(RETINA_IMAGE) \ - --set image.tag=$(RETINA_PLATFORM_TAG) \ - --set operator.repository=$(IMAGE_REGISTRY)/$(RETINA_OPERATOR_IMAGE) \ - --set operator.tag=$(RETINA_PLATFORM_TAG) \ - --set image.initRepository=$(IMAGE_REGISTRY)/$(RETINA_INIT_IMAGE) \ - --set image.pullPolicy=Never \ - --set logLevel=debug \ - --set os.windows=false \ - --namespace kube-system --dependency-update - sleep 5s -# Wait for retina agent to be ready. - kubectl -n kube-system wait --for=condition=ready --timeout=120s pod -l app=retina -# Port forward the retina api server. - kubectl -n kube-system port-forward svc/retina-svc 8889:10093 2>&1 >/dev/null & - -kind-uninstall: # Uninstall Retina from kind cluster. - helm uninstall retina -n kube-system - ## Reusable targets for building multiplat container image manifests. -IMAGE_ARCHIVE_DIR ?= $(shell pwd) - -manifest-create: # util target to compose multiarch container manifests from platform specific images. - cat /usr/share/containers/containers.conf - $(CONTAINER_BUILDER) manifest create $(IMAGE_REGISTRY)/$(IMAGE):$(TAG) - for PLATFORM in $(PLATFORMS); do $(MAKE) manifest-add PLATFORM=$$PLATFORM IMAGE=$(IMAGE) TAG=$(TAG); done - -manifest-add: - if [ "$(PLATFORM)" = "windows/amd64/2022" ]; then \ - echo "Adding windows/amd64/2022"; \ - $(CONTAINER_BUILDER) manifest add --os-version=$(WINVER2022) --os=windows $(IMAGE_REGISTRY)/$(IMAGE):$(TAG) docker://$(IMAGE_REGISTRY)/$(IMAGE):windows-ltsc2022-amd64-$(TAG); \ - elif [ "$(PLATFORM)" = "windows/amd64/2019" ]; then \ - echo "Adding windows/amd64/2019"; \ - $(CONTAINER_BUILDER) manifest add --os-version=$(WINVER2019) --os=windows $(IMAGE_REGISTRY)/$(IMAGE):$(TAG) docker://$(IMAGE_REGISTRY)/$(IMAGE):windows-ltsc2019-amd64-$(TAG); \ - else \ - echo "Adding $(PLATFORM)"; \ - $(CONTAINER_BUILDER) manifest add $(IMAGE_REGISTRY)/$(IMAGE):$(TAG) docker://$(IMAGE_REGISTRY)/$(IMAGE):$(subst /,-,$(PLATFORM))-$(TAG); \ - fi; - -manifest-push: # util target to push multiarch container manifest. - $(CONTAINER_BUILDER) manifest inspect $(IMAGE_REGISTRY)/$(IMAGE):$(TAG) - $(CONTAINER_BUILDER) manifest push --all $(IMAGE_REGISTRY)/$(IMAGE):$(TAG) docker://$(IMAGE_REGISTRY)/$(IMAGE):$(TAG) - -manifest-skopeo-archive: # util target to export tar archive of multiarch container manifest. - skopeo copy --all docker://$(IMAGE_REGISTRY)/$(IMAGE):$(TAG) oci-archive:$(IMAGE_ARCHIVE_DIR)/$(IMAGE)-$(TAG).tar - -## Build specific multiplat images. - -retina-builder-manifest-create: ## build retina multiplat container manifest. - $(MAKE) manifest-create \ - PLATFORMS="$(PLATFORMS)" \ - IMAGE=$(RETINA_BUILDER_IMAGE) \ - TAG=$(TAG) - -retina-builder-manifest-push: ## push retina multiplat container manifest - $(MAKE) manifest-push \ - IMAGE=$(RETINA_BUILDER_IMAGE) \ - TAG=$(TAG) - -retina-builder-skopeo-archive: ## export tar archive of retina multiplat container manifest. - $(MAKE) manifest-skopeo-archive \ - IMAGE=$(RETINA_BUILDER_IMAGE) \ - TAG=$(TAG) - -retina-tools-manifest-create: ## build retina multiplat container manifest. - $(MAKE) manifest-create \ - PLATFORMS="$(PLATFORMS)" \ - IMAGE=$(RETINA_TOOLS_IMAGE) \ - TAG=$(TAG) - -retina-tools-manifest-push: ## push retina multiplat container manifest - $(MAKE) manifest-push \ - IMAGE=$(RETINA_TOOLS_IMAGE) \ - TAG=$(TAG) - -retina-tools-skopeo-archive: ## export tar archive of retina multiplat container manifest. - $(MAKE) manifest-skopeo-archive \ - IMAGE=$(RETINA_TOOLS_IMAGE) \ - TAG=$(TAG) - -retina-init-manifest-create: ## build retina multiplat container manifest. - $(MAKE) manifest-create \ - PLATFORMS="$(PLATFORMS)" \ - IMAGE=$(RETINA_INIT_IMAGE) \ - TAG=$(TAG) - -retina-init-manifest-push: ## push retina multiplat container manifest - $(MAKE) manifest-push \ - IMAGE=$(RETINA_INIT_IMAGE) \ - TAG=$(TAG) - -retina-init-skopeo-archive: ## export tar archive of retina multiplat container manifest. - $(MAKE) manifest-skopeo-archive \ - IMAGE=$(RETINA_INIT_IMAGE) \ - TAG=$(TAG) - -retina-agent-manifest-create: ## build retina multiplat container manifest. - $(MAKE) manifest-create \ - PLATFORMS="$(PLATFORMS)" \ - IMAGE=$(RETINA_IMAGE) \ - TAG=$(TAG) - -retina-agent-manifest-push: ## push retina multiplat container manifest - $(MAKE) manifest-push \ - IMAGE=$(RETINA_IMAGE) \ - TAG=$(TAG) - -retina-agent-skopeo-archive: ## export tar archive of retina multiplat container manifest. - $(MAKE) manifest-skopeo-archive \ - IMAGE=$(RETINA_IMAGE) \ - TAG=$(TAG) - -retina-operator-manifest-create: ## build retina multiplat container manifest. - $(MAKE) manifest-create \ - PLATFORMS="$(PLATFORMS)" \ - IMAGE=$(RETINA_OPERATOR_IMAGE) \ - TAG=$(TAG) - -retina-operator-manifest-push: ## push retina multiplat container manifest - $(MAKE) manifest-push \ - IMAGE=$(RETINA_OPERATOR_IMAGE) \ - TAG=$(TAG) - -retina-operator-skopeo-archive: ## export tar archive of retina multiplat container manifest. - $(MAKE) manifest-skopeo-archive \ - IMAGE=$(RETINA_OPERATOR_IMAGE) \ - TAG=$(TAG) - -kubectl-retina-manifest-create: ## build kubectl plugin multiplat container manifest. - $(MAKE) manifest-create \ - PLATFORMS="$(PLATFORMS)" \ - IMAGE=$(KUBECTL_RETINA_IMAGE) \ - TAG=$(TAG) - -kubectl-retina-manifest-push: ## push kubectl plugin multiplat container manifest. - $(MAKE) manifest-push \ - IMAGE=$(KUBECTL_RETINA_IMAGE) \ - TAG=$(TAG) - -kubectl-retina-skopeo-archive: ## export tar archive of retina kubectl plugin container manifest. - $(MAKE) manifest-skopeo-archive \ - IMAGE=$(KUBECTL_RETINA_IMAGE) \ - TAG=$(TAG) - - .PHONY: manifests manifests: cd crd && make manifests && make generate @@ -691,22 +418,6 @@ helm-install: manifests --set operator.enabled=false \ --set enabledPlugin_linux="[\"dropreason\"\,\"packetforward\"\,\"linuxutil\"\,\"dns\"]" -helm-install-with-operator: manifests - helm install retina ./deploy/manifests/controller/helm/retina/ \ - --namespace kube-system \ - --set image.repository=$(IMAGE_REGISTRY)/$(RETINA_IMAGE) \ - --set image.tag=$(RETINA_PLATFORM_TAG) \ - --set image.initRepository=$(IMAGE_REGISTRY)/$(RETINA_INIT_IMAGE) \ - --set image.pullPolicy=Always \ - --set logLevel=info \ - --set os.windows=true \ - --set operator.enabled=true \ - --set operator.enableRetinaEndpoint=true \ - --set operator.tag=$(RETINA_PLATFORM_TAG) \ - --set operator.repository=$(IMAGE_REGISTRY)/$(RETINA_OPERATOR_IMAGE) \ - --skip-crds \ - --set enabledPlugin_linux="[\"dropreason\"\,\"packetforward\"\,\"linuxutil\"\,\"dns\"]" - # advanced/pod-level mode with scale limitations, where metrics are aggregated by source and destination Pod helm-install-advanced-remote-context: manifests helm install retina ./deploy/manifests/controller/helm/retina/ \ @@ -757,7 +468,9 @@ docs: .PHONY: docs-pod docs-prod: docker run -i -p 3000:3000 -v $(PWD):/retina -w /retina/ node:20-alpine npm install --prefix site && npm run build --prefix site - + +# Kapinger is a tool to generate traffic for testing Retina. + kapinger-image: ## build the retina container image. echo "Building for $(PLATFORM)" $(MAKE) container-$(CONTAINER_BUILDER) \ diff --git a/controller/Dockerfile.builder b/controller/Dockerfile.builder deleted file mode 100644 index e7c24467a0..0000000000 --- a/controller/Dockerfile.builder +++ /dev/null @@ -1,37 +0,0 @@ -# Stage: Build binary -FROM mcr.microsoft.com/oss/go/microsoft/golang:1.21 AS builder -LABEL Name=retina-builder Version=0.0.1 - -RUN apt-get update &&\ - apt-get -y install lsb-release wget software-properties-common gnupg file git make - -RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - -RUN add-apt-repository "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-14 main" -RUN apt-get update - -RUN apt-get install -y clang-14 lldb-14 lld-14 clangd-14 man-db -RUN apt-get install -y bpftool libbpf-dev - -RUN ln -s /usr/bin/clang-14 /usr/bin/clang - -COPY . /go/src/github.com/microsoft/retina -WORKDIR /go/src/github.com/microsoft/retina - -# RUN go mod edit -module retina -# RUN go generate ./... - -# Default linux/architecture. -ARG GOOS=linux -ENV GOOS=${GOOS} - -ARG GOARCH=amd64 -ENV GOARCH=${GOARCH} - -ENV CGO_ENABLED=0 - -ARG VERSION -ARG APP_INSIGHTS_ID - -RUN go build -v -o /go/bin/retina/controller -ldflags "-X main.version="$VERSION" -X "main.applicationInsightsID"="$APP_INSIGHTS_ID"" controller/main.go -RUN go build -v -o /go/bin/retina/captureworkload -ldflags "-X main.version="$VERSION" -X "main.applicationInsightsID"="$APP_INSIGHTS_ID"" captureworkload/main.go -RUN go build -v -o /go/bin/retina/initretina -ldflags "-X main.version="$VERSION" -X "main.applicationInsightsID"="$APP_INSIGHTS_ID"" ././init/retina/main.go diff --git a/controller/Dockerfile.controller b/controller/Dockerfile.controller index 917aeb475b..a6fb9e64f4 100644 --- a/controller/Dockerfile.controller +++ b/controller/Dockerfile.controller @@ -1,30 +1,121 @@ ARG builderImage="acnpublic.azurecr.io/retina-builder:0.0.1" ARG toolsImage="acnpublic.azurecr.io/retina-tools:0.0.1" -# Stage: Builder -FROM ${builderImage} as builder +# Stage: Build binary +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/oss/go/microsoft/golang:1.21 AS builder +LABEL Name=retina-builder Version=0.0.1 + +RUN apt-get update &&\ + apt-get -y install lsb-release wget software-properties-common gnupg file git make + +RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - +RUN add-apt-repository "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-14 main" +RUN apt-get update + +RUN apt-get install -y clang-14 lldb-14 lld-14 clangd-14 man-db +RUN apt-get install -y bpftool libbpf-dev + +RUN ln -s /usr/bin/clang-14 /usr/bin/clang + +COPY . /go/src/github.com/microsoft/retina +WORKDIR /go/src/github.com/microsoft/retina + +# Default linux/architecture. +ARG GOOS=linux +ENV GOOS=${GOOS} + +ARG GOARCH=amd64 +ENV GOARCH=${GOARCH} + +ENV CGO_ENABLED=0 + +ARG VERSION +# Application Insights ID for telemetry. +# Default value is empty. +# Set this if you want to enable Application Insights telemetry. +ARG APP_INSIGHTS_ID + +RUN go build -v -o /go/bin/retina/controller -ldflags "-X main.version="$VERSION" -X "main.applicationInsightsID"="$APP_INSIGHTS_ID"" controller/main.go +RUN go build -v -o /go/bin/retina/captureworkload -ldflags "-X main.version="$VERSION" -X "main.applicationInsightsID"="$APP_INSIGHTS_ID"" captureworkload/main.go +RUN go build -v -o /go/bin/retina/initretina -ldflags "-X main.version="$VERSION" -X "main.applicationInsightsID"="$APP_INSIGHTS_ID"" ././init/retina/main.go + # ----------------------------------------------------------------------------------- # -# Stage: Tools -FROM ${toolsImage} as clang +# Stage: Prepare clang and tools +# Bullseye -> debian11 +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/mirror/docker/library/debian:bullseye@sha256:a648e10e02af129706b1fb89e1ac9694ae3db7f2b8439aa906321e68cc281bc0 AS tools +LABEL Name=retina-tools Version=0.0.1 + +WORKDIR /tmp +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + curl xz-utils binutils wget gnupg2 +RUN apt-get install -y --no-install-recommends \ + ca-certificates tcpdump iproute2 iptables + +RUN mkdir -p /usr/local + +ARG GOARCH=amd64 +ENV GOARCH=${GOARCH} + +RUN if [ "$GOARCH" = "amd64" ] ; then \ + # Download clang and llvm. + wget https://releases.llvm.org/release-keys.asc; \ + gpg2 --import release-keys.asc; \ + wget -O clang+llvm.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.0/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz; \ + wget -O clang+llvm.tar.xz.sig https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.0/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz.sig; \ + gpg2 --verify clang+llvm.tar.xz.sig clang+llvm.tar.xz; \ + tar -C /usr/local -xJf ./clang+llvm.tar.xz --no-same-owner; \ + mv /usr/local/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04 /usr/local/clang+llvm; \ + else \ + # GOARCH=Arm64. + # Download clang and llvm. + wget -O clang+llvm.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.0/clang+llvm-14.0.0-aarch64-linux-gnu.tar.xz; \ + tar -C /usr/local -xJf ./clang+llvm.tar.xz --no-same-owner; \ + mv /usr/local/clang+llvm-14.0.0-aarch64-linux-gnu /usr/local/clang+llvm; \ + # Prepare dependencies for clang to be copied to base. + apt-get install -y --no-install-recommends libncurses5; \ + fi + +# Copy tools. +RUN mkdir -p /tmp/bin +RUN arr="tcpdump ip ss iptables-legacy iptables-legacy-save iptables-nft iptables-nft-save cp uname" ;\ + for i in $arr; do \ + cp $(which $i) /tmp/bin; \ + done + +# ----------------------------------------------------------------------------------- # + +# Stage: Base distroless init image +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/mirror/gcr/distroless/cc-debian11:latest@sha256:b53fbf5f81f4a120a489fedff2092e6fcbeacf7863fce3e45d99cc58dc230ccc as init +LABEL Name=retina-init Version=0.0.1 + +COPY --from=builder /go/bin/retina/initretina /retina/initretina + +# Copy dependencies for mount. Needed for initretina. +COPY --from=tools /lib/ /lib +COPY --from=tools /usr/lib/ /usr/lib +COPY --from=tools /bin/mount /bin/mount + +ENTRYPOINT ["./retina/initretina"] # ----------------------------------------------------------------------------------- # # Stage: Base distroless image -FROM mcr.microsoft.com/mirror/gcr/distroless/cc-debian11:latest@sha256:b53fbf5f81f4a120a489fedff2092e6fcbeacf7863fce3e45d99cc58dc230ccc as base +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/mirror/gcr/distroless/cc-debian11:latest@sha256:b53fbf5f81f4a120a489fedff2092e6fcbeacf7863fce3e45d99cc58dc230ccc as agent LABEL Name=retina-controller Version=0.0.1 # Copy dependencies for clang and tools. -COPY --from=clang /lib/ /lib -COPY --from=clang /usr/lib/ /usr/lib +COPY --from=tools /lib/ /lib +COPY --from=tools /usr/lib/ /usr/lib # Copy clang+llvm. -COPY --from=clang /usr/local/clang+llvm/bin/clang /bin +COPY --from=tools /usr/local/clang+llvm/bin/clang /bin # Copy tools. -COPY --from=clang /tmp/bin/ /bin +COPY --from=tools /tmp/bin/ /bin # Copy the Retina binary. COPY --from=builder /go/bin/retina/controller /retina/controller diff --git a/controller/Dockerfile.gogen b/controller/Dockerfile.gogen index ffc35dc990..cdc327da7a 100644 --- a/controller/Dockerfile.gogen +++ b/controller/Dockerfile.gogen @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/oss/go/microsoft/golang:1.21 +FROM mcr.microsoft.com/oss/go/microsoft/golang:1.21 as gogen-tools # Default linux/architecture. ARG GOOS=linux @@ -10,6 +10,7 @@ ENV GOARCH=${GOARCH} RUN apt-get update &&\ apt-get -y install lsb-release wget software-properties-common gnupg file git make +# Install clang and llvm. RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - RUN add-apt-repository "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-14 main" RUN apt-get update @@ -21,4 +22,5 @@ RUN ln -s /usr/bin/clang-14 /usr/bin/clang WORKDIR /app -ENTRYPOINT mkdir /tmp/.cache && export GOCACHE=/tmp/.cache && go generate ./... +# Generate go code. +ENTRYPOINT mkdir /tmp/.cache && export GOCACHE=/tmp/.cache && CGO_ENABLED=0 go generate ./... diff --git a/controller/Dockerfile.init b/controller/Dockerfile.init deleted file mode 100644 index 2509ec9c05..0000000000 --- a/controller/Dockerfile.init +++ /dev/null @@ -1,25 +0,0 @@ -ARG builderImage="acnpublic.azurecr.io/retina-builder:0.0.1" -ARG toolsImage="acnpublic.azurecr.io/retina-tools:0.0.1" - -# Stage: Builder -FROM ${builderImage} as builder - -# ----------------------------------------------------------------------------------- # - -# Stage: Tools -FROM ${toolsImage} as tools - -# ----------------------------------------------------------------------------------- # - -# Stage: Base distroless image -FROM mcr.microsoft.com/mirror/gcr/distroless/cc-debian11:latest@sha256:b53fbf5f81f4a120a489fedff2092e6fcbeacf7863fce3e45d99cc58dc230ccc as base -LABEL Name=retina-filtermap-init Version=0.0.1 - -COPY --from=builder /go/bin/retina/initretina /retina/initretina - -# Copy dependencies for mount. Needed for initretina. -COPY --from=tools /lib/ /lib -COPY --from=tools /usr/lib/ /usr/lib -COPY --from=tools /bin/mount /bin/mount - -ENTRYPOINT ["./retina/initretina"] diff --git a/controller/Dockerfile.proto b/controller/Dockerfile.proto index 68d6fb015f..efbc5b57ad 100644 --- a/controller/Dockerfile.proto +++ b/controller/Dockerfile.proto @@ -1,4 +1,4 @@ -FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.21 AS builder +FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.21 AS proto-tools LABEL Name=retina-builder Version=0.0.1 RUN apt-get update &&\ @@ -6,7 +6,7 @@ RUN apt-get update &&\ WORKDIR /tmp -RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@latest +RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.31.0 RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v24.2/protoc-24.2-linux-x86_64.zip RUN unzip protoc-24.2-linux-x86_64.zip -d protoc RUN mv protoc/bin/protoc /usr/bin/protoc diff --git a/controller/Dockerfile.tools b/controller/Dockerfile.tools deleted file mode 100644 index 094eb15756..0000000000 --- a/controller/Dockerfile.tools +++ /dev/null @@ -1,41 +0,0 @@ -# Stage: Prepare clang and tools -# Bullseye -> debian11 -FROM mcr.microsoft.com/mirror/docker/library/debian:bullseye@sha256:a648e10e02af129706b1fb89e1ac9694ae3db7f2b8439aa906321e68cc281bc0 -LABEL Name=retina-tools Version=0.0.1 - -WORKDIR /tmp -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - curl xz-utils binutils wget gnupg2 \ - ca-certificates tcpdump iproute2 iptables - -RUN mkdir -p /usr/local - -ARG GOARCH=amd64 -ENV GOARCH=${GOARCH} - -RUN if [ "$GOARCH" = "amd64" ] ; then \ - # Download clang and llvm. - wget https://releases.llvm.org/release-keys.asc; \ - gpg2 --import release-keys.asc; \ - wget -O clang+llvm.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.0/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz; \ - wget -O clang+llvm.tar.xz.sig https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.0/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz.sig; \ - gpg2 --verify clang+llvm.tar.xz.sig clang+llvm.tar.xz; \ - tar -C /usr/local -xJf ./clang+llvm.tar.xz --no-same-owner; \ - mv /usr/local/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04 /usr/local/clang+llvm; \ - else \ - # GOARCH=Arm64. - # Download clang and llvm. - wget -O clang+llvm.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.0/clang+llvm-14.0.0-aarch64-linux-gnu.tar.xz; \ - tar -C /usr/local -xJf ./clang+llvm.tar.xz --no-same-owner; \ - mv /usr/local/clang+llvm-14.0.0-aarch64-linux-gnu /usr/local/clang+llvm; \ - # Prepare dependencies for clang to be copied to base. - apt-get install -y --no-install-recommends libncurses5; \ - fi - -# Copy tools. -RUN mkdir -p /tmp/bin -RUN arr="tcpdump ip ss iptables-legacy iptables-legacy-save iptables-nft iptables-nft-save cp uname" ;\ - for i in $arr; do \ - cp $(which $i) /tmp/bin; \ - done