diff --git a/.azure-pipelines/template-variables.yml b/.azure-pipelines/template-variables.yml index 462a417c0173..027dfee9986a 100644 --- a/.azure-pipelines/template-variables.yml +++ b/.azure-pipelines/template-variables.yml @@ -2,3 +2,5 @@ variables: DEFAULT_CONTAINER_REGISTRY: 'publicmirror.azurecr.io' COMMON_LIB_BUILD_ENVS: 'bullseye' SONIC_SLAVE_DOCKER_DRIVER: 'overlay2' + SONIC_BUILD_RETRY_COUNT: 3 + SONIC_BUILD_RETRY_INTERVAL: 600 diff --git a/Makefile b/Makefile index cd9c91a7bf16..5ebb6b6a05ae 100644 --- a/Makefile +++ b/Makefile @@ -33,20 +33,21 @@ PLATFORM_PATH := platform/$(if $(PLATFORM),$(PLATFORM),$(CONFIGURED_PLATFORM)) PLATFORM_CHECKOUT := platform/checkout PLATFORM_CHECKOUT_FILE := $(PLATFORM_CHECKOUT)/$(PLATFORM).ini PLATFORM_CHECKOUT_CMD := $(shell if [ -f $(PLATFORM_CHECKOUT_FILE) ]; then PLATFORM_PATH=$(PLATFORM_PATH) j2 $(PLATFORM_CHECKOUT)/template.j2 $(PLATFORM_CHECKOUT_FILE); fi) +MAKE_WITH_RETRY := ./scripts/run_with_retry $(MAKE) %:: @echo "+++ --- Making $@ --- +++" ifeq ($(NOJESSIE), 0) - EXTRA_DOCKER_TARGETS=$(notdir $@) $(MAKE) -f Makefile.work jessie + $(MAKE_WITH_RETRY) EXTRA_DOCKER_TARGETS=$(notdir $@) -f Makefile.work jessie endif ifeq ($(NOSTRETCH), 0) - EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=stretch $(MAKE) -f Makefile.work stretch + $(MAKE_WITH_RETRY) EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=stretch -f Makefile.work stretch endif ifeq ($(NOBUSTER), 0) - EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=buster $(MAKE) -f Makefile.work buster + $(MAKE_WITH_RETRY) EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=buster -f Makefile.work buster endif ifeq ($(NOBULLSEYE), 0) - BLDENV=bullseye $(MAKE) -f Makefile.work $@ + $(MAKE_WITH_RETRY) BLDENV=bullseye -f Makefile.work $@ endif BLDENV=bullseye $(MAKE) -f Makefile.work docker-cleanup diff --git a/scripts/run_with_retry b/scripts/run_with_retry new file mode 100755 index 000000000000..9e709bbf1bda --- /dev/null +++ b/scripts/run_with_retry @@ -0,0 +1,17 @@ +#!/bin/bash + +run_with_retry(){ + [ "$SONIC_BUILD_RETRY_COUNT" -gt 0 ] || SONIC_BUILD_RETRY_COUNT=0 + [[ "$*" == "" ]] && { echo "run_with_retry: input command can't be empty." 1>&2;exit 1; } + for ((i=0; i<=$SONIC_BUILD_RETRY_COUNT; i++)) + do + if [[ $i != 0 ]];then + echo "==============================================================================" 1>&2 + echo "Waiting $SONIC_BUILD_RETRY_INTERVAL to run again, $i/$SONIC_BUILD_RETRY_COUNT" 1>&2 + echo "==============================================================================" 1>&2 + sleep $SONIC_BUILD_RETRY_INTERVAL + fi + "$@" && break + done +} +run_with_retry "$@"