refactor canary test to access images from AWS registries

scripts/run-canary-test.sh

@@ -7,6 +7,9 @@ set -e
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 GINKGO_TEST_BUILD="$SCRIPT_DIR/../test/build"
+# TEST_IMAGE_REGISTRY is the registry in test-infra-* accounts where e2e test images are stored
+TEST_IMAGE_REGISTRY=${TEST_IMAGE_REGISTRY:-"617930562442.dkr.ecr.us-west-2.amazonaws.com"}
+ADC_REGIONS="us-iso-east-1 us-isob-east-1 us-iso-west-1"
 
 source "$SCRIPT_DIR"/lib/add-on.sh
 source "$SCRIPT_DIR"/lib/cluster.sh
@@ -15,8 +18,8 @@ source "$SCRIPT_DIR"/lib/canary.sh
 function run_ginkgo_test() {
   local focus=$1
   echo "Running ginkgo tests with focus: $focus"
-  (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --no-color --focus="$focus" -v --timeout 30m --fail-on-pending $GINKGO_TEST_BUILD/cni.test -- --cluster-kubeconfig="$KUBE_CONFIG_PATH" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux")
-  (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --no-color --focus="$focus" -v --timeout 30m --fail-on-pending $GINKGO_TEST_BUILD/ipamd.test -- --cluster-kubeconfig="$KUBE_CONFIG_PATH" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux")
+  (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --no-color --focus="$focus" -v --timeout 30m --fail-on-pending $GINKGO_TEST_BUILD/cni.test -- --cluster-kubeconfig="$KUBE_CONFIG_PATH" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux" --test-image-registry=$TEST_IMAGE_REGISTRY)
+  (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --no-color --focus="$focus" -v --timeout 30m --fail-on-pending $GINKGO_TEST_BUILD/ipamd.test -- --cluster-kubeconfig="$KUBE_CONFIG_PATH" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux" --test-image-registry=$TEST_IMAGE_REGISTRY)
 }
 
 load_cluster_details
@@ -31,8 +34,13 @@ run_ginkgo_test "CANARY"
 
 # Run smoke test on the latest addon version. Smoke tests consist of a subset of tests
 # from Canary suite.
-echo "Running Smoke tests on the latest addon version"
-install_add_on "$LATEST_ADDON_VERSION"
-run_ginkgo_test "SMOKE"
+# skip the latest addon version for ADC regions
+if [[ $ADC_REGIONS == *"$REGION"* ]]; then
+  echo "Skipping Smoke tests on the latest addon version"
+else
+  echo "Running Smoke tests on the latest addon version"
+  install_add_on "$LATEST_ADDON_VERSION"
+  run_ginkgo_test "SMOKE"
+fi
 
 echo "all tests ran successfully in $(($SECONDS / 60)) minutes and $(($SECONDS % 60)) seconds"

test/framework/options.go

@@ -47,6 +47,7 @@ type Options struct {
 	AvailabilityZones  string
 	PublicRouteTableID string
 	NgK8SVersion       string
+	TestImageRegistry  string
 }
 
 func (options *Options) BindFlags() {
@@ -70,6 +71,7 @@ func (options *Options) BindFlags() {
 	flag.StringVar(&options.AvailabilityZones, "availability-zones", "", "Comma separated list of private subnets (optional, if specified you must specify all of public/private-subnets, public-route-table-id, and availability-zones)")
 	flag.StringVar(&options.PublicRouteTableID, "public-route-table-id", "", "Public route table ID (optional, if specified you must specify all of public/private-subnets, public-route-table-id, and availability-zones)")
 	flag.StringVar(&options.NgK8SVersion, "ng-kubernetes-version", "1.25", `Kubernetes version for self-managed node groups (optional, default is "1.25")`)
+	flag.StringVar(&options.TestImageRegistry, "test-image-registry", "617930562442.dkr.ecr.us-west-2.amazonaws.com", `AWS registry where the e2e test images are stored`)
 }
 
 func (options *Options) Validate() error {
@@ -85,5 +87,8 @@ func (options *Options) Validate() error {
 	if len(options.AWSVPCID) == 0 {
 		return errors.Errorf("%s must be set!", "aws-vpc-id")
 	}
+	if len(options.TestImageRegistry) == 0 {
+		return errors.Errorf("%s must be set!", "test-image-registry")
+	}
 	return nil
 }

test/framework/resources/agent/traffic_tester.go

@@ -170,7 +170,7 @@ func (t *TrafficTest) TestTraffic() (float64, error) {
 }
 
 func (t *TrafficTest) startTrafficServer() (*appsV1.Deployment, error) {
-	serverContainer := manifest.NewTestHelperContainer().
+	serverContainer := manifest.NewTestHelperContainer(t.Framework.Options.TestImageRegistry).
 		Name("server").
 		Command([]string{"./traffic-server"}).
 		Args([]string{
@@ -190,7 +190,7 @@ func (t *TrafficTest) startTrafficServer() (*appsV1.Deployment, error) {
 }
 
 func (t *TrafficTest) startTrafficClient(serverAddList string, metricServerIP string) (*batchV1.Job, error) {
-	trafficClientContainer := manifest.NewTestHelperContainer().
+	trafficClientContainer := manifest.NewTestHelperContainer(t.Framework.Options.TestImageRegistry).
 		Name("client-regular-pods").
 		Command([]string{"./traffic-client"}).
 		Args([]string{
@@ -213,7 +213,7 @@ func (t *TrafficTest) startTrafficClient(serverAddList string, metricServerIP st
 }
 
 func (t *TrafficTest) startMetricServerPod() (*v1.Pod, error) {
-	metricContainer := manifest.NewTestHelperContainer().
+	metricContainer := manifest.NewTestHelperContainer(t.Framework.Options.TestImageRegistry).
 		Name("metric-container").
 		Command([]string{"./metric-server"}).
 		Build()

test/framework/resources/k8s/manifest/container.go

@@ -30,10 +30,10 @@ type Container struct {
 	securityContext *v1.SecurityContext
 }
 
-func NewBusyBoxContainerBuilder() *Container {
+func NewBusyBoxContainerBuilder(testImageRegistry string) *Container {
 	return &Container{
 		name:            "busybox",
-		image:           "busybox",
+		image:           utils.GetTestImage(testImageRegistry, utils.BusyBoxImage),
 		imagePullPolicy: v1.PullIfNotPresent,
 		command:         []string{"sleep", "3600"},
 		args:            []string{},
@@ -49,20 +49,20 @@ func NewCurlContainer() *Container {
 }
 
 // See test/agent/README.md in this repository for more details
-func NewTestHelperContainer() *Container {
+func NewTestHelperContainer(testImageRegistry string) *Container {
 	return &Container{
 		name:            "test-helper",
-		image:           utils.TestAgentImage,
+		image:           utils.GetTestImage(testImageRegistry, utils.TestAgentImage),
 		imagePullPolicy: v1.PullIfNotPresent,
 	}
 }
 
-func NewNetCatAlpineContainer() *Container {
+func NewNetCatAlpineContainer(testImageRegistry string) *Container {
 	return &Container{
 		name: "net-cat",
 		// simple netcat OpenBSD version with alpine as the base image
 		// compatible with arm64 and amd64
-		image:           "public.ecr.aws/e6v3k1j4/netcat-openbsd:v1.0",
+		image:           utils.GetTestImage(testImageRegistry, utils.NetCatImage),
 		imagePullPolicy: v1.PullIfNotPresent,
 	}
 }

test/framework/resources/k8s/manifest/deployment.go

@@ -36,12 +36,12 @@ type DeploymentBuilder struct {
 	volumeMount            []corev1.VolumeMount
 }
 
-func NewBusyBoxDeploymentBuilder() *DeploymentBuilder {
+func NewBusyBoxDeploymentBuilder(testImageRegistry string) *DeploymentBuilder {
 	return &DeploymentBuilder{
 		namespace:              utils.DefaultTestNamespace,
 		name:                   "deployment-test",
 		replicas:               10,
-		container:              NewBusyBoxContainerBuilder().Build(),
+		container:              NewBusyBoxContainerBuilder(testImageRegistry).Build(),
 		labels:                 map[string]string{"role": "test"},
 		nodeSelector:           map[string]string{"kubernetes.io/os": "linux"},
 		terminationGracePeriod: 1,

test/framework/utils/const.go

@@ -24,7 +24,10 @@ const (
 	MultusContainerName  = "kube-multus"
 
 	// See https://gallery.ecr.aws/eks/aws-vpc-cni-test-helper
-	TestAgentImage = "public.ecr.aws/eks/aws-vpc-cni-test-helper:770278ef"
+	TestAgentImage   = "networking-e2e-test-images/aws-vpc-cni-test-helper:770278ef"
+	BusyBoxImage     = "networking-e2e-test-images/busybox:latest"
+	NginxImage       = "networking-e2e-test-images/nginx:1.21.4"
+	NetCatImage      = "networking-e2e-test-images/netcat-openbsd:v1.0"
 
 	PollIntervalShort  = time.Second * 2
 	PollIntervalMedium = time.Second * 5

test/framework/utils/image.go

@@ -0,0 +1,5 @@
+package utils
+
+func GetTestImage(registry string, image string) string {
+	return registry + "/" + image
+}

test/integration/addon-tests/cni_addon_test.go

@@ -105,7 +105,7 @@ func deleteTestDeployment() {
 }
 
 func getTestPodList() common.InterfaceTypeToPodList {
-	deployment = manifest.NewBusyBoxDeploymentBuilder().
+	deployment = manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
 		Replicas(maxIPPerInterface*2).
 		PodLabel(podLabelKey, podLabelVal).
 		NodeName(primaryNode.Name).

test/integration/cni-upgrade-downgrade/host_networking_test.go

@@ -25,7 +25,7 @@ var _ = Describe("test host networking", func() {
 
 			// Launch enough pods so some pods end up using primary ENI IP and some using secondary
 			// ENI IP
-			deployment = manifest.NewBusyBoxDeploymentBuilder().
+			deployment = manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
 				Replicas(maxIPPerInterface*2).
 				PodLabel(podLabelKey, podLabelVal).
 				NodeName(primaryNode.Name).

test/integration/cni/host_networking_test.go

@@ -64,7 +64,7 @@ var _ = Describe("test host networking", func() {
 		It("should have correct host networking setup when running and cleaned up once terminated", func() {
 			// Launch enough pods so some pods end up using primary ENI IP and some using secondary
 			// ENI IP
-			deployment := manifest.NewBusyBoxDeploymentBuilder().
+			deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
 				Replicas(maxIPPerInterface*2).
 				PodLabel(podLabelKey, podLabelVal).
 				NodeName(primaryNode.Name).
@@ -105,7 +105,7 @@ var _ = Describe("test host networking", func() {
 		})
 
 		It("Validate Host Networking setup after changing MTU and Veth Prefix", func() {
-			deployment := manifest.NewBusyBoxDeploymentBuilder().
+			deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
 				Replicas(maxIPPerInterface*2).
 				PodLabel(podLabelKey, podLabelVal).
 				NodeName(primaryNode.Name).
@@ -156,7 +156,7 @@ var _ = Describe("test host networking", func() {
 		It("tester pod should error out", func() {
 			By("creating a single pod on the test node")
 			parkingPod := manifest.NewDefaultPodBuilder().
-				Container(manifest.NewBusyBoxContainerBuilder().Build()).
+				Container(manifest.NewBusyBoxContainerBuilder(f.Options.TestImageRegistry).Build()).
 				Name("parking-pod").
 				NodeName(primaryNode.Name).
 				Build()

test/integration/cni/pod_traffic_test.go

@@ -81,7 +81,7 @@ var _ = Describe("test pod networking", func() {
 		Expect(err).ToNot(HaveOccurred())
 
 		serverContainer := manifest.
-			NewNetCatAlpineContainer().
+			NewNetCatAlpineContainer(f.Options.TestImageRegistry).
 			Command(serverListenCmd).
 			Args(serverListenCmdArgs).
 			Build()

test/integration/cni/service_connectivity_test.go

@@ -57,8 +57,8 @@ var _ = Describe("[CANARY] test service connectivity", func() {
 	var negativeTesterContainer v1.Container
 
 	JustBeforeEach(func() {
-		deploymentContainer = manifest.NewBusyBoxContainerBuilder().
-			Image("nginx:1.21.4").
+		deploymentContainer = manifest.NewBusyBoxContainerBuilder(f.Options.TestImageRegistry).
+			Image(utils.GetTestImage(f.Options.TestImageRegistry, utils.NginxImage)).
 			Command(nil).
 			Port(v1.ContainerPort{
 				ContainerPort: 80,
@@ -94,7 +94,7 @@ var _ = Describe("[CANARY] test service connectivity", func() {
 		By("sleeping for some time to allow service to become ready")
 		time.Sleep(utils.PollIntervalLong)
 
-		testerContainer = manifest.NewBusyBoxContainerBuilder().
+		testerContainer = manifest.NewBusyBoxContainerBuilder(f.Options.TestImageRegistry).
 			Command([]string{"wget"}).
 			Args([]string{"--spider", "-T", "5", fmt.Sprintf("%s:%d", service.Spec.ClusterIP,
 				service.Spec.Ports[0].Port)}).
@@ -111,7 +111,7 @@ var _ = Describe("[CANARY] test service connectivity", func() {
 		Expect(err).ToNot(HaveOccurred())
 
 		// Test connection to an unreachable port should fail
-		negativeTesterContainer = manifest.NewBusyBoxContainerBuilder().
+		negativeTesterContainer = manifest.NewBusyBoxContainerBuilder(f.Options.TestImageRegistry).
 			Command([]string{"wget"}).
 			Args([]string{"--spider", "-T", "1", fmt.Sprintf("%s:%d", service.Spec.ClusterIP, 2273)}).
 			Build()

test/integration/cni/vpc_cni_logfile_test.go

@@ -25,7 +25,7 @@ var _ = Describe("aws-node env test", func() {
 
 	var _ = JustBeforeEach(func() {
 		By("Deploying a host network deployment with Volume mount")
-		container := manifest.NewBusyBoxContainerBuilder().Build()
+		container := manifest.NewBusyBoxContainerBuilder(f.Options.TestImageRegistry).Build()
 
 		volume := []v1.Volume{
 			{

test/integration/common/util.go

@@ -87,7 +87,7 @@ func ValidateHostNetworking(testType TestType, podValidationInputString string,
 		shouldTestPodError = true
 	}
 
-	testContainer := manifest.NewTestHelperContainer().
+	testContainer := manifest.NewTestHelperContainer(f.Options.TestImageRegistry).
 		Command([]string{"./networking"}).
 		Args(testerArgs).
 		Build()

test/integration/custom-networking/custom_networking_test.go

@@ -48,12 +48,12 @@ var _ = Describe("Custom Networking Test", func() {
 		})
 
 		JustBeforeEach(func() {
-			container := manifest.NewNetCatAlpineContainer().
+			container := manifest.NewNetCatAlpineContainer(f.Options.TestImageRegistry).
 				Command([]string{"nc"}).
 				Args([]string{"-k", "-l", strconv.Itoa(port)}).
 				Build()
 
-			deployment = manifest.NewBusyBoxDeploymentBuilder().
+			deployment = manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
 				Container(container).
 				Replicas(replicaCount).
 				NodeSelector(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal).
@@ -76,7 +76,7 @@ var _ = Describe("Custom Networking Test", func() {
 				ip := net.ParseIP(pod.Status.PodIP)
 				Expect(cidrRange.Contains(ip)).To(BeTrue())
 
-				testContainer := manifest.NewNetCatAlpineContainer().
+				testContainer := manifest.NewNetCatAlpineContainer(f.Options.TestImageRegistry).
 					Command([]string{"nc"}).
 					Args([]string{"-v", "-w2", pod.Status.PodIP, strconv.Itoa(port)}).
 					Build()
@@ -169,7 +169,7 @@ var _ = Describe("Custom Networking Test", func() {
 
 			// Nodes should be stuck in NotReady state since no ENIs could be attached and no pod
 			// IP addresses are available.
-			deployment := manifest.NewBusyBoxDeploymentBuilder().
+			deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
 				Replicas(2).
 				NodeSelector(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal).
 				Build()

test/integration/ipamd/eni_ip_leak_test.go

@@ -29,7 +29,7 @@ var _ = Describe("[CANARY][SMOKE] ENI/IP Leak Test", func() {
 			oldIP, oldENI := getCountOfIPandENIOnPrimaryInstance()
 
 			maxPods := getMaxApplicationPodsOnPrimaryInstance()
-			deploymentSpec := manifest.NewBusyBoxDeploymentBuilder().
+			deploymentSpec := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
 				Namespace("default").
 				Name("busybox").
 				NodeName(primaryNode.Name).

test/integration/ipv6/ipv6_host_networking_test.go

@@ -63,7 +63,7 @@ var _ = Describe("[CANARY] test ipv6 host netns setup", func() {
 			time.Sleep(utils.PollIntervalMedium)
 		})
 		It("should have correct host netns setup when running and cleaned up once terminated", func() {
-			deployment := manifest.NewBusyBoxDeploymentBuilder().
+			deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
 				Replicas(2).
 				PodLabel(podLabelKey, podLabelVal).
 				NodeName(primaryNode.Name).
@@ -99,7 +99,7 @@ var _ = Describe("[CANARY] test ipv6 host netns setup", func() {
 		})
 
 		It("Validate host netns setup after changing MTU and Veth Prefix", func() {
-			deployment := manifest.NewBusyBoxDeploymentBuilder().
+			deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
 				Replicas(2).
 				PodLabel(podLabelKey, podLabelVal).
 				NodeName(primaryNode.Name).
@@ -150,7 +150,7 @@ var _ = Describe("[CANARY] test ipv6 host netns setup", func() {
 		It("tester pod should error out", func() {
 			By("creating a single pod on the test node")
 			parkingPod := manifest.NewDefaultPodBuilder().
-				Container(manifest.NewBusyBoxContainerBuilder().Build()).
+				Container(manifest.NewBusyBoxContainerBuilder(f.Options.TestImageRegistry).Build()).
 				Name("parking-pod").
 				NodeName(primaryNode.Name).
 				Build()
@@ -219,7 +219,7 @@ func ValidateHostNetworking(testType TestType, podValidationInputString string)
 		shouldTestPodError = true
 	}
 
-	testContainer := manifest.NewTestHelperContainer().
+	testContainer := manifest.NewTestHelperContainer(f.Options.TestImageRegistry).
 		Command([]string{"./networking"}).
 		Args(testerArgs).
 		Build()

test/integration/ipv6/ipv6_service_connectivity_test.go

@@ -57,7 +57,7 @@ var _ = Describe("[CANARY] test service connectivity", func() {
 	var negativeTesterContainer v1.Container
 
 	JustBeforeEach(func() {
-		serverContainer = manifest.NewTestHelperContainer().
+		serverContainer = manifest.NewTestHelperContainer(f.Options.TestImageRegistry).
 			Name("server").
 			Command([]string{"./traffic-server"}).
 			Args([]string{
@@ -95,7 +95,7 @@ var _ = Describe("[CANARY] test service connectivity", func() {
 		By("sleeping for some time to allow service to become ready")
 		time.Sleep(utils.PollIntervalLong)
 
-		testerContainer = manifest.NewBusyBoxContainerBuilder().
+		testerContainer = manifest.NewBusyBoxContainerBuilder(f.Options.TestImageRegistry).
 			Command([]string{"wget"}).
 			Args([]string{"--spider", "-T", "5", fmt.Sprintf("[%s]:%d", service.Spec.ClusterIP,
 				service.Spec.Ports[0].Port)}).
@@ -112,7 +112,7 @@ var _ = Describe("[CANARY] test service connectivity", func() {
 		Expect(err).ToNot(HaveOccurred())
 
 		// Test connection to an unreachable port should fail
-		negativeTesterContainer = manifest.NewBusyBoxContainerBuilder().
+		negativeTesterContainer = manifest.NewBusyBoxContainerBuilder(f.Options.TestImageRegistry).
 			Command([]string{"wget"}).
 			Args([]string{"--spider", "-T", "5", fmt.Sprintf("[%s]:%d", service.Spec.ClusterIP, 2273)}).
 			Build()

test/integration/metrics-helper/metric_helper_test.go

@@ -34,7 +34,7 @@ var _ = Describe("test cni-metrics-helper publishes metrics", func() {
 		It("the updated metric is published to CW", func() {
 			// Create a new deployment to verify addReqCount is updated
 			var deployment *v1.Deployment
-			deployment = manifest.NewBusyBoxDeploymentBuilder().
+			deployment = manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
 				Replicas(10).
 				NodeName(nodeName).
 				Build()

test/integration/pod-eni/security_group_per_pod_test.go

@@ -280,7 +280,7 @@ var _ = Describe("Security Group for Pods Test", func() {
 			branchPodLabelVal = []string{busyboxPodLabelVal}
 		})
 		It("Deploy BusyBox Pods with branch ENI and verify HostNetworking", func() {
-			deployment := manifest.NewBusyBoxDeploymentBuilder().
+			deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
 				Replicas(totalBranchInterface/asgSize).
 				PodLabel(labelKey, busyboxPodLabelVal).
 				NodeName(node.Name).
@@ -346,7 +346,7 @@ func ValidateHostNetworking(testType TestType, podValidationInputString string)
 		testerArgs = append(testerArgs, "-test-cleanup=true", "-test-ppsg=true")
 	}
 
-	testContainer := manifest.NewTestHelperContainer().
+	testContainer := manifest.NewTestHelperContainer(f.Options.TestImageRegistry).
 		Command([]string{"./networking"}).
 		Args(testerArgs).
 		Build()