test: print more logs if the upgrade tests fail (#8878)

Description: Sometimes tests fail and container logs are empty. Now, we
also print the output of inspecting the docker containers. We
additionally also use existing dgraph repo on disk to clone a copy
instead of downloading the whole repo again from GitHub for building
binaries.
Closes: DGRAPHCORE-285 DGRAPHCORE-280

.github/workflows/ci-dgraph-upgrade-tests.yml

@@ -21,6 +21,8 @@ jobs:
     runs-on: [self-hosted, x64]
     steps:
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
       - name: Get Go Version
         run: |
           #!/bin/bash

dgraphtest/image.go

@@ -64,18 +64,41 @@ func ensureDgraphClone() error {
 		return runGitClone()
 	}
 
-	return runGitFetch()
+	// we do not return error if git fetch fails because maybe there are no changes
+	// to pull and it doesn't make sense to fail right now. We can fail later when we
+	// do not find the reference that we are looking for.
+	if err := runGitFetch(); err != nil {
+		log.Printf("[WARNING] error in fetching latest git changes: %v", err)
+	}
+	return nil
 }
 
 func cleanupRepo() error {
 	return os.RemoveAll(repoDir)
 }
 
 func runGitClone() error {
-	cmd := exec.Command("git", "clone", dgraphRepoUrl, repoDir)
+	// The dgraph repo is already cloned for running the test. We can just create
+	// a copy of this folder by running git clone using this already cloned dgraph
+	// repo. After the quick clone, we update the original URL to point to the
+	// GitHub dgraph repo and perform a "git fetch".
+	cmd := exec.Command("git", "clone", baseRepoDir, repoDir)
 	if out, err := cmd.CombinedOutput(); err != nil {
 		return errors.Wrapf(err, "error cloning dgraph repo\noutput:%v", string(out))
 	}
+
+	cmd = exec.Command("git", "remote", "set-url", "origin", dgraphRepoUrl)
+	cmd.Dir = repoDir
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return errors.Wrapf(err, "error setting remote URL\noutput:%v", string(out))
+	}
+
+	// we do not return error if git fetch fails because maybe there are no changes
+	// to pull and it doesn't make sense to fail right now. We can fail later when we
+	// do not find the reference that we are looking for.
+	if err := runGitFetch(); err != nil {
+		log.Printf("[WARNING] error in fetching latest git changes in runGitClone: %v", err)
+	}
 	return nil
 }
 

dgraphtest/local_cluster.go

@@ -211,6 +211,9 @@ func (c *LocalCluster) Cleanup(verbose bool) {
 		if err := c.printAllLogs(); err != nil {
 			log.Printf("[WARNING] error printing container logs: %v", err)
 		}
+		if err := c.printInspectContainers(); err != nil {
+			log.Printf("[WARNING] error printing inspect container output: %v", err)
+		}
 	}
 
 	log.Printf("[INFO] cleaning up cluster with prefix [%v]", c.conf.prefix)
@@ -734,3 +737,41 @@ func (c *LocalCluster) getLogs(containerID string) (string, error) {
 	}
 	return string(data), nil
 }
+
+func (c *LocalCluster) printInspectContainers() error {
+	log.Printf("[INFO] inspecting all container for cluster with prefix [%v]", c.conf.prefix)
+	var finalErr error
+	for _, zo := range c.zeros {
+		if err := c.printInspectFor(zo.containerName); err != nil {
+			finalErr = fmt.Errorf("%v; %v", finalErr, err)
+		}
+	}
+	for _, aa := range c.alphas {
+		if err := c.printInspectFor(aa.containerName); err != nil {
+			finalErr = fmt.Errorf("%v; %v", finalErr, err)
+		}
+	}
+	return finalErr
+}
+
+func (c *LocalCluster) printInspectFor(containerID string) error {
+	inspectData, err := c.inspectContainer(containerID)
+	if err != nil {
+		return err
+	}
+
+	log.Printf("[INFO] ======== INSPECTING CONTAINER [%v] ========", containerID)
+	log.Println(inspectData)
+	return nil
+}
+
+func (c *LocalCluster) inspectContainer(containerID string) (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
+	defer cancel()
+
+	_, raw, err := c.dcli.ContainerInspectWithRaw(ctx, containerID, true)
+	if err != nil {
+		return "", errors.Wrapf(err, "error inspecting container %v", containerID)
+	}
+	return string(raw), nil
+}

dgraphtest/paths.go

@@ -26,6 +26,7 @@ import (
 )
 
 var (
+	baseRepoDir   string // baseRepoDir points to the dgraph repo from where tests are running
 	repoDir       string // repoDir to store cloned repository of dgraph
 	binDir        string // binDir to store multiple binary versions
 	encKeyPath    string
@@ -44,13 +45,15 @@ func init() {
 	// setup paths
 	_, thisFilePath, _, _ := runtime.Caller(0)
 	basePath := strings.ReplaceAll(thisFilePath, "/paths.go", "")
-	binDir = filepath.Join(basePath, "binaries")
-	encKeyPath = filepath.Join(basePath, "data", "enc-key")
-	aclSecretPath = filepath.Join(basePath, "data", "hmac-secret")
+	baseRepoDir = strings.ReplaceAll(basePath, "/dgraphtest", "")
 
 	var err error
 	repoDir, err = os.MkdirTemp("", "dgraph-repo")
 	if err != nil {
 		panic(err)
 	}
+
+	binDir = filepath.Join(basePath, "binaries")
+	encKeyPath = filepath.Join(basePath, "data", "enc-key")
+	aclSecretPath = filepath.Join(basePath, "data", "hmac-secret")
 }

systest/multi-tenancy/upgrade_test.go

@@ -41,7 +41,10 @@ func (msuite *MultitenancyTestSuite) SetupTest() {
 		WithACL(20 * time.Second).WithEncryption().WithVersion(msuite.uc.Before)
 	c, err := dgraphtest.NewLocalCluster(conf)
 	x.Panic(err)
-	x.Panic(c.Start())
+	if err := c.Start(); err != nil {
+		c.Cleanup(true)
+		panic(err)
+	}
 
 	msuite.dc = c
 	msuite.lc = c