From 28a7cb98add5a8e2414e235e7f2838a872b19a74 Mon Sep 17 00:00:00 2001 From: Dan Lorenc Date: Wed, 24 Jul 2019 10:19:57 -0500 Subject: [PATCH] Change the behavior of outputs that are also used as inputs. This change makes the handling of Resources within a Task consistent, regardless of whether the same Resource is used as both an input and an output. Previously these were special cased, which made it hard to write Tasks consistently. This commit also makes a few minor changes to the way our bash output gets logged. I discovered this was missing during debugging, and made it consistent with the gsutil wrapper. This is a followup to https://github.com/tektoncd/pipeline/pull/1119 and should be submitted once the next release is cut. --- cmd/bash/main.go | 7 +++-- examples/pipelineruns/output-pipelinerun.yaml | 2 +- .../taskrun/resources/output_resource.go | 26 ++++--------------- .../taskrun/resources/output_resource_test.go | 14 +++++----- .../v1alpha1/taskrun/taskrun_test.go | 4 +-- test/dag_test.go | 1 + 6 files changed, 19 insertions(+), 35 deletions(-) diff --git a/cmd/bash/main.go b/cmd/bash/main.go index 1c04f0b0548..2f1fd4826e7 100644 --- a/cmd/bash/main.go +++ b/cmd/bash/main.go @@ -42,8 +42,8 @@ package main import ( "flag" - "log" "os/exec" + "strings" "github.com/tektoncd/pipeline/pkg/logging" ) @@ -59,8 +59,7 @@ func main() { cmd := exec.Command("sh", "-c", *args) stdoutStderr, err := cmd.CombinedOutput() if err != nil { - logger.Errorf("Error executing command %q with arguments %s", *args, stdoutStderr) - log.Fatal(err) + logger.Fatalf("Error executing command %q ; error %s; cmd_output %s", strings.Join(cmd.Args, " "), err.Error(), stdoutStderr) } - logger.Infof("Successfully executed command %q", *args) + logger.Infof("Successfully executed command %q; output %s", strings.Join(cmd.Args, " "), stdoutStderr) } diff --git a/examples/pipelineruns/output-pipelinerun.yaml b/examples/pipelineruns/output-pipelinerun.yaml index 53e151f14b9..67eb4b86e1f 100644 --- a/examples/pipelineruns/output-pipelinerun.yaml +++ b/examples/pipelineruns/output-pipelinerun.yaml @@ -33,7 +33,7 @@ spec: - name: write-new-stuff image: ubuntu command: ['bash'] - args: ['-c', 'echo some stuff > /workspace/damnworkspace/stuff'] + args: ['-c', 'ln -s /workspace/damnworkspace /workspace/output/workspace && echo some stuff > /workspace/output/workspace/stuff'] --- # Reads a file from a predefined path in the workspace git PipelineResource apiVersion: tekton.dev/v1alpha1 diff --git a/pkg/reconciler/v1alpha1/taskrun/resources/output_resource.go b/pkg/reconciler/v1alpha1/taskrun/resources/output_resource.go index 9633ef249e7..8c1f4f4cbe8 100644 --- a/pkg/reconciler/v1alpha1/taskrun/resources/output_resource.go +++ b/pkg/reconciler/v1alpha1/taskrun/resources/output_resource.go @@ -47,7 +47,7 @@ var ( // upload steps (like upload to blob store ) // // Resource source path determined -// 1. If resource is declared in inputs then target path from input resource is used to identify source path +// 2. If resource has a targetpath that is used. Otherwise: // 2. If resource is declared in outputs only then the default is /output/resource_name func AddOutputResources( kubeclient kubernetes.Interface, @@ -70,15 +70,6 @@ func AddOutputResources( return nil, err } - // track resources that are present in input of task cuz these resources will be copied onto PVC - inputResourceMap := map[string]string{} - - if taskSpec.Inputs != nil { - for _, input := range taskSpec.Inputs.Resources { - inputResourceMap[input.Name] = destinationPath(input.Name, input.TargetPath) - } - } - for _, output := range taskSpec.Outputs.Resources { boundResource, err := getBoundResource(output.Name, taskRun.Spec.Outputs.Resources) if err != nil { @@ -90,18 +81,11 @@ func AddOutputResources( return nil, xerrors.Errorf("failed to get output pipeline Resource for task %q resource %v", taskName, boundResource) } - // if resource is declared in input then copy outputs to pvc - // To build copy step it needs source path(which is targetpath of input resourcemap) from task input source - sourcePath := inputResourceMap[boundResource.Name] - if sourcePath != "" { - logger.Warn(`This task uses the same resource as an input and output. The behavior of this will change in a future release. - See https://github.com/tektoncd/pipeline/issues/1118 for more information.`) + var sourcePath string + if output.TargetPath == "" { + sourcePath = filepath.Join(outputDir, boundResource.Name) } else { - if output.TargetPath == "" { - sourcePath = filepath.Join(outputDir, boundResource.Name) - } else { - sourcePath = output.TargetPath - } + sourcePath = output.TargetPath } resourceSteps, err := resource.GetUploadSteps(sourcePath) diff --git a/pkg/reconciler/v1alpha1/taskrun/resources/output_resource_test.go b/pkg/reconciler/v1alpha1/taskrun/resources/output_resource_test.go index ad8605b202f..9c2587716a9 100644 --- a/pkg/reconciler/v1alpha1/taskrun/resources/output_resource_test.go +++ b/pkg/reconciler/v1alpha1/taskrun/resources/output_resource_test.go @@ -162,7 +162,7 @@ func TestValidOutputResources(t *testing.T) { Name: "create-dir-source-workspace-mssqb", Image: "override-with-bash-noop:latest", Command: []string{"/ko-app/bash"}, - Args: []string{"-args", "mkdir -p /workspace/source-workspace"}, + Args: []string{"-args", "mkdir -p /workspace/output/source-workspace"}, }}, {Container: corev1.Container{ Name: "source-mkdir-source-git-9l9zj", Image: "override-with-bash-noop:latest", @@ -176,7 +176,7 @@ func TestValidOutputResources(t *testing.T) { Name: "source-copy-source-git-mz4c7", Image: "override-with-bash-noop:latest", Command: []string{"/ko-app/bash"}, - Args: []string{"-args", "cp -r /workspace/source-workspace/. pipeline-task-name"}, + Args: []string{"-args", "cp -r /workspace/output/source-workspace/. pipeline-task-name"}, VolumeMounts: []corev1.VolumeMount{{ Name: "pipelinerun-pvc", MountPath: "/pvc", @@ -385,7 +385,7 @@ func TestValidOutputResources(t *testing.T) { Name: "create-dir-source-workspace-78c5n", Image: "override-with-bash-noop:latest", Command: []string{"/ko-app/bash"}, - Args: []string{"-args", "mkdir -p /workspace/faraway-disk"}, + Args: []string{"-args", "mkdir -p /workspace/output/source-workspace"}, }}, {Container: corev1.Container{ Name: "upload-source-gcs-9l9zj", Image: "override-with-gsutil-image:latest", @@ -394,7 +394,7 @@ func TestValidOutputResources(t *testing.T) { MountPath: "/var/secret/sname", }}, Command: []string{"/ko-app/gsutil"}, - Args: []string{"-args", "rsync -d -r /workspace/faraway-disk gs://some-bucket"}, + Args: []string{"-args", "rsync -d -r /workspace/output/source-workspace gs://some-bucket"}, Env: []corev1.EnvVar{{ Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: "/var/secret/sname/key.json", }}, @@ -408,7 +408,7 @@ func TestValidOutputResources(t *testing.T) { Name: "source-copy-source-gcs-mssqb", Image: "override-with-bash-noop:latest", Command: []string{"/ko-app/bash"}, - Args: []string{"-args", "cp -r /workspace/faraway-disk/. pipeline-task-path"}, + Args: []string{"-args", "cp -r /workspace/output/source-workspace/. pipeline-task-path"}, VolumeMounts: []corev1.VolumeMount{{Name: "pipelinerun-parent-pvc", MountPath: "/pvc"}}, }}}, wantVolumes: []corev1.Volume{{ @@ -827,12 +827,12 @@ func TestValidOutputResourcesWithBucketStorage(t *testing.T) { Name: "create-dir-source-workspace-mz4c7", Image: "override-with-bash-noop:latest", Command: []string{"/ko-app/bash"}, - Args: []string{"-args", "mkdir -p /workspace/source-workspace"}, + Args: []string{"-args", "mkdir -p /workspace/output/source-workspace"}, }}, {Container: corev1.Container{ Name: "artifact-copy-to-source-git-9l9zj", Image: "override-with-gsutil-image:latest", Command: []string{"/ko-app/gsutil"}, - Args: []string{"-args", "cp -P -r /workspace/source-workspace gs://fake-bucket/pipeline-task-name"}, + Args: []string{"-args", "cp -P -r /workspace/output/source-workspace gs://fake-bucket/pipeline-task-name"}, }}}, }, { name: "git resource in output only with bucket storage", diff --git a/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go b/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go index 9fb610ead08..ce64bbecffd 100644 --- a/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go +++ b/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go @@ -600,7 +600,7 @@ func TestReconcile(t *testing.T) { tb.PodRestartPolicy(corev1.RestartPolicyNever), getCredentialsInitContainer("l22wn"), getPlaceToolsInitContainer(), - getMkdirResourceContainer("git-resource", "/workspace/git-resource", "vr6ds"), + getMkdirResourceContainer("git-resource", "/workspace/output/git-resource", "vr6ds"), tb.PodContainer("step-create-dir-another-git-resource-78c5n", "override-with-bash-noop:latest", tb.Command(entrypointLocation), tb.Args("-wait_file", "/builder/tools/0", "-post_file", "/builder/tools/1", "-entrypoint", "/ko-app/bash", "--", @@ -696,7 +696,7 @@ func TestReconcile(t *testing.T) { tb.PodContainer("step-source-copy-git-resource-j2tds", "override-with-bash-noop:latest", tb.Command(entrypointLocation), tb.Args("-wait_file", "/builder/tools/6", "-post_file", "/builder/tools/7", "-entrypoint", "/ko-app/bash", "--", - "-args", "cp -r /workspace/git-resource/. output-folder"), + "-args", "cp -r /workspace/output/git-resource/. output-folder"), tb.WorkingDir(workspaceDir), tb.EnvVar("HOME", "/builder/home"), tb.VolumeMount("test-pvc", "/pvc"), diff --git a/test/dag_test.go b/test/dag_test.go index f3c750fbecb..e2fb18e12de 100644 --- a/test/dag_test.go +++ b/test/dag_test.go @@ -56,6 +56,7 @@ func TestDAGPipelineRun(t *testing.T) { ), tb.TaskOutputs(tb.OutputsResource("repo", v1alpha1.PipelineResourceTypeGit)), tb.Step("echo-text", "busybox", tb.StepCommand("echo"), tb.StepArgs("$(inputs.params.text)")), + tb.Step("ln", "busybox", tb.StepCommand("ln"), tb.Args("-s", "${inputs.resources.repo.path}", "${outputs.resources.repo.path}")), )) if _, err := c.TaskClient.Create(echoTask); err != nil { t.Fatalf("Failed to create echo Task: %s", err)