diff --git a/cmd/s2i/main.go b/cmd/s2i/main.go index 0c66b07d6..fe7eb2ef2 100644 --- a/cmd/s2i/main.go +++ b/cmd/s2i/main.go @@ -25,6 +25,7 @@ import ( "github.com/openshift/source-to-image/pkg/docker" "github.com/openshift/source-to-image/pkg/errors" "github.com/openshift/source-to-image/pkg/run" + "github.com/openshift/source-to-image/pkg/tar" "github.com/openshift/source-to-image/pkg/util" "github.com/openshift/source-to-image/pkg/version" ) @@ -171,6 +172,7 @@ $ s2i build . centos/ruby-22-centos7 hello-world-app buildCmd.Flags().StringVarP(&(cfg.Ref), "ref", "r", "", "Specify a ref to check-out") buildCmd.Flags().StringVarP(&(cfg.AssembleUser), "assemble-user", "", "", "Specify the user to run assemble with") buildCmd.Flags().StringVarP(&(cfg.ContextDir), "context-dir", "", "", "Specify the sub-directory inside the repository with the application sources") + buildCmd.Flags().StringVarP(&(cfg.ExcludeRegExp), "exclude", "", tar.DefaultExclusionPattern.String(), "Regular expression for selecting files from the source tree to exclude from the build, where the default excludes the '.git' directory (see https://golang.org/pkg/regexp for syntax, but note that \"\" will be interpreted as allow all files and exclude no files)") buildCmd.Flags().StringVarP(&(cfg.ScriptsURL), "scripts-url", "s", "", "Specify a URL for the assemble and run scripts") buildCmd.Flags().StringVar(&(oldScriptsFlag), "scripts", "", "DEPRECATED: Specify a URL for the assemble and run scripts") buildCmd.Flags().BoolVar(&(useConfig), "use-config", false, "Store command line options to .stifile") diff --git a/contrib/bash/s2i b/contrib/bash/s2i index fec6ed215..87418fb49 100644 --- a/contrib/bash/s2i +++ b/contrib/bash/s2i @@ -212,6 +212,7 @@ _s2i_build() two_word_flags+=("-e") flags+=("--environment-file=") two_word_flags+=("-E") + flags+=("--exclude=") flags+=("--force-pull") flags+=("--incremental") flags+=("--incremental-pull-policy=") diff --git a/docs/cli.md b/docs/cli.md index 73391d54f..8e26a8259 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -78,6 +78,7 @@ that image and add them to the tar streamed to the container into `/artifacts`. | `--incremental` | Try to perform an incremental build | | `-e (--env)` | Environment variables to be passed to the builder eg. `NAME=VALUE,NAME2=VALUE2,...` | | `-E (--environment-file)` | Specify the path to the file with environment | +| `--exclude` | Regular expression for selecting files from the source tree to exclude from the build, where the default excludes the '.git' directory (see https://golang.org/pkg/regexp for syntax, but note that \"\" will be interpreted as allow all files and exclude no files) | | `--force-pull` | Always pull the builder image, even if it is present locally (defaults to true) | | `--run` | Launch the resulting image after a successful build. All output from the image is being printed to help determine image's validity. In case of a long running image you will have to Ctrl-C to exit both s2i and the running container. (defaults to false) | | `-r (--ref)` | A branch/tag that the build should use instead of MASTER (applies only to Git source) | diff --git a/pkg/api/types.go b/pkg/api/types.go index 53c175e0b..72d4c3795 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -188,6 +188,10 @@ type Config struct { // ScriptDownloadProxyConfig optionally specifies the http and https proxy // to use when downloading scripts ScriptDownloadProxyConfig *ProxyConfig + + // ExcludeRegExp contains a string representation of the regular expression desired for + // deciding which files to exclude from the tar stream + ExcludeRegExp string } type ProxyConfig struct { diff --git a/pkg/build/strategies/layered/layered.go b/pkg/build/strategies/layered/layered.go index 65b1676db..e1d75b843 100644 --- a/pkg/build/strategies/layered/layered.go +++ b/pkg/build/strategies/layered/layered.go @@ -9,6 +9,7 @@ import ( "os" "path" "path/filepath" + "regexp" "time" "github.com/golang/glog" @@ -35,11 +36,13 @@ func New(config *api.Config, scripts build.ScriptsHandler, overrides build.Overr if err != nil { return nil, err } + tarHandler := tar.New() + tarHandler.SetExclusionPattern(regexp.MustCompile(config.ExcludeRegExp)) return &Layered{ docker: d, config: config, fs: util.NewFileSystem(), - tar: tar.New(), + tar: tarHandler, scripts: scripts, }, nil } diff --git a/pkg/build/strategies/sti/sti.go b/pkg/build/strategies/sti/sti.go index fe5e427bb..352517d45 100644 --- a/pkg/build/strategies/sti/sti.go +++ b/pkg/build/strategies/sti/sti.go @@ -84,6 +84,8 @@ func New(req *api.Config, overrides build.Overrides) (*STI, error) { } inst := scripts.NewInstaller(req.BuilderImage, req.ScriptsURL, req.ScriptDownloadProxyConfig, docker, req.PullAuthentication) + tarHandler := tar.New() + tarHandler.SetExclusionPattern(regexp.MustCompile(req.ExcludeRegExp)) b := &STI{ installer: inst, @@ -92,7 +94,7 @@ func New(req *api.Config, overrides build.Overrides) (*STI, error) { incrementalDocker: incrementalDocker, git: git.New(), fs: util.NewFileSystem(), - tar: tar.New(), + tar: tarHandler, callbackInvoker: util.NewCallbackInvoker(), requiredScripts: []string{api.Assemble, api.Run}, optionalScripts: []string{api.SaveArtifacts}, diff --git a/pkg/tar/tar.go b/pkg/tar/tar.go index 22c9f99a8..9ae22be6d 100644 --- a/pkg/tar/tar.go +++ b/pkg/tar/tar.go @@ -22,9 +22,9 @@ import ( // connections in which it would wait for a long time to untar and nothing would happen const defaultTimeout = 30 * time.Second -// defaultExclusionPattern is the pattern of files that will not be included in a tar +// DefaultExclusionPattern is the pattern of files that will not be included in a tar // file when creating one. By default it is any file inside a .git metadata directory -var defaultExclusionPattern = regexp.MustCompile("((^\\.git\\/)|(\\/.git\\/)|(\\/.git$))") +var DefaultExclusionPattern = regexp.MustCompile("((^\\.git\\/)|(\\/.git\\/)|(\\/.git$))") // Tar can create and extract tar files used in an STI build type Tar interface { @@ -74,7 +74,7 @@ type Tar interface { // New creates a new Tar func New() Tar { return &stiTar{ - exclude: defaultExclusionPattern, + exclude: DefaultExclusionPattern, timeout: defaultTimeout, } } @@ -174,7 +174,7 @@ func (t *stiTar) CreateTarFile(base, dir string) (string, error) { } func (t *stiTar) shouldExclude(path string) bool { - return t.exclude != nil && t.exclude.MatchString(path) + return t.exclude != nil && t.exclude.String() != "" && t.exclude.MatchString(path) } // CreateTarStream calls CreateTarStreamWithLogging with a nil logger diff --git a/pkg/tar/tar_test.go b/pkg/tar/tar_test.go index 1466878d5..b3cbd694f 100644 --- a/pkg/tar/tar_test.go +++ b/pkg/tar/tar_test.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "os" "path/filepath" + "regexp" "sync" "testing" "time" @@ -119,7 +120,7 @@ func verifyTarFile(t *testing.T, filename string, files []fileDesc, links []link } if len(filesToVerify) > 0 || len(linksToVerify) > 0 { - t.Errorf("Did not find all expected files in tar: %v, %v", filesToVerify, linksToVerify) + t.Errorf("Did not find all expected files in tar: fileToVerify %v, linksToVerify %v", filesToVerify, linksToVerify) } } @@ -192,6 +193,78 @@ func TestCreateTar(t *testing.T) { verifyTarFile(t, tarFile, testFiles, testLinks) } +func TestCreateTarIncludeDotGit(t *testing.T) { + th := New() + th.SetExclusionPattern(regexp.MustCompile("test3.txt")) + tempDir, err := ioutil.TempDir("", "testtar") + defer os.RemoveAll(tempDir) + if err != nil { + t.Fatalf("Cannot create temp directory for test: %v", err) + } + modificationDate := time.Date(2011, time.March, 5, 23, 30, 1, 0, time.UTC) + testFiles := []fileDesc{ + {"dir01/dir02/test1.txt", modificationDate, 0700, "Test1 file content", false, ""}, + {"dir01/test2.git", modificationDate, 0660, "Test2 file content", false, ""}, + {"dir01/dir03/test3.txt", modificationDate, 0444, "Test3 file content", true, ""}, + {"dir01/.git/hello.txt", modificationDate, 0600, "Allow .git content", false, ""}, + } + if err := createTestFiles(tempDir, testFiles); err != nil { + t.Fatalf("Cannot create test files: %v", err) + } + testLinks := []linkDesc{ + {"link/okfilelink", "../dir01/dir02/test1.txt"}, + {"link/errfilelink", "../dir01/missing.target"}, + {"link/okdirlink", "../dir01/dir02"}, + {"link/okdirlink2", "../dir01/.git"}, + } + if err := createTestLinks(tempDir, testLinks); err != nil { + t.Fatalf("Cannot create link files: %v", err) + } + + tarFile, err := th.CreateTarFile("", tempDir) + defer os.Remove(tarFile) + if err != nil { + t.Fatalf("Unable to create new tar upload file: %v", err) + } + verifyTarFile(t, tarFile, testFiles, testLinks) +} + +func TestCreateTarEmptyRegexp(t *testing.T) { + th := New() + th.SetExclusionPattern(regexp.MustCompile("")) + tempDir, err := ioutil.TempDir("", "testtar") + defer os.RemoveAll(tempDir) + if err != nil { + t.Fatalf("Cannot create temp directory for test: %v", err) + } + modificationDate := time.Date(2011, time.March, 5, 23, 30, 1, 0, time.UTC) + testFiles := []fileDesc{ + {"dir01/dir02/test1.txt", modificationDate, 0700, "Test1 file content", false, ""}, + {"dir01/test2.git", modificationDate, 0660, "Test2 file content", false, ""}, + {"dir01/dir03/test3.txt", modificationDate, 0444, "Test3 file content", false, ""}, + {"dir01/.git/hello.txt", modificationDate, 0600, "Allow .git content", false, ""}, + } + if err := createTestFiles(tempDir, testFiles); err != nil { + t.Fatalf("Cannot create test files: %v", err) + } + testLinks := []linkDesc{ + {"link/okfilelink", "../dir01/dir02/test1.txt"}, + {"link/errfilelink", "../dir01/missing.target"}, + {"link/okdirlink", "../dir01/dir02"}, + {"link/okdirlink2", "../dir01/.git"}, + } + if err := createTestLinks(tempDir, testLinks); err != nil { + t.Fatalf("Cannot create link files: %v", err) + } + + tarFile, err := th.CreateTarFile("", tempDir) + defer os.Remove(tarFile) + if err != nil { + t.Fatalf("Unable to create new tar upload file: %v", err) + } + verifyTarFile(t, tarFile, testFiles, testLinks) +} + func createTestTar(files []fileDesc, writer io.Writer) error { tw := tar.NewWriter(writer) defer tw.Close()