Merge pull request #460 from gabemontero/457

Merged by openshift-bot

cmd/s2i/main.go

@@ -25,6 +25,7 @@ import (
 	"github.com/openshift/source-to-image/pkg/docker"
 	"github.com/openshift/source-to-image/pkg/errors"
 	"github.com/openshift/source-to-image/pkg/run"
+	"github.com/openshift/source-to-image/pkg/tar"
 	"github.com/openshift/source-to-image/pkg/util"
 	"github.com/openshift/source-to-image/pkg/version"
 )
@@ -171,6 +172,7 @@ $ s2i build . centos/ruby-22-centos7 hello-world-app
 	buildCmd.Flags().StringVarP(&(cfg.Ref), "ref", "r", "", "Specify a ref to check-out")
 	buildCmd.Flags().StringVarP(&(cfg.AssembleUser), "assemble-user", "", "", "Specify the user to run assemble with")
 	buildCmd.Flags().StringVarP(&(cfg.ContextDir), "context-dir", "", "", "Specify the sub-directory inside the repository with the application sources")
+	buildCmd.Flags().StringVarP(&(cfg.ExcludeRegExp), "exclude", "", tar.DefaultExclusionPattern.String(), "Regular expression for selecting files from the source tree to exclude from the build, where the default excludes the '.git' directory (see https://golang.org/pkg/regexp for syntax, but note that \"\" will be interpreted as allow all files and exclude no files)")
 	buildCmd.Flags().StringVarP(&(cfg.ScriptsURL), "scripts-url", "s", "", "Specify a URL for the assemble and run scripts")
 	buildCmd.Flags().StringVar(&(oldScriptsFlag), "scripts", "", "DEPRECATED: Specify a URL for the assemble and run scripts")
 	buildCmd.Flags().BoolVar(&(useConfig), "use-config", false, "Store command line options to .stifile")

contrib/bash/s2i

@@ -212,6 +212,7 @@ _s2i_build()
     two_word_flags+=("-e")
     flags+=("--environment-file=")
     two_word_flags+=("-E")
+    flags+=("--exclude=")
     flags+=("--force-pull")
     flags+=("--incremental")
     flags+=("--incremental-pull-policy=")

docs/cli.md

@@ -78,6 +78,7 @@ that image and add them to the tar streamed to the container into `/artifacts`.
 | `--incremental`            | Try to perform an incremental build |
 | `-e (--env)`               | Environment variables to be passed to the builder eg. `NAME=VALUE,NAME2=VALUE2,...` |
 | `-E (--environment-file)`  | Specify the path to the file with environment |
+| `--exclude`  | Regular expression for selecting files from the source tree to exclude from the build, where the default excludes the '.git' directory (see https://golang.org/pkg/regexp for syntax, but note that \"\" will be interpreted as allow all files and exclude no files) |
 | `--force-pull`             | Always pull the builder image, even if it is present locally (defaults to true) |
 | `--run`                    | Launch the resulting image after a successful build. All output from the image is being printed to help determine image's validity. In case of a long running image you will have to Ctrl-C to exit both s2i and the running container.  (defaults to false) |
 | `-r (--ref)`               | A branch/tag that the build should use instead of MASTER (applies only to Git source) |

pkg/api/types.go

@@ -188,6 +188,10 @@ type Config struct {
 	// ScriptDownloadProxyConfig optionally specifies the http and https proxy
 	// to use when downloading scripts
 	ScriptDownloadProxyConfig *ProxyConfig
+
+	// ExcludeRegExp contains a string representation of the regular expression desired for
+	// deciding which files to exclude from the tar stream
+	ExcludeRegExp string
 }
 
 type ProxyConfig struct {

pkg/build/strategies/layered/layered.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"time"
 
 	"github.com/golang/glog"
@@ -35,11 +36,13 @@ func New(config *api.Config, scripts build.ScriptsHandler, overrides build.Overr
 	if err != nil {
 		return nil, err
 	}
+	tarHandler := tar.New()
+	tarHandler.SetExclusionPattern(regexp.MustCompile(config.ExcludeRegExp))
 	return &Layered{
 		docker:  d,
 		config:  config,
 		fs:      util.NewFileSystem(),
-		tar:     tar.New(),
+		tar:     tarHandler,
 		scripts: scripts,
 	}, nil
 }

pkg/build/strategies/sti/sti.go

@@ -84,6 +84,8 @@ func New(req *api.Config, overrides build.Overrides) (*STI, error) {
 	}
 
 	inst := scripts.NewInstaller(req.BuilderImage, req.ScriptsURL, req.ScriptDownloadProxyConfig, docker, req.PullAuthentication)
+	tarHandler := tar.New()
+	tarHandler.SetExclusionPattern(regexp.MustCompile(req.ExcludeRegExp))
 
 	b := &STI{
 		installer:         inst,
@@ -92,7 +94,7 @@ func New(req *api.Config, overrides build.Overrides) (*STI, error) {
 		incrementalDocker: incrementalDocker,
 		git:               git.New(),
 		fs:                util.NewFileSystem(),
-		tar:               tar.New(),
+		tar:               tarHandler,
 		callbackInvoker:   util.NewCallbackInvoker(),
 		requiredScripts:   []string{api.Assemble, api.Run},
 		optionalScripts:   []string{api.SaveArtifacts},

pkg/tar/tar.go

@@ -22,9 +22,9 @@ import (
 // connections in which it would wait for a long time to untar and nothing would happen
 const defaultTimeout = 30 * time.Second
 
-// defaultExclusionPattern is the pattern of files that will not be included in a tar
+// DefaultExclusionPattern is the pattern of files that will not be included in a tar
 // file when creating one. By default it is any file inside a .git metadata directory
-var defaultExclusionPattern = regexp.MustCompile("((^\\.git\\/)|(\\/.git\\/)|(\\/.git$))")
+var DefaultExclusionPattern = regexp.MustCompile("((^\\.git\\/)|(\\/.git\\/)|(\\/.git$))")
 
 // Tar can create and extract tar files used in an STI build
 type Tar interface {
@@ -74,7 +74,7 @@ type Tar interface {
 // New creates a new Tar
 func New() Tar {
 	return &stiTar{
-		exclude: defaultExclusionPattern,
+		exclude: DefaultExclusionPattern,
 		timeout: defaultTimeout,
 	}
 }
@@ -174,7 +174,7 @@ func (t *stiTar) CreateTarFile(base, dir string) (string, error) {
 }
 
 func (t *stiTar) shouldExclude(path string) bool {
-	return t.exclude != nil && t.exclude.MatchString(path)
+	return t.exclude != nil && t.exclude.String() != "" && t.exclude.MatchString(path)
 }
 
 // CreateTarStream calls CreateTarStreamWithLogging with a nil logger

pkg/tar/tar_test.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sync"
 	"testing"
 	"time"
@@ -119,7 +120,7 @@ func verifyTarFile(t *testing.T, filename string, files []fileDesc, links []link
 	}
 
 	if len(filesToVerify) > 0 || len(linksToVerify) > 0 {
-		t.Errorf("Did not find all expected files in tar: %v, %v", filesToVerify, linksToVerify)
+		t.Errorf("Did not find all expected files in tar: fileToVerify %v, linksToVerify %v", filesToVerify, linksToVerify)
 	}
 }
 
@@ -192,6 +193,78 @@ func TestCreateTar(t *testing.T) {
 	verifyTarFile(t, tarFile, testFiles, testLinks)
 }
 
+func TestCreateTarIncludeDotGit(t *testing.T) {
+	th := New()
+	th.SetExclusionPattern(regexp.MustCompile("test3.txt"))
+	tempDir, err := ioutil.TempDir("", "testtar")
+	defer os.RemoveAll(tempDir)
+	if err != nil {
+		t.Fatalf("Cannot create temp directory for test: %v", err)
+	}
+	modificationDate := time.Date(2011, time.March, 5, 23, 30, 1, 0, time.UTC)
+	testFiles := []fileDesc{
+		{"dir01/dir02/test1.txt", modificationDate, 0700, "Test1 file content", false, ""},
+		{"dir01/test2.git", modificationDate, 0660, "Test2 file content", false, ""},
+		{"dir01/dir03/test3.txt", modificationDate, 0444, "Test3 file content", true, ""},
+		{"dir01/.git/hello.txt", modificationDate, 0600, "Allow .git content", false, ""},
+	}
+	if err := createTestFiles(tempDir, testFiles); err != nil {
+		t.Fatalf("Cannot create test files: %v", err)
+	}
+	testLinks := []linkDesc{
+		{"link/okfilelink", "../dir01/dir02/test1.txt"},
+		{"link/errfilelink", "../dir01/missing.target"},
+		{"link/okdirlink", "../dir01/dir02"},
+		{"link/okdirlink2", "../dir01/.git"},
+	}
+	if err := createTestLinks(tempDir, testLinks); err != nil {
+		t.Fatalf("Cannot create link files: %v", err)
+	}
+
+	tarFile, err := th.CreateTarFile("", tempDir)
+	defer os.Remove(tarFile)
+	if err != nil {
+		t.Fatalf("Unable to create new tar upload file: %v", err)
+	}
+	verifyTarFile(t, tarFile, testFiles, testLinks)
+}
+
+func TestCreateTarEmptyRegexp(t *testing.T) {
+	th := New()
+	th.SetExclusionPattern(regexp.MustCompile(""))
+	tempDir, err := ioutil.TempDir("", "testtar")
+	defer os.RemoveAll(tempDir)
+	if err != nil {
+		t.Fatalf("Cannot create temp directory for test: %v", err)
+	}
+	modificationDate := time.Date(2011, time.March, 5, 23, 30, 1, 0, time.UTC)
+	testFiles := []fileDesc{
+		{"dir01/dir02/test1.txt", modificationDate, 0700, "Test1 file content", false, ""},
+		{"dir01/test2.git", modificationDate, 0660, "Test2 file content", false, ""},
+		{"dir01/dir03/test3.txt", modificationDate, 0444, "Test3 file content", false, ""},
+		{"dir01/.git/hello.txt", modificationDate, 0600, "Allow .git content", false, ""},
+	}
+	if err := createTestFiles(tempDir, testFiles); err != nil {
+		t.Fatalf("Cannot create test files: %v", err)
+	}
+	testLinks := []linkDesc{
+		{"link/okfilelink", "../dir01/dir02/test1.txt"},
+		{"link/errfilelink", "../dir01/missing.target"},
+		{"link/okdirlink", "../dir01/dir02"},
+		{"link/okdirlink2", "../dir01/.git"},
+	}
+	if err := createTestLinks(tempDir, testLinks); err != nil {
+		t.Fatalf("Cannot create link files: %v", err)
+	}
+
+	tarFile, err := th.CreateTarFile("", tempDir)
+	defer os.Remove(tarFile)
+	if err != nil {
+		t.Fatalf("Unable to create new tar upload file: %v", err)
+	}
+	verifyTarFile(t, tarFile, testFiles, testLinks)
+}
+
 func createTestTar(files []fileDesc, writer io.Writer) error {
 	tw := tar.NewWriter(writer)
 	defer tw.Close()