Add data path handling for regolith

regolith/git_getter.go

@@ -6,13 +6,14 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/google/go-github/v39/github"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
+
+	"github.com/google/go-github/v39/github"
 )
 
 type TreeResponse struct {
@@ -49,7 +50,7 @@ const treeApiUrl = "https://api.github.com/repos/%s/%s/git/trees/%s"
 func DownloadGitHubUrl(url string, localPath string) (bool, error) {
 	split := strings.Split(path.Clean(url), "/")
 	if len(split) < 4 || !strings.HasSuffix(split[0], "github.com") {
-		return false, nil
+		return false, wrapError("Remote repositories must be of the form: github.com/owner/repo/folder", nil)
 	}
 	client := github.NewClient(nil)
 	repo, _, err := client.Repositories.Get(context.Background(), split[1], split[2])

regolith/profiles.go

@@ -24,7 +24,7 @@ func SetupTmpFiles(config Config, profile Profile) error {
 		return err
 	}
 
-	// Copy the contents of the `regolith` folder to `.regolith/tmp`
+	// Copy the contents of the 'regolith' folder to '.regolith/tmp'
 	Logger.Debug("Copying project files to .regolith/tmp")
 
 	err = copy.Copy(config.Packs.BehaviorFolder, ".regolith/tmp/BP", copy.Options{PreserveTimes: false, Sync: false})
@@ -36,12 +36,14 @@ func SetupTmpFiles(config Config, profile Profile) error {
 	if err != nil {
 		return err
 	}
+
+	// Copy the contents of 'data' folder to '.regolith/tmp'
 	if profile.DataPath != "" { // datapath copied only if specified
 		err = copy.Copy(profile.DataPath, ".regolith/tmp/data", copy.Options{PreserveTimes: false, Sync: false})
 		if err != nil {
 			return err
 		}
-	} else { // create empty data path
+	} else { // create empty data path otherwise.
 		err = os.MkdirAll(".regolith/data", 0666)
 		if err != nil {
 			return err

regolith/project.go

@@ -126,8 +126,9 @@ func InitializeRegolithProject(isForced bool) error {
 			RegolithProject: RegolithProject{
 				Profiles: map[string]Profile{
 					"dev": {
-						Unsafe:  false,
-						Filters: []Filter{},
+						Unsafe:   false,
+						DataPath: "./packs/data",
+						Filters:  []Filter{},
 						ExportTarget: ExportTarget{
 							Target: "development",
 						},
@@ -163,6 +164,11 @@ func InitializeRegolithProject(isForced bool) error {
 			Logger.Error("Could not create ./packs/BP folder", err)
 		}
 
+		err = os.Mkdir("./packs/data", 0666)
+		if err != nil {
+			Logger.Error("Could not create ./packs/data folder", err)
+		}
+
 		err = os.Mkdir(".regolith", 0666)
 		if err != nil {
 			Logger.Error("Could not create .regolith folder", err)

regolith/remote_filters.go

@@ -5,8 +5,11 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
+	"strings"
 
 	getter "github.com/hashicorp/go-getter"
+	"github.com/otiai10/copy"
 )
 
 const StandardLibraryUrl = "github.com/Bedrock-OSS/regolith-filters"
@@ -62,7 +65,7 @@ func InstallDependencies() error {
 
 // InstallDependency recursively downloads the filters of a profile and the
 // filters specified in other filters.
-func InstallDependency(profile Profile) error { // TODO - rename that and split into two functions?
+func InstallDependency(profile Profile) error {
 	for _, filter := range profile.Filters {
 		// Get the url of the dependency
 		var url string
@@ -76,30 +79,46 @@ func InstallDependency(profile Profile) error { // TODO - rename that and split
 		Logger.Infof("Installing dependency %s...", url)
 
 		// Download the filter into the cache folder
-		path := UrlToPath(url)
-		ok, err := DownloadGitHubUrl(url, path)
+		downloadPath := UrlToPath(url)
+		ok, err := DownloadGitHubUrl(url, downloadPath)
 		if err != nil {
 			Logger.Debug(err)
 		}
 		if !ok {
 			Logger.Debug("Failed to download filter " + filter.Filter + " without git")
-			err := getter.Get(path, url)
+			err := getter.Get(downloadPath, url)
 			if err != nil {
 				return err
 			}
 		}
 
+		// Move filters 'data' folder contents into 'data'
+		filterName := strings.Split(path.Clean(url), "/")[3]
+
+		filterDataPath := path.Join(profile.DataPath, filterName)
+		err = os.MkdirAll(filterDataPath, 0666)
+		if err != nil {
+			Logger.Error("Could not create filter data folder", err)
+		}
+
+		if profile.DataPath != "" {
+			err = copy.Copy(path.Join(downloadPath, "data"), filterDataPath, copy.Options{PreserveTimes: false, Sync: false})
+			if err != nil {
+				Logger.Error("Could not initialize filter data", err)
+			}
+		}
+
 		// Check required files
-		file, err := ioutil.ReadFile(path + "/filter.json")
+		file, err := ioutil.ReadFile(downloadPath + "/filter.json")
 		if err != nil {
-			return wrapError(fmt.Sprintf("Couldn't find %s/filter.json!", path), err)
+			return wrapError(fmt.Sprintf("Couldn't find %s/filter.json!", downloadPath), err)
 		}
 
 		// Load subprofile (remote filter)
 		var remoteProfile Profile
 		err = json.Unmarshal(file, &remoteProfile)
 		if err != nil {
-			return wrapError(fmt.Sprintf("Couldn't load %s/filter.json: ", path), err)
+			return wrapError(fmt.Sprintf("Couldn't load %s/filter.json: ", downloadPath), err)
 		}
 		// Propagate venvSlot property
 		for f := range remoteProfile.Filters {
@@ -116,7 +135,7 @@ func InstallDependency(profile Profile) error { // TODO - rename that and split
 		for _, filter := range remoteProfile.Filters {
 			if filter.RunWith != "" {
 				if f, ok := FilterTypes[filter.RunWith]; ok {
-					err := f.install(filter, path)
+					err := f.install(filter, downloadPath)
 					if err != nil {
 						return wrapError(fmt.Sprintf("Couldn't install filter %s", filter.Name), err)
 					}