Add a --bundle flag accepting a bundle file of repos to clone

Also refactor CloneRepo to accept CloneOpts as agruments - makes it cleaner and easier to test.

cfg/config.go

@@ -13,20 +13,21 @@ import (
 // Gitconfig section name and env var prefix
 const GitgetPrefix = "gitget"
 
-// Flag keys and default values
+// Flag keys and their default values
 const (
-	KeyReposRoot   = "reposRoot"
-	DefReposRoot   = "repositories"
-	KeyDefaultHost = "defaultHost"
-	DefDefaultHost = "github.com"
-	KeyPrivateKey  = "privateKey"
-	DefPrivateKey  = "id_rsa"
-	KeyOutput      = "out"
-	DefOutput      = OutFlat
 	KeyBranch      = "branch"
 	DefBranch      = "master"
+	KeyBundle      = "bundle"
+	KeyDefaultHost = "defaultHost"
+	DefDefaultHost = "github.com"
 	KeyFetch       = "fetch"
 	KeyList        = "list"
+	KeyOutput      = "out"
+	DefOutput      = OutFlat
+	KeyPrivateKey  = "privateKey"
+	DefPrivateKey  = "id_rsa"
+	KeyReposRoot   = "reposRoot"
+	DefReposRoot   = "repositories"
 )
 
 // Allowed values for the --out flag

cmd/main.go

@@ -35,6 +35,7 @@ func init() {
 	cmd.PersistentFlags().StringP(cfg.KeyPrivateKey, "p", "", "SSH private key path")
 	cmd.PersistentFlags().StringP(cfg.KeyOutput, "o", cfg.DefOutput, "output format.")
 	cmd.PersistentFlags().StringP(cfg.KeyBranch, "b", cfg.DefBranch, "Branch (or tag) to checkout after cloning")
+	cmd.PersistentFlags().StringP(cfg.KeyBundle, "u", "", "Bundle file path")
 
 	viper.BindPFlag(cfg.KeyList, cmd.PersistentFlags().Lookup(cfg.KeyList))
 	viper.BindPFlag(cfg.KeyFetch, cmd.PersistentFlags().Lookup(cfg.KeyFetch))
@@ -42,6 +43,7 @@ func init() {
 	viper.BindPFlag(cfg.KeyPrivateKey, cmd.PersistentFlags().Lookup(cfg.KeyReposRoot))
 	viper.BindPFlag(cfg.KeyOutput, cmd.PersistentFlags().Lookup(cfg.KeyOutput))
 	viper.BindPFlag(cfg.KeyBranch, cmd.PersistentFlags().Lookup(cfg.KeyBranch))
+	viper.BindPFlag(cfg.KeyBundle, cmd.PersistentFlags().Lookup(cfg.KeyBundle))
 }
 
 func Run(cmd *cobra.Command, args []string) {
@@ -74,12 +76,32 @@ func Run(cmd *cobra.Command, args []string) {
 		os.Exit(0)
 	}
 
+	bundle := viper.GetString(cfg.KeyBundle)
+	if bundle != "" {
+		opts, err := path.ParseBundleFile(bundle)
+		exitIfError(err)
+
+		for _, opt := range opts {
+			path := pathpkg.Join(root, path.URLToPath(opt.URL))
+			opt.Path = path
+			_, _ = git.CloneRepo(opt)
+		}
+		os.Exit(0)
+	}
+
 	url, err := path.ParseURL(args[0])
 	exitIfError(err)
 
 	branch := viper.GetString(cfg.KeyBranch)
-	repoPath := pathpkg.Join(root, path.URLToPath(url))
+	path := pathpkg.Join(root, path.URLToPath(url))
-	_, err = git.CloneRepo(url, repoPath, branch, false)
+
+	cloneOpts := &git.CloneOpts{
+		URL:    url,
+		Path:   path,
+		Branch: branch,
+	}
+
+	_, err = git.CloneRepo(cloneOpts)
 	exitIfError(err)
 }
 

git/repo.go

@@ -26,17 +26,26 @@ type Repo struct {
 	Status *RepoStatus
 }
 
-func CloneRepo(url *url.URL, path string, branch string, quiet bool) (*Repo, error) {
+// CloneOpts specify details about repository to clone.
+type CloneOpts struct {
+	URL            *url.URL
+	Path           string // TODO: should Path be a part of clone opts?
+	Branch         string
+	Quiet          bool
+	IgnoreExisting bool // TODO: implement!
+}
+
+func CloneRepo(opts *CloneOpts) (*Repo, error) {
 	var progress io.Writer
-	if !quiet {
+	if !opts.Quiet {
 		progress = os.Stdout
-		fmt.Printf("Cloning into '%s'...\n", path)
+		fmt.Printf("Cloning into '%s'...\n", opts.Path)
 	}
 
 	// TODO: can this be cleaner?
 	var auth transport.AuthMethod
 	var err error
-	if url.Scheme == "ssh" {
+	if opts.URL.Scheme == "ssh" {
 		if auth, err = sshKeyAuth(); err != nil {
 			return nil, err
 		}
@@ -44,13 +53,13 @@ func CloneRepo(url *url.URL, path string, branch string, quiet bool) (*Repo, err
 
 	// If branch name is actually a tag (ie. is prefixed with refs/tags) - check out that tag.
 	// Otherwise, assume it's a branch name and check it out.
-	refName := plumbing.ReferenceName(branch)
+	refName := plumbing.ReferenceName(opts.Branch)
 	if !refName.IsTag() {
-		refName = plumbing.NewBranchReferenceName(branch)
+		refName = plumbing.NewBranchReferenceName(opts.Branch)
 	}
 
-	opts := &git.CloneOptions{
+	gitOpts := &git.CloneOptions{
-		URL:               url.String(),
+		URL:               opts.URL.String(),
 		Auth:              auth,
 		RemoteName:        git.DefaultRemoteName,
 		ReferenceName:     refName,
@@ -62,12 +71,12 @@ func CloneRepo(url *url.URL, path string, branch string, quiet bool) (*Repo, err
 		Tags:              git.AllTags,
 	}
 
-	repo, err := git.PlainClone(path, false, opts)
+	repo, err := git.PlainClone(opts.Path, false, gitOpts)
 	if err != nil {
 		return nil, errors.Wrap(err, "Failed cloning repo")
 	}
 
-	return NewRepo(repo, path), nil
+	return NewRepo(repo, opts.Path), nil
 }
 
 func OpenRepo(repoPath string) (*Repo, error) {

git/repo_test.go

@@ -247,7 +247,14 @@ func (r *Repo) clone(t *testing.T, branch string) *Repo {
 	repoURL, err := url.Parse("file://" + r.Path)
 	checkFatal(t, err)
 
-	repo, err := CloneRepo(repoURL, dir, branch, true)
+	cloneOpts := &CloneOpts{
+		URL:    repoURL,
+		Path:   dir,
+		Branch: branch,
+		Quiet:  true,
+	}
+
+	repo, err := CloneRepo(cloneOpts)
 	checkFatal(t, err)
 
 	return repo

path/bundle.go

@@ -0,0 +1,66 @@
+package path
+
+import (
+	"bufio"
+	"git-get/git"
+	"os"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	ErrInvalidNumberOfElements = errors.New("More than two space-separated 2 elements on the line")
+)
+
+// ParseBundleFile opens a given gitgetfile and parses its content into a slice of CloneOpts.
+func ParseBundleFile(path string) ([]*git.CloneOpts, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Failed opening gitgetfile %s", path)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+
+	var opts []*git.CloneOpts
+	var line int
+	for scanner.Scan() {
+		line++
+		opt, err := parseLine(scanner.Text())
+		if err != nil {
+			return nil, errors.Wrapf(err, "Failed parsing line %d", line)
+		}
+
+		opts = append(opts, opt)
+	}
+
+	return opts, nil
+}
+
+// parseLine splits a gitgetfile line into space-separated segments.
+// First part is the URL to clone. Second, optional, is the branch (or tag) to checkout after cloning
+func parseLine(line string) (*git.CloneOpts, error) {
+	parts := strings.Split(line, " ")
+
+	if len(parts) > 2 {
+		return nil, ErrInvalidNumberOfElements
+	}
+
+	url, err := ParseURL(parts[0])
+	if err != nil {
+		return nil, err
+	}
+
+	branch := ""
+	if len(parts) == 2 {
+		branch = parts[1]
+	}
+
+	return &git.CloneOpts{
+		URL:    url,
+		Branch: branch,
+		// When cloning a bundle we ignore errors about already cloned repos
+		IgnoreExisting: true,
+	}, nil
+}

path/bundle_test.go

@@ -0,0 +1,55 @@
+package path
+
+import (
+	"testing"
+)
+
+func TestParsingRefs(t *testing.T) {
+	var tests = []struct {
+		line       string
+		wantBranch string
+		wantErr    error
+	}{
+		{
+			line:       "https://github.com/grdl/git-get",
+			wantBranch: "",
+			wantErr:    nil,
+		},
+		{
+			line:       "https://github.com/grdl/git-get master",
+			wantBranch: "master",
+			wantErr:    nil,
+		},
+		{
+			line:       "https://github.com/grdl/git-get refs/tags/v1.0.0",
+			wantBranch: "refs/tags/v1.0.0",
+			wantErr:    nil,
+		},
+		{
+			line:       "https://github.com/grdl/git-get master branch",
+			wantBranch: "",
+			wantErr:    ErrInvalidNumberOfElements,
+		},
+		{
+			line:       "https://github.com",
+			wantBranch: "",
+			wantErr:    ErrEmptyURLPath,
+		},
+	}
+
+	for i, test := range tests {
+		got, err := parseLine(test.line)
+		if err != nil && test.wantErr == nil {
+			t.Fatalf("Test case %d should not return an error", i)
+		}
+
+		if err != nil && test.wantErr != nil {
+			continue
+		}
+
+		if got.Branch != test.wantBranch {
+			t.Errorf("Failed test case %d, got: %s; wantBranch: %s", i, got.Branch, test.wantBranch)
+		}
+	}
+
+}

path/url.go

@@ -11,6 +11,10 @@ import (
 	"github.com/spf13/viper"
 )
 
+var (
+	ErrEmptyURLPath = errors.New("Parsed URL path is empty")
+)
+
 // scpSyntax matches the SCP-like addresses used by the ssh protocol (eg, [user@]host.xz:path/to/repo.git/).
 // See: https://golang.org/src/cmd/go/internal/get/vcs.go
 var scpSyntax = regexp.MustCompile(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`)
@@ -28,12 +32,12 @@ func ParseURL(rawURL string) (url *urlpkg.URL, err error) {
 	} else {
 		url, err = urlpkg.Parse(rawURL)
 		if err != nil {
-			return nil, errors.Wrap(err, "Failed parsing Path")
+			return nil, errors.Wrap(err, "Failed parsing URL")
 		}
 	}
 
 	if url.Host == "" && url.Path == "" {
-		return nil, errors.New("Parsed Path is empty")
+		return nil, ErrEmptyURLPath
 	}
 
 	if url.Scheme == "git+ssh" {

path/url_test.go

@@ -61,7 +61,7 @@ func TestURLParse(t *testing.T) {
 		got := URLToPath(url)
 
 		if got != test.want {
-			t.Errorf("Wrong result of parsing Path: %s, got: %s; want: %s", test.in, got, test.want)
+			t.Errorf("Wrong result of parsing Path: %s, got: %s; wantBranch: %s", test.in, got, test.want)
 		}
 	}
 }
@@ -79,7 +79,7 @@ func TestInvalidURLParse(t *testing.T) {
 	for _, in := range invalidURLs {
 		got, err := ParseURL(in)
 		if err == nil {
-			t.Errorf("Wrong result of parsing invalid Path: %s, got: %s, want: error", in, got)
+			t.Errorf("Wrong result of parsing invalid Path: %s, got: %s, wantBranch: error", in, got)
 		}
 	}
 }