Refactor cloning dump file

cmd/get/main.go

@@ -38,10 +38,11 @@ func run(cmd *cobra.Command, args []string) error {
 	}
 
 	config := &pkg.GetCfg{
-		Branch: viper.GetString(cfg.KeyBranch),
-		Dump:   viper.GetString(cfg.KeyDump),
-		Root:   viper.GetString(cfg.KeyReposRoot),
-		URL:    url,
+		Branch:  viper.GetString(cfg.KeyBranch),
+		DefHost: viper.GetString(cfg.KeyDefaultHost),
+		Dump:    viper.GetString(cfg.KeyDump),
+		Root:    viper.GetString(cfg.KeyReposRoot),
+		URL:     url,
 	}
 	return pkg.Get(config)
 }

pkg/dump.go

@@ -2,17 +2,24 @@ package pkg
 
 import (
 	"bufio"
-	"git-get/pkg/repo"
 	"os"
 	"strings"
 
 	"github.com/pkg/errors"
 )
 
-var errInvalidNumberOfElements = errors.New("More than two space-separated 2 elements on the line")
+var (
+	errInvalidNumberOfElements = errors.New("More than two space-separated 2 elements on the line")
+	errEmptyLine               = errors.New("Empty line")
+)
+
+type parsedLine struct {
+	rawurl string
+	branch string
+}
 
 // ParseDumpFile opens a given gitgetfile and parses its content into a slice of CloneOpts.
-func ParseDumpFile(path string) ([]*repo.CloneOpts, error) {
+func parseDumpFile(path string) ([]parsedLine, error) {
 	file, err := os.Open(path)
 	if err != nil {
 		return nil, errors.Wrapf(err, "Failed opening dump file %s", path)
@@ -21,44 +28,39 @@ func ParseDumpFile(path string) ([]*repo.CloneOpts, error) {
 
 	scanner := bufio.NewScanner(file)
 
-	var opts []*repo.CloneOpts
+	var parsedLines []parsedLine
 	var line int
 	for scanner.Scan() {
 		line++
-		opt, err := parseLine(scanner.Text())
-		if err != nil {
+		parsed, err := parseLine(scanner.Text())
+		if err != nil && !errors.Is(errEmptyLine, err) {
 			return nil, errors.Wrapf(err, "Failed parsing line %d", line)
 		}
 
-		opts = append(opts, opt)
+		parsedLines = append(parsedLines, parsed)
 	}
 
-	return opts, nil
+	return parsedLines, nil
 }
 
 // parseLine splits a dump file line into space-separated segments.
 // First part is the URL to clone. Second, optional, is the branch (or tag) to checkout after cloning
-func parseLine(line string) (*repo.CloneOpts, error) {
-	parts := strings.Split(line, " ")
+func parseLine(line string) (parsedLine, error) {
+	var parsed parsedLine
 
+	if len(strings.TrimSpace(line)) == 0 {
+		return parsed, errEmptyLine
+	}
+
+	parts := strings.Split(strings.TrimSpace(line), " ")
 	if len(parts) > 2 {
-		return nil, errInvalidNumberOfElements
+		return parsed, errInvalidNumberOfElements
 	}
 
-	url, err := ParseURL(parts[0])
-	if err != nil {
-		return nil, err
-	}
-
-	branch := ""
+	parsed.rawurl = parts[0]
 	if len(parts) == 2 {
-		branch = parts[1]
+		parsed.branch = parts[1]
 	}
 
-	return &repo.CloneOpts{
-		URL:    url,
-		Branch: branch,
-		// When cloning a bundle we ignore errors about already cloned repos.
-		IgnoreExisting: true,
-	}, nil
+	return parsed, nil
 }

pkg/dump_test.go

@@ -47,8 +47,8 @@ func TestParsingRefs(t *testing.T) {
 			continue
 		}
 
-		if got.Branch != test.wantBranch {
-			t.Errorf("Failed test case %d, got: %s; wantBranch: %s", i, got.Branch, test.wantBranch)
+		if got.branch != test.wantBranch {
+			t.Errorf("Failed test case %d, got: %s; wantBranch: %s", i, got.branch, test.wantBranch)
 		}
 	}
 

pkg/get.go

@@ -7,27 +7,27 @@ import (
 
 // GetCfg provides configuration for the Get command.
 type GetCfg struct {
-	Branch string
-	Dump   string
-	Root   string
-	URL    string
+	Branch  string
+	DefHost string
+	Dump    string
+	Root    string
+	URL     string
 }
 
 // Get executes the "git get" command.
 func Get(c *GetCfg) error {
-	if c.Dump != "" {
-		return cloneDumpFile(c)
-	}
-
 	if c.URL != "" {
 		return cloneSingleRepo(c)
 	}
 
+	if c.Dump != "" {
+		return cloneDumpFile(c)
+	}
 	return nil
 }
 
 func cloneSingleRepo(c *GetCfg) error {
-	url, err := ParseURL(c.URL)
+	url, err := ParseURL(c.URL, c.DefHost)
 	if err != nil {
 		return err
 	}
@@ -44,15 +44,28 @@ func cloneSingleRepo(c *GetCfg) error {
 }
 
 func cloneDumpFile(c *GetCfg) error {
-	opts, err := ParseDumpFile(c.Dump)
+	parsedLines, err := parseDumpFile(c.Dump)
 	if err != nil {
 		return err
 	}
 
-	for _, opt := range opts {
-		path := path.Join(c.Root, URLToPath(opt.URL))
-		opt.Path = path
-		_, _ = repo.Clone(opt)
+	for _, line := range parsedLines {
+		url, err := ParseURL(line.rawurl, c.DefHost)
+		if err != nil {
+			return err
+		}
+
+		cloneOpts := &repo.CloneOpts{
+			URL:            url,
+			Path:           path.Join(c.Root, URLToPath(url)),
+			Branch:         line.branch,
+			IgnoreExisting: true,
+		}
+
+		_, err = repo.Clone(cloneOpts)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }

pkg/url.go

@@ -1,14 +1,12 @@
 package pkg
 
 import (
-	"git-get/pkg/cfg"
 	urlpkg "net/url"
 	"path"
 	"regexp"
 	"strings"
 
 	"github.com/pkg/errors"
-	"github.com/spf13/viper"
 )
 
 var errEmptyURLPath = errors.New("Parsed URL path is empty")
@@ -17,7 +15,9 @@ var errEmptyURLPath = errors.New("Parsed URL path is empty")
 // See: https://golang.org/src/cmd/go/internal/get/vcs.go
 var scpSyntax = regexp.MustCompile(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`)
 
-func ParseURL(rawURL string) (url *urlpkg.URL, err error) {
+// ParseURL parses given rawURL string into a URL.
+// The defaultHost argument defines the host to use (eg, github.com) in case parsed URL has an empty host.
+func ParseURL(rawURL string, defaultHost string) (url *urlpkg.URL, err error) {
 	// If rawURL matches the SCP-like syntax, convert it into a standard ssh Path.
 	// eg, git@github.com:user/repo => ssh://git@github.com/user/repo
 	if m := scpSyntax.FindStringSubmatch(rawURL); m != nil {
@@ -49,7 +49,7 @@ func ParseURL(rawURL string) (url *urlpkg.URL, err error) {
 
 	// Default to configured defaultHost when host is empty
 	if url.Host == "" {
-		url.Host = viper.GetString(cfg.KeyDefaultHost)
+		url.Host = defaultHost
 	}
 
 	// Default to https when scheme is empty
@@ -60,6 +60,8 @@ func ParseURL(rawURL string) (url *urlpkg.URL, err error) {
 	return url, nil
 }
 
+// URLToPath cleans up the URL and converts it into a path string.
+// Eg, ssh://git@github.com:22/~user/repo.git => github.com/user/repo
 func URLToPath(url *urlpkg.URL) (repoPath string) {
 	// Remove port numbers from host
 	repoHost := strings.Split(url.Host, ":")[0]

pkg/url_test.go

@@ -49,11 +49,8 @@ func TestURLParse(t *testing.T) {
 		{"file://local/grdl/git-get", "local/grdl/git-get"},
 	}
 
-	// We need to init config first so the default values are correctly loaded
-	cfg.Init()
-
 	for _, test := range tests {
-		url, err := ParseURL(test.in)
+		url, err := ParseURL(test.in, cfg.DefDefaultHost)
 		if err != nil {
 			t.Errorf("Error parsing Path: %+v", err)
 		}
@@ -77,7 +74,7 @@ func TestInvalidURLParse(t *testing.T) {
 	}
 
 	for _, in := range invalidURLs {
-		got, err := ParseURL(in)
+		got, err := ParseURL(in, cfg.DefDefaultHost)
 		if err == nil {
 			t.Errorf("Wrong result of parsing invalid Path: %s, got: %s, wantBranch: error", in, got)
 		}