From 823a522a97ed58e323cb3d6dc0ec4224dc69e53e Mon Sep 17 00:00:00 2001 From: Grzegorz Dlugoszewski Date: Fri, 12 Jun 2020 17:01:35 +0200 Subject: [PATCH] Add a --bundle flag accepting a bundle file of repos to clone Also refactor CloneRepo to accept CloneOpts as agruments - makes it cleaner and easier to test. --- cfg/config.go | 19 ++++++------- cmd/main.go | 26 ++++++++++++++++-- git/repo.go | 29 +++++++++++++------- git/repo_test.go | 9 ++++++- path/bundle.go | 66 +++++++++++++++++++++++++++++++++++++++++++++ path/bundle_test.go | 55 +++++++++++++++++++++++++++++++++++++ path/url.go | 8 ++++-- path/url_test.go | 4 +-- 8 files changed, 190 insertions(+), 26 deletions(-) create mode 100644 path/bundle.go create mode 100644 path/bundle_test.go diff --git a/cfg/config.go b/cfg/config.go index 7569af5..774763b 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -13,20 +13,21 @@ import ( // Gitconfig section name and env var prefix const GitgetPrefix = "gitget" -// Flag keys and default values +// Flag keys and their default values const ( - KeyReposRoot = "reposRoot" - DefReposRoot = "repositories" - KeyDefaultHost = "defaultHost" - DefDefaultHost = "github.com" - KeyPrivateKey = "privateKey" - DefPrivateKey = "id_rsa" - KeyOutput = "out" - DefOutput = OutFlat KeyBranch = "branch" DefBranch = "master" + KeyBundle = "bundle" + KeyDefaultHost = "defaultHost" + DefDefaultHost = "github.com" KeyFetch = "fetch" KeyList = "list" + KeyOutput = "out" + DefOutput = OutFlat + KeyPrivateKey = "privateKey" + DefPrivateKey = "id_rsa" + KeyReposRoot = "reposRoot" + DefReposRoot = "repositories" ) // Allowed values for the --out flag diff --git a/cmd/main.go b/cmd/main.go index 7139225..13d8500 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -35,6 +35,7 @@ func init() { cmd.PersistentFlags().StringP(cfg.KeyPrivateKey, "p", "", "SSH private key path") cmd.PersistentFlags().StringP(cfg.KeyOutput, "o", cfg.DefOutput, "output format.") cmd.PersistentFlags().StringP(cfg.KeyBranch, "b", cfg.DefBranch, "Branch (or tag) to checkout after cloning") + cmd.PersistentFlags().StringP(cfg.KeyBundle, "u", "", "Bundle file path") viper.BindPFlag(cfg.KeyList, cmd.PersistentFlags().Lookup(cfg.KeyList)) viper.BindPFlag(cfg.KeyFetch, cmd.PersistentFlags().Lookup(cfg.KeyFetch)) @@ -42,6 +43,7 @@ func init() { viper.BindPFlag(cfg.KeyPrivateKey, cmd.PersistentFlags().Lookup(cfg.KeyReposRoot)) viper.BindPFlag(cfg.KeyOutput, cmd.PersistentFlags().Lookup(cfg.KeyOutput)) viper.BindPFlag(cfg.KeyBranch, cmd.PersistentFlags().Lookup(cfg.KeyBranch)) + viper.BindPFlag(cfg.KeyBundle, cmd.PersistentFlags().Lookup(cfg.KeyBundle)) } func Run(cmd *cobra.Command, args []string) { @@ -74,12 +76,32 @@ func Run(cmd *cobra.Command, args []string) { os.Exit(0) } + bundle := viper.GetString(cfg.KeyBundle) + if bundle != "" { + opts, err := path.ParseBundleFile(bundle) + exitIfError(err) + + for _, opt := range opts { + path := pathpkg.Join(root, path.URLToPath(opt.URL)) + opt.Path = path + _, _ = git.CloneRepo(opt) + } + os.Exit(0) + } + url, err := path.ParseURL(args[0]) exitIfError(err) branch := viper.GetString(cfg.KeyBranch) - repoPath := pathpkg.Join(root, path.URLToPath(url)) - _, err = git.CloneRepo(url, repoPath, branch, false) + path := pathpkg.Join(root, path.URLToPath(url)) + + cloneOpts := &git.CloneOpts{ + URL: url, + Path: path, + Branch: branch, + } + + _, err = git.CloneRepo(cloneOpts) exitIfError(err) } diff --git a/git/repo.go b/git/repo.go index eb24bb6..b4c253d 100644 --- a/git/repo.go +++ b/git/repo.go @@ -26,17 +26,26 @@ type Repo struct { Status *RepoStatus } -func CloneRepo(url *url.URL, path string, branch string, quiet bool) (*Repo, error) { +// CloneOpts specify details about repository to clone. +type CloneOpts struct { + URL *url.URL + Path string // TODO: should Path be a part of clone opts? + Branch string + Quiet bool + IgnoreExisting bool // TODO: implement! +} + +func CloneRepo(opts *CloneOpts) (*Repo, error) { var progress io.Writer - if !quiet { + if !opts.Quiet { progress = os.Stdout - fmt.Printf("Cloning into '%s'...\n", path) + fmt.Printf("Cloning into '%s'...\n", opts.Path) } // TODO: can this be cleaner? var auth transport.AuthMethod var err error - if url.Scheme == "ssh" { + if opts.URL.Scheme == "ssh" { if auth, err = sshKeyAuth(); err != nil { return nil, err } @@ -44,13 +53,13 @@ func CloneRepo(url *url.URL, path string, branch string, quiet bool) (*Repo, err // If branch name is actually a tag (ie. is prefixed with refs/tags) - check out that tag. // Otherwise, assume it's a branch name and check it out. - refName := plumbing.ReferenceName(branch) + refName := plumbing.ReferenceName(opts.Branch) if !refName.IsTag() { - refName = plumbing.NewBranchReferenceName(branch) + refName = plumbing.NewBranchReferenceName(opts.Branch) } - opts := &git.CloneOptions{ - URL: url.String(), + gitOpts := &git.CloneOptions{ + URL: opts.URL.String(), Auth: auth, RemoteName: git.DefaultRemoteName, ReferenceName: refName, @@ -62,12 +71,12 @@ func CloneRepo(url *url.URL, path string, branch string, quiet bool) (*Repo, err Tags: git.AllTags, } - repo, err := git.PlainClone(path, false, opts) + repo, err := git.PlainClone(opts.Path, false, gitOpts) if err != nil { return nil, errors.Wrap(err, "Failed cloning repo") } - return NewRepo(repo, path), nil + return NewRepo(repo, opts.Path), nil } func OpenRepo(repoPath string) (*Repo, error) { diff --git a/git/repo_test.go b/git/repo_test.go index 60e0d61..f389613 100644 --- a/git/repo_test.go +++ b/git/repo_test.go @@ -247,7 +247,14 @@ func (r *Repo) clone(t *testing.T, branch string) *Repo { repoURL, err := url.Parse("file://" + r.Path) checkFatal(t, err) - repo, err := CloneRepo(repoURL, dir, branch, true) + cloneOpts := &CloneOpts{ + URL: repoURL, + Path: dir, + Branch: branch, + Quiet: true, + } + + repo, err := CloneRepo(cloneOpts) checkFatal(t, err) return repo diff --git a/path/bundle.go b/path/bundle.go new file mode 100644 index 0000000..1f641e1 --- /dev/null +++ b/path/bundle.go @@ -0,0 +1,66 @@ +package path + +import ( + "bufio" + "git-get/git" + "os" + "strings" + + "github.com/pkg/errors" +) + +var ( + ErrInvalidNumberOfElements = errors.New("More than two space-separated 2 elements on the line") +) + +// ParseBundleFile opens a given gitgetfile and parses its content into a slice of CloneOpts. +func ParseBundleFile(path string) ([]*git.CloneOpts, error) { + file, err := os.Open(path) + if err != nil { + return nil, errors.Wrapf(err, "Failed opening gitgetfile %s", path) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + + var opts []*git.CloneOpts + var line int + for scanner.Scan() { + line++ + opt, err := parseLine(scanner.Text()) + if err != nil { + return nil, errors.Wrapf(err, "Failed parsing line %d", line) + } + + opts = append(opts, opt) + } + + return opts, nil +} + +// parseLine splits a gitgetfile line into space-separated segments. +// First part is the URL to clone. Second, optional, is the branch (or tag) to checkout after cloning +func parseLine(line string) (*git.CloneOpts, error) { + parts := strings.Split(line, " ") + + if len(parts) > 2 { + return nil, ErrInvalidNumberOfElements + } + + url, err := ParseURL(parts[0]) + if err != nil { + return nil, err + } + + branch := "" + if len(parts) == 2 { + branch = parts[1] + } + + return &git.CloneOpts{ + URL: url, + Branch: branch, + // When cloning a bundle we ignore errors about already cloned repos + IgnoreExisting: true, + }, nil +} diff --git a/path/bundle_test.go b/path/bundle_test.go new file mode 100644 index 0000000..2c05419 --- /dev/null +++ b/path/bundle_test.go @@ -0,0 +1,55 @@ +package path + +import ( + "testing" +) + +func TestParsingRefs(t *testing.T) { + var tests = []struct { + line string + wantBranch string + wantErr error + }{ + { + line: "https://github.com/grdl/git-get", + wantBranch: "", + wantErr: nil, + }, + { + line: "https://github.com/grdl/git-get master", + wantBranch: "master", + wantErr: nil, + }, + { + line: "https://github.com/grdl/git-get refs/tags/v1.0.0", + wantBranch: "refs/tags/v1.0.0", + wantErr: nil, + }, + { + line: "https://github.com/grdl/git-get master branch", + wantBranch: "", + wantErr: ErrInvalidNumberOfElements, + }, + { + line: "https://github.com", + wantBranch: "", + wantErr: ErrEmptyURLPath, + }, + } + + for i, test := range tests { + got, err := parseLine(test.line) + if err != nil && test.wantErr == nil { + t.Fatalf("Test case %d should not return an error", i) + } + + if err != nil && test.wantErr != nil { + continue + } + + if got.Branch != test.wantBranch { + t.Errorf("Failed test case %d, got: %s; wantBranch: %s", i, got.Branch, test.wantBranch) + } + } + +} diff --git a/path/url.go b/path/url.go index 48cc332..73df039 100644 --- a/path/url.go +++ b/path/url.go @@ -11,6 +11,10 @@ import ( "github.com/spf13/viper" ) +var ( + ErrEmptyURLPath = errors.New("Parsed URL path is empty") +) + // scpSyntax matches the SCP-like addresses used by the ssh protocol (eg, [user@]host.xz:path/to/repo.git/). // See: https://golang.org/src/cmd/go/internal/get/vcs.go var scpSyntax = regexp.MustCompile(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`) @@ -28,12 +32,12 @@ func ParseURL(rawURL string) (url *urlpkg.URL, err error) { } else { url, err = urlpkg.Parse(rawURL) if err != nil { - return nil, errors.Wrap(err, "Failed parsing Path") + return nil, errors.Wrap(err, "Failed parsing URL") } } if url.Host == "" && url.Path == "" { - return nil, errors.New("Parsed Path is empty") + return nil, ErrEmptyURLPath } if url.Scheme == "git+ssh" { diff --git a/path/url_test.go b/path/url_test.go index e60cc35..00b6fee 100644 --- a/path/url_test.go +++ b/path/url_test.go @@ -61,7 +61,7 @@ func TestURLParse(t *testing.T) { got := URLToPath(url) if got != test.want { - t.Errorf("Wrong result of parsing Path: %s, got: %s; want: %s", test.in, got, test.want) + t.Errorf("Wrong result of parsing Path: %s, got: %s; wantBranch: %s", test.in, got, test.want) } } } @@ -79,7 +79,7 @@ func TestInvalidURLParse(t *testing.T) { for _, in := range invalidURLs { got, err := ParseURL(in) if err == nil { - t.Errorf("Wrong result of parsing invalid Path: %s, got: %s, want: error", in, got) + t.Errorf("Wrong result of parsing invalid Path: %s, got: %s, wantBranch: error", in, got) } } }