diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index be7b496..9cb1daf 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -7,7 +7,7 @@ entry point import os import sys - +from concurrent import futures import shutil # local imports @@ -24,6 +24,7 @@ from dotdrop.exceptions import YamlException, UndefinedException LOG = Logger() TRANS_SUFFIX = 'trans' +INST_WORKERS = 10 ########################################################### # entry point @@ -70,6 +71,89 @@ def action_executor(o, actions, defactions, templater, post=False): return execute +def _dotfile_install(o, dotfile, tmpdir=None): + """install a dotfile""" + # installer + inst = _get_install_installer(o, tmpdir=tmpdir) + + # templater + t = _get_templater(o) + + # add dotfile variables + newvars = dotfile.get_dotfile_variables() + t.add_tmp_vars(newvars=newvars) + + preactions = [] + if not o.install_temporary: + preactions.extend(dotfile.get_pre_actions()) + defactions = o.install_default_actions_pre + pre_actions_exec = action_executor(o, preactions, defactions, + t, post=False) + + if o.debug: + LOG.dbg('installing dotfile: \"{}\"'.format(dotfile.key)) + LOG.dbg(dotfile.prt()) + + if hasattr(dotfile, 'link') and dotfile.link == LinkTypes.LINK: + # link + r, err = inst.link(t, dotfile.src, dotfile.dst, + actionexec=pre_actions_exec, + template=dotfile.template) + elif hasattr(dotfile, 'link') and \ + dotfile.link == LinkTypes.LINK_CHILDREN: + # link_children + r, err = inst.link_children(t, dotfile.src, dotfile.dst, + actionexec=pre_actions_exec, + template=dotfile.template) + else: + # nolink + src = dotfile.src + tmp = None + if dotfile.trans_r: + tmp = apply_trans(o.dotpath, dotfile, t, debug=o.debug) + if not tmp: + return False, dotfile.key, None + src = tmp + ignores = list(set(o.install_ignore + dotfile.instignore)) + ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) + r, err = inst.install(t, src, dotfile.dst, + actionexec=pre_actions_exec, + noempty=dotfile.noempty, + ignore=ignores, + template=dotfile.template) + if tmp: + tmp = os.path.join(o.dotpath, tmp) + if os.path.exists(tmp): + removepath(tmp, LOG) + + # check result of installation + if r: + # dotfile was installed + if not o.install_temporary: + defactions = o.install_default_actions_post + postactions = dotfile.get_post_actions() + post_actions_exec = action_executor(o, postactions, defactions, + t, post=True) + post_actions_exec() + else: + # dotfile was NOT installed + if o.install_force_action: + # pre-actions + if o.debug: + LOG.dbg('force pre action execution ...') + pre_actions_exec() + # post-actions + if o.debug: + LOG.dbg('force post action execution ...') + defactions = o.install_default_actions_post + postactions = dotfile.get_post_actions() + post_actions_exec = action_executor(o, postactions, defactions, + t, post=True) + post_actions_exec() + + return r, dotfile.key, err + + def cmd_install(o): """install dotfiles for this profile""" dotfiles = o.dotfiles @@ -86,100 +170,44 @@ def cmd_install(o): LOG.warn(msg.format(o.profile)) return False - t = Templategen(base=o.dotpath, variables=o.variables, - func_file=o.func_file, filter_file=o.filter_file, - debug=o.debug) + # the installer tmpdir = None if o.install_temporary: tmpdir = get_tmpdir() - inst = Installer(create=o.create, backup=o.backup, - dry=o.dry, safe=o.safe, - base=o.dotpath, workdir=o.workdir, - diff=o.install_diff, debug=o.debug, - totemp=tmpdir, - showdiff=o.install_showdiff, - backup_suffix=o.install_backup_suffix, - diff_cmd=o.diff_command) + installed = 0 - tvars = t.add_tmp_vars() # execute profile pre-action if o.debug: LOG.dbg('run {} profile pre actions'.format(len(pro_pre_actions))) + t = _get_templater(o) ret, err = action_executor(o, pro_pre_actions, [], t, post=False)() if not ret: return False # install each dotfile - for dotfile in dotfiles: - # add dotfile variables - t.restore_vars(tvars) - newvars = dotfile.get_dotfile_variables() - t.add_tmp_vars(newvars=newvars) + if o.install_parallel: + # in parallel + ex = futures.ThreadPoolExecutor(max_workers=INST_WORKERS) - preactions = [] - if not o.install_temporary: - preactions.extend(dotfile.get_pre_actions()) - defactions = o.install_default_actions_pre - pre_actions_exec = action_executor(o, preactions, defactions, - t, post=False) - - if o.debug: - LOG.dbg('installing dotfile: \"{}\"'.format(dotfile.key)) - LOG.dbg(dotfile.prt()) - if hasattr(dotfile, 'link') and dotfile.link == LinkTypes.LINK: - r, err = inst.link(t, dotfile.src, dotfile.dst, - actionexec=pre_actions_exec, - template=dotfile.template) - elif hasattr(dotfile, 'link') and \ - dotfile.link == LinkTypes.LINK_CHILDREN: - r, err = inst.link_children(t, dotfile.src, dotfile.dst, - actionexec=pre_actions_exec, - template=dotfile.template) - else: - src = dotfile.src - tmp = None - if dotfile.trans_r: - tmp = apply_trans(o.dotpath, dotfile, t, debug=o.debug) - if not tmp: - continue - src = tmp - ignores = list(set(o.install_ignore + dotfile.instignore)) - ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) - r, err = inst.install(t, src, dotfile.dst, - actionexec=pre_actions_exec, - noempty=dotfile.noempty, - ignore=ignores, - template=dotfile.template) - if tmp: - tmp = os.path.join(o.dotpath, tmp) - if os.path.exists(tmp): - removepath(tmp, LOG) - if r: - # dotfile was installed - if not o.install_temporary: - defactions = o.install_default_actions_post - postactions = dotfile.get_post_actions() - post_actions_exec = action_executor(o, postactions, defactions, - t, post=True) - post_actions_exec() - installed += 1 - elif not r: - # dotfile was NOT installed - if o.install_force_action: - # pre-actions - if o.debug: - LOG.dbg('force pre action execution ...') - pre_actions_exec() - # post-actions - if o.debug: - LOG.dbg('force post action execution ...') - defactions = o.install_default_actions_post - postactions = dotfile.get_post_actions() - post_actions_exec = action_executor(o, postactions, defactions, - t, post=True) - post_actions_exec() - if err: + wait_for = [ + ex.submit(_dotfile_install, o, dotfile, tmpdir=tmpdir) + for dotfile in dotfiles + ] + for f in futures.as_completed(wait_for): + r, key, err = f.result() + if r: + installed += 1 + elif err: + LOG.err('installing \"{}\" failed: {}'.format(key, + err)) + else: + # sequentially + for dotfile in dotfiles: + r, err = _dotfile_install(o, dotfile, tmpdir=tmpdir) + if r: + installed += 1 + elif err: LOG.err('installing \"{}\" failed: {}'.format(dotfile.key, err)) @@ -217,9 +245,7 @@ def cmd_compare(o, tmp): if len(selected) < 1: return False - t = Templategen(base=o.dotpath, variables=o.variables, - func_file=o.func_file, filter_file=o.filter_file, - debug=o.debug) + t = _get_templater(o) tvars = t.add_tmp_vars() inst = Installer(create=o.create, backup=o.backup, dry=o.dry, base=o.dotpath, @@ -627,6 +653,27 @@ def cmd_remove(o): ########################################################### +def _get_install_installer(o, tmpdir=None): + """get an installer instance for cmd_install""" + inst = Installer(create=o.create, backup=o.backup, + dry=o.dry, safe=o.safe, + base=o.dotpath, workdir=o.workdir, + diff=o.install_diff, debug=o.debug, + totemp=tmpdir, + showdiff=o.install_showdiff, + backup_suffix=o.install_backup_suffix, + diff_cmd=o.diff_command) + return inst + + +def _get_templater(o): + """get an templater instance""" + t = Templategen(base=o.dotpath, variables=o.variables, + func_file=o.func_file, filter_file=o.filter_file, + debug=o.debug) + return t + + def _detail(dotpath, dotfile): """display details on all files under a dotfile entry""" LOG.log('{} (dst: \"{}\", link: {})'.format(dotfile.key, dotfile.dst, diff --git a/dotdrop/options.py b/dotdrop/options.py index 47dc835..d270200 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -67,21 +67,21 @@ Usage: Options: -a --force-actions Execute all actions even if no dotfile is installed. + -b --no-banner Do not display the banner. -c --cfg= Path to the config. -C --file= Path of dotfile to compare. - -i --ignore= Pattern to ignore. + -d --dry Dry run. -l --link= Link option (nolink|link|link_children). -L --file-only Do not show diff but only the files that differ. -p --profile= Specify the profile to use [default: {}]. - -s --as= Import as a different path from actual path. - -b --no-banner Do not display the banner. - -d --dry Dry run. -D --showdiff Show a diff before overwriting. -f --force Do not ask user confirmation for anything. -G --grepable Grepable output. + -i --ignore= Pattern to ignore. -k --key Treat as a dotfile key. -n --nodiff Do not diff when installing. -P --show-patch Provide a one-liner to manually patch template. + -s --as= Import as a different path from actual path. -t --temp Install to a temporary directory for review. -T --template Only template dotfiles. -V --verbose Be verbose. @@ -234,6 +234,8 @@ class Options(AttrMonitor): self.install_default_actions_post = [a for a in self.default_actions if a.kind == Action.post] self.install_ignore = self.instignore + # TODO add as option + self.install_parallel = True # "compare" specifics self.compare_focus = self.args['--file'] self.compare_ignore = self.args['--ignore']