Merge pull request #288 from deadc0de6/concurrent

Concurrent

dotdrop/dotdrop.py

@@ -7,7 +7,7 @@ entry point
 
 import os
 import sys
-
+from concurrent import futures
 import shutil
 
 # local imports
@@ -70,6 +70,92 @@ def action_executor(o, actions, defactions, templater, post=False):
     return execute
 
 
+def _dotfile_install(o, dotfile, tmpdir=None):
+    """
+    install a dotfile
+    returns <success, dotfile key, err>
+    """
+    # installer
+    inst = _get_install_installer(o, tmpdir=tmpdir)
+
+    # templater
+    t = _get_templater(o)
+
+    # add dotfile variables
+    newvars = dotfile.get_dotfile_variables()
+    t.add_tmp_vars(newvars=newvars)
+
+    preactions = []
+    if not o.install_temporary:
+        preactions.extend(dotfile.get_pre_actions())
+    defactions = o.install_default_actions_pre
+    pre_actions_exec = action_executor(o, preactions, defactions,
+                                       t, post=False)
+
+    if o.debug:
+        LOG.dbg('installing dotfile: \"{}\"'.format(dotfile.key))
+        LOG.dbg(dotfile.prt())
+
+    if hasattr(dotfile, 'link') and dotfile.link == LinkTypes.LINK:
+        # link
+        r, err = inst.link(t, dotfile.src, dotfile.dst,
+                           actionexec=pre_actions_exec,
+                           template=dotfile.template)
+    elif hasattr(dotfile, 'link') and \
+            dotfile.link == LinkTypes.LINK_CHILDREN:
+        # link_children
+        r, err = inst.link_children(t, dotfile.src, dotfile.dst,
+                                    actionexec=pre_actions_exec,
+                                    template=dotfile.template)
+    else:
+        # nolink
+        src = dotfile.src
+        tmp = None
+        if dotfile.trans_r:
+            tmp = apply_trans(o.dotpath, dotfile, t, debug=o.debug)
+            if not tmp:
+                return False, dotfile.key, None
+            src = tmp
+        ignores = list(set(o.install_ignore + dotfile.instignore))
+        ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug)
+        r, err = inst.install(t, src, dotfile.dst,
+                              actionexec=pre_actions_exec,
+                              noempty=dotfile.noempty,
+                              ignore=ignores,
+                              template=dotfile.template)
+        if tmp:
+            tmp = os.path.join(o.dotpath, tmp)
+            if os.path.exists(tmp):
+                removepath(tmp, LOG)
+
+    # check result of installation
+    if r:
+        # dotfile was installed
+        if not o.install_temporary:
+            defactions = o.install_default_actions_post
+            postactions = dotfile.get_post_actions()
+            post_actions_exec = action_executor(o, postactions, defactions,
+                                                t, post=True)
+            post_actions_exec()
+    else:
+        # dotfile was NOT installed
+        if o.install_force_action:
+            # pre-actions
+            if o.debug:
+                LOG.dbg('force pre action execution ...')
+            pre_actions_exec()
+            # post-actions
+            if o.debug:
+                LOG.dbg('force post action execution ...')
+            defactions = o.install_default_actions_post
+            postactions = dotfile.get_post_actions()
+            post_actions_exec = action_executor(o, postactions, defactions,
+                                                t, post=True)
+            post_actions_exec()
+
+    return r, dotfile.key, err
+
+
 def cmd_install(o):
     """install dotfiles for this profile"""
     dotfiles = o.dotfiles
@@ -86,101 +172,45 @@ def cmd_install(o):
         LOG.warn(msg.format(o.profile))
         return False
 
-    t = Templategen(base=o.dotpath, variables=o.variables,
+    # the installer
-                    func_file=o.func_file, filter_file=o.filter_file,
-                    debug=o.debug)
     tmpdir = None
     if o.install_temporary:
         tmpdir = get_tmpdir()
-    inst = Installer(create=o.create, backup=o.backup,
+
-                     dry=o.dry, safe=o.safe,
-                     base=o.dotpath, workdir=o.workdir,
-                     diff=o.install_diff, debug=o.debug,
-                     totemp=tmpdir,
-                     showdiff=o.install_showdiff,
-                     backup_suffix=o.install_backup_suffix,
-                     diff_cmd=o.diff_command)
     installed = 0
-    tvars = t.add_tmp_vars()
 
     # execute profile pre-action
     if o.debug:
         LOG.dbg('run {} profile pre actions'.format(len(pro_pre_actions)))
+    t = _get_templater(o)
     ret, err = action_executor(o, pro_pre_actions, [], t, post=False)()
     if not ret:
         return False
 
     # install each dotfile
-    for dotfile in dotfiles:
+    if o.install_parallel > 1:
-        # add dotfile variables
+        # in parallel
-        t.restore_vars(tvars)
+        ex = futures.ThreadPoolExecutor(max_workers=o.install_parallel)
-        newvars = dotfile.get_dotfile_variables()
-        t.add_tmp_vars(newvars=newvars)
 
-        preactions = []
+        wait_for = [
-        if not o.install_temporary:
+            ex.submit(_dotfile_install, o, dotfile, tmpdir=tmpdir)
-            preactions.extend(dotfile.get_pre_actions())
+            for dotfile in dotfiles
-        defactions = o.install_default_actions_pre
+        ]
-        pre_actions_exec = action_executor(o, preactions, defactions,
+        for f in futures.as_completed(wait_for):
-                                           t, post=False)
+            r, key, err = f.result()
-
+            if r:
-        if o.debug:
+                installed += 1
-            LOG.dbg('installing dotfile: \"{}\"'.format(dotfile.key))
+            elif err:
-            LOG.dbg(dotfile.prt())
+                LOG.err('installing \"{}\" failed: {}'.format(key,
-        if hasattr(dotfile, 'link') and dotfile.link == LinkTypes.LINK:
+                                                              err))
-            r, err = inst.link(t, dotfile.src, dotfile.dst,
+    else:
-                               actionexec=pre_actions_exec,
+        # sequentially
-                               template=dotfile.template)
+        for dotfile in dotfiles:
-        elif hasattr(dotfile, 'link') and \
+            r, key, err = _dotfile_install(o, dotfile, tmpdir=tmpdir)
-                dotfile.link == LinkTypes.LINK_CHILDREN:
+            if r:
-            r, err = inst.link_children(t, dotfile.src, dotfile.dst,
+                installed += 1
-                                        actionexec=pre_actions_exec,
+            elif err:
-                                        template=dotfile.template)
+                LOG.err('installing \"{}\" failed: {}'.format(key,
-        else:
-            src = dotfile.src
-            tmp = None
-            if dotfile.trans_r:
-                tmp = apply_trans(o.dotpath, dotfile, t, debug=o.debug)
-                if not tmp:
-                    continue
-                src = tmp
-            ignores = list(set(o.install_ignore + dotfile.instignore))
-            ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug)
-            r, err = inst.install(t, src, dotfile.dst,
-                                  actionexec=pre_actions_exec,
-                                  noempty=dotfile.noempty,
-                                  ignore=ignores,
-                                  template=dotfile.template)
-            if tmp:
-                tmp = os.path.join(o.dotpath, tmp)
-                if os.path.exists(tmp):
-                    removepath(tmp, LOG)
-        if r:
-            # dotfile was installed
-            if not o.install_temporary:
-                defactions = o.install_default_actions_post
-                postactions = dotfile.get_post_actions()
-                post_actions_exec = action_executor(o, postactions, defactions,
-                                                    t, post=True)
-                post_actions_exec()
-            installed += 1
-        elif not r:
-            # dotfile was NOT installed
-            if o.install_force_action:
-                # pre-actions
-                if o.debug:
-                    LOG.dbg('force pre action execution ...')
-                pre_actions_exec()
-                # post-actions
-                if o.debug:
-                    LOG.dbg('force post action execution ...')
-                defactions = o.install_default_actions_post
-                postactions = dotfile.get_post_actions()
-                post_actions_exec = action_executor(o, postactions, defactions,
-                                                    t, post=True)
-                post_actions_exec()
-            if err:
-                LOG.err('installing \"{}\" failed: {}'.format(dotfile.key,
                                                               err))
 
     # execute profile post-action
@@ -193,7 +223,7 @@ def cmd_install(o):
             return False
 
     if o.debug:
-        LOG.dbg('install done')
+        LOG.dbg('install done - {} installed'.format(installed))
 
     if o.install_temporary:
         LOG.log('\ninstalled to tmp \"{}\".'.format(tmpdir))
@@ -217,9 +247,7 @@ def cmd_compare(o, tmp):
     if len(selected) < 1:
         return False
 
-    t = Templategen(base=o.dotpath, variables=o.variables,
+    t = _get_templater(o)
-                    func_file=o.func_file, filter_file=o.filter_file,
-                    debug=o.debug)
     tvars = t.add_tmp_vars()
     inst = Installer(create=o.create, backup=o.backup,
                      dry=o.dry, base=o.dotpath,
@@ -627,6 +655,27 @@ def cmd_remove(o):
 ###########################################################
 
 
+def _get_install_installer(o, tmpdir=None):
+    """get an installer instance for cmd_install"""
+    inst = Installer(create=o.create, backup=o.backup,
+                     dry=o.dry, safe=o.safe,
+                     base=o.dotpath, workdir=o.workdir,
+                     diff=o.install_diff, debug=o.debug,
+                     totemp=tmpdir,
+                     showdiff=o.install_showdiff,
+                     backup_suffix=o.install_backup_suffix,
+                     diff_cmd=o.diff_command)
+    return inst
+
+
+def _get_templater(o):
+    """get an templater instance"""
+    t = Templategen(base=o.dotpath, variables=o.variables,
+                    func_file=o.func_file, filter_file=o.filter_file,
+                    debug=o.debug)
+    return t
+
+
 def _detail(dotpath, dotfile):
     """display details on all files under a dotfile entry"""
     LOG.log('{} (dst: \"{}\", link: {})'.format(dotfile.key, dotfile.dst,

dotdrop/options.py

@@ -52,7 +52,8 @@ USAGE = """
 {}
 
 Usage:
-  dotdrop install   [-VbtfndDa] [-c <path>] [-p <profile>] [<key>...]
+  dotdrop install   [-VbtfndDa] [-c <path>] [-p <profile>]
+                                [-w <nb>] [<key>...]
   dotdrop import    [-Vbdf]     [-c <path>] [-p <profile>] [-s <path>]
                                 [-l <link>] <path>...
   dotdrop compare   [-LVb]      [-c <path>] [-p <profile>]
@@ -68,24 +69,25 @@ Usage:
 
 Options:
   -a --force-actions      Execute all actions even if no dotfile is installed.
+  -b --no-banner          Do not display the banner.
   -c --cfg=<path>         Path to the config.
   -C --file=<path>        Path of dotfile to compare.
-  -i --ignore=<pattern>   Pattern to ignore.
+  -d --dry                Dry run.
   -l --link=<link>        Link option (nolink|link|link_children).
   -L --file-only          Do not show diff but only the files that differ.
   -p --profile=<profile>  Specify the profile to use [default: {}].
-  -s --as=<path>          Import as a different path from actual path.
-  -b --no-banner          Do not display the banner.
-  -d --dry                Dry run.
   -D --showdiff           Show a diff before overwriting.
   -f --force              Do not ask user confirmation for anything.
   -G --grepable           Grepable output.
+  -i --ignore=<pattern>   Pattern to ignore.
   -k --key                Treat <path> as a dotfile key.
   -n --nodiff             Do not diff when installing.
   -P --show-patch         Provide a one-liner to manually patch template.
+  -s --as=<path>          Import as a different path from actual path.
   -t --temp               Install to a temporary directory for review.
   -T --template           Only template dotfiles.
   -V --verbose            Be verbose.
+  -w --workers=<nb>       Number of concurrent workers [default: 1].
   -v --version            Show version.
   -h --help               Show this screen.
 """.format(BANNER, PROFILE)
@@ -237,6 +239,7 @@ class Options(AttrMonitor):
         self.install_default_actions_post = [a for a in self.default_actions
                                              if a.kind == Action.post]
         self.install_ignore = self.instignore
+        self.install_parallel = int(self.args['--workers'])
         # "compare" specifics
         self.compare_focus = self.args['--file']
         self.compare_ignore = self.args['--ignore']

tests-ng/global-update-ignore.sh

@@ -98,8 +98,10 @@ cd ${ddpath} | ${bin} update -f -c ${cfg} --verbose --profile=p1 --key f_abc
 #tree ${dt}
 
 # check files haven't been updated
+[ ! -e ${dt}/a/c/acfile ] && echo "acfile not found" && exit 1
+cat ${dt}/a/c/acfile
 grep 'b' ${dt}/a/c/acfile >/dev/null
-[ -e ${dt}/a/newfile ] && exit 1
+[ -e ${dt}/a/newfile ] && echo "newfile found" && exit 1
 
 ## CLEANING
 rm -rf ${tmps} ${tmpd}

tests-ng/include-order.sh

@@ -108,7 +108,7 @@ echo "third" > ${tmps}/dotfiles/third
 attempts="3"
 for ((i=0;i<${attempts};i++)); do
   # install
-  cd ${ddpath} | ${bin} install -f -c ${cfg} -p p0 -V
+  cd ${ddpath} | ${bin} install -w 1 -f -c ${cfg} -p p0 -V
 
   # checks timestamp
   echo "first timestamp: `stat -c %y ${tmpd}/first`"

tests.sh

@@ -52,13 +52,13 @@ unset DOTDROP_FORCE_NODEBUG
     set +e
     ${scr} > "${logfile}" 2>&1
     if [ "$?" -ne 0 ]; then
-      echo "test ${scr} finished with error"
       cat ${logfile}
+      echo "test ${scr} finished with error"
       rm -rf ${logdir}
       exit 1
     elif grep Traceback ${logfile}; then
-      echo "test ${scr} crashed"
       cat ${logfile}
+      echo "test ${scr} crashed"
       rm -rf ${logdir}
       exit 1
     fi

tests/helpers.py

@@ -131,6 +131,7 @@ def _fake_args():
     args['--grepable'] = False
     args['--as'] = None
     args['--file-only'] = False
+    args['--workers'] = 1
     # cmds
     args['profiles'] = False
     args['files'] = False