From fe4e881b008117f2c36b7a1f9f1c57e890fd1e56 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 7 Nov 2020 22:05:13 +0100 Subject: [PATCH 1/9] concurrent installation --- dotdrop/dotdrop.py | 215 +++++++++++++++++++++++++++------------------ dotdrop/options.py | 10 ++- 2 files changed, 137 insertions(+), 88 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index be7b496..9cb1daf 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -7,7 +7,7 @@ entry point import os import sys - +from concurrent import futures import shutil # local imports @@ -24,6 +24,7 @@ from dotdrop.exceptions import YamlException, UndefinedException LOG = Logger() TRANS_SUFFIX = 'trans' +INST_WORKERS = 10 ########################################################### # entry point @@ -70,6 +71,89 @@ def action_executor(o, actions, defactions, templater, post=False): return execute +def _dotfile_install(o, dotfile, tmpdir=None): + """install a dotfile""" + # installer + inst = _get_install_installer(o, tmpdir=tmpdir) + + # templater + t = _get_templater(o) + + # add dotfile variables + newvars = dotfile.get_dotfile_variables() + t.add_tmp_vars(newvars=newvars) + + preactions = [] + if not o.install_temporary: + preactions.extend(dotfile.get_pre_actions()) + defactions = o.install_default_actions_pre + pre_actions_exec = action_executor(o, preactions, defactions, + t, post=False) + + if o.debug: + LOG.dbg('installing dotfile: \"{}\"'.format(dotfile.key)) + LOG.dbg(dotfile.prt()) + + if hasattr(dotfile, 'link') and dotfile.link == LinkTypes.LINK: + # link + r, err = inst.link(t, dotfile.src, dotfile.dst, + actionexec=pre_actions_exec, + template=dotfile.template) + elif hasattr(dotfile, 'link') and \ + dotfile.link == LinkTypes.LINK_CHILDREN: + # link_children + r, err = inst.link_children(t, dotfile.src, dotfile.dst, + actionexec=pre_actions_exec, + template=dotfile.template) + else: + # nolink + src = dotfile.src + tmp = None + if dotfile.trans_r: + tmp = apply_trans(o.dotpath, dotfile, t, debug=o.debug) + if not tmp: + return False, dotfile.key, None + src = tmp + ignores = list(set(o.install_ignore + dotfile.instignore)) + ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) + r, err = inst.install(t, src, dotfile.dst, + actionexec=pre_actions_exec, + noempty=dotfile.noempty, + ignore=ignores, + template=dotfile.template) + if tmp: + tmp = os.path.join(o.dotpath, tmp) + if os.path.exists(tmp): + removepath(tmp, LOG) + + # check result of installation + if r: + # dotfile was installed + if not o.install_temporary: + defactions = o.install_default_actions_post + postactions = dotfile.get_post_actions() + post_actions_exec = action_executor(o, postactions, defactions, + t, post=True) + post_actions_exec() + else: + # dotfile was NOT installed + if o.install_force_action: + # pre-actions + if o.debug: + LOG.dbg('force pre action execution ...') + pre_actions_exec() + # post-actions + if o.debug: + LOG.dbg('force post action execution ...') + defactions = o.install_default_actions_post + postactions = dotfile.get_post_actions() + post_actions_exec = action_executor(o, postactions, defactions, + t, post=True) + post_actions_exec() + + return r, dotfile.key, err + + def cmd_install(o): """install dotfiles for this profile""" dotfiles = o.dotfiles @@ -86,100 +170,44 @@ def cmd_install(o): LOG.warn(msg.format(o.profile)) return False - t = Templategen(base=o.dotpath, variables=o.variables, - func_file=o.func_file, filter_file=o.filter_file, - debug=o.debug) + # the installer tmpdir = None if o.install_temporary: tmpdir = get_tmpdir() - inst = Installer(create=o.create, backup=o.backup, - dry=o.dry, safe=o.safe, - base=o.dotpath, workdir=o.workdir, - diff=o.install_diff, debug=o.debug, - totemp=tmpdir, - showdiff=o.install_showdiff, - backup_suffix=o.install_backup_suffix, - diff_cmd=o.diff_command) + installed = 0 - tvars = t.add_tmp_vars() # execute profile pre-action if o.debug: LOG.dbg('run {} profile pre actions'.format(len(pro_pre_actions))) + t = _get_templater(o) ret, err = action_executor(o, pro_pre_actions, [], t, post=False)() if not ret: return False # install each dotfile - for dotfile in dotfiles: - # add dotfile variables - t.restore_vars(tvars) - newvars = dotfile.get_dotfile_variables() - t.add_tmp_vars(newvars=newvars) + if o.install_parallel: + # in parallel + ex = futures.ThreadPoolExecutor(max_workers=INST_WORKERS) - preactions = [] - if not o.install_temporary: - preactions.extend(dotfile.get_pre_actions()) - defactions = o.install_default_actions_pre - pre_actions_exec = action_executor(o, preactions, defactions, - t, post=False) - - if o.debug: - LOG.dbg('installing dotfile: \"{}\"'.format(dotfile.key)) - LOG.dbg(dotfile.prt()) - if hasattr(dotfile, 'link') and dotfile.link == LinkTypes.LINK: - r, err = inst.link(t, dotfile.src, dotfile.dst, - actionexec=pre_actions_exec, - template=dotfile.template) - elif hasattr(dotfile, 'link') and \ - dotfile.link == LinkTypes.LINK_CHILDREN: - r, err = inst.link_children(t, dotfile.src, dotfile.dst, - actionexec=pre_actions_exec, - template=dotfile.template) - else: - src = dotfile.src - tmp = None - if dotfile.trans_r: - tmp = apply_trans(o.dotpath, dotfile, t, debug=o.debug) - if not tmp: - continue - src = tmp - ignores = list(set(o.install_ignore + dotfile.instignore)) - ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) - r, err = inst.install(t, src, dotfile.dst, - actionexec=pre_actions_exec, - noempty=dotfile.noempty, - ignore=ignores, - template=dotfile.template) - if tmp: - tmp = os.path.join(o.dotpath, tmp) - if os.path.exists(tmp): - removepath(tmp, LOG) - if r: - # dotfile was installed - if not o.install_temporary: - defactions = o.install_default_actions_post - postactions = dotfile.get_post_actions() - post_actions_exec = action_executor(o, postactions, defactions, - t, post=True) - post_actions_exec() - installed += 1 - elif not r: - # dotfile was NOT installed - if o.install_force_action: - # pre-actions - if o.debug: - LOG.dbg('force pre action execution ...') - pre_actions_exec() - # post-actions - if o.debug: - LOG.dbg('force post action execution ...') - defactions = o.install_default_actions_post - postactions = dotfile.get_post_actions() - post_actions_exec = action_executor(o, postactions, defactions, - t, post=True) - post_actions_exec() - if err: + wait_for = [ + ex.submit(_dotfile_install, o, dotfile, tmpdir=tmpdir) + for dotfile in dotfiles + ] + for f in futures.as_completed(wait_for): + r, key, err = f.result() + if r: + installed += 1 + elif err: + LOG.err('installing \"{}\" failed: {}'.format(key, + err)) + else: + # sequentially + for dotfile in dotfiles: + r, err = _dotfile_install(o, dotfile, tmpdir=tmpdir) + if r: + installed += 1 + elif err: LOG.err('installing \"{}\" failed: {}'.format(dotfile.key, err)) @@ -217,9 +245,7 @@ def cmd_compare(o, tmp): if len(selected) < 1: return False - t = Templategen(base=o.dotpath, variables=o.variables, - func_file=o.func_file, filter_file=o.filter_file, - debug=o.debug) + t = _get_templater(o) tvars = t.add_tmp_vars() inst = Installer(create=o.create, backup=o.backup, dry=o.dry, base=o.dotpath, @@ -627,6 +653,27 @@ def cmd_remove(o): ########################################################### +def _get_install_installer(o, tmpdir=None): + """get an installer instance for cmd_install""" + inst = Installer(create=o.create, backup=o.backup, + dry=o.dry, safe=o.safe, + base=o.dotpath, workdir=o.workdir, + diff=o.install_diff, debug=o.debug, + totemp=tmpdir, + showdiff=o.install_showdiff, + backup_suffix=o.install_backup_suffix, + diff_cmd=o.diff_command) + return inst + + +def _get_templater(o): + """get an templater instance""" + t = Templategen(base=o.dotpath, variables=o.variables, + func_file=o.func_file, filter_file=o.filter_file, + debug=o.debug) + return t + + def _detail(dotpath, dotfile): """display details on all files under a dotfile entry""" LOG.log('{} (dst: \"{}\", link: {})'.format(dotfile.key, dotfile.dst, diff --git a/dotdrop/options.py b/dotdrop/options.py index 47dc835..d270200 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -67,21 +67,21 @@ Usage: Options: -a --force-actions Execute all actions even if no dotfile is installed. + -b --no-banner Do not display the banner. -c --cfg= Path to the config. -C --file= Path of dotfile to compare. - -i --ignore= Pattern to ignore. + -d --dry Dry run. -l --link= Link option (nolink|link|link_children). -L --file-only Do not show diff but only the files that differ. -p --profile= Specify the profile to use [default: {}]. - -s --as= Import as a different path from actual path. - -b --no-banner Do not display the banner. - -d --dry Dry run. -D --showdiff Show a diff before overwriting. -f --force Do not ask user confirmation for anything. -G --grepable Grepable output. + -i --ignore= Pattern to ignore. -k --key Treat as a dotfile key. -n --nodiff Do not diff when installing. -P --show-patch Provide a one-liner to manually patch template. + -s --as= Import as a different path from actual path. -t --temp Install to a temporary directory for review. -T --template Only template dotfiles. -V --verbose Be verbose. @@ -234,6 +234,8 @@ class Options(AttrMonitor): self.install_default_actions_post = [a for a in self.default_actions if a.kind == Action.post] self.install_ignore = self.instignore + # TODO add as option + self.install_parallel = True # "compare" specifics self.compare_focus = self.args['--file'] self.compare_ignore = self.args['--ignore'] From 5ef3ae454077f82c69a9334afd47a8ab8e949758 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 7 Nov 2020 22:05:13 +0100 Subject: [PATCH 2/9] concurrent installation --- dotdrop/dotdrop.py | 215 +++++++++++++++++++++++++++------------------ dotdrop/options.py | 10 ++- 2 files changed, 137 insertions(+), 88 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index be7b496..9cb1daf 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -7,7 +7,7 @@ entry point import os import sys - +from concurrent import futures import shutil # local imports @@ -24,6 +24,7 @@ from dotdrop.exceptions import YamlException, UndefinedException LOG = Logger() TRANS_SUFFIX = 'trans' +INST_WORKERS = 10 ########################################################### # entry point @@ -70,6 +71,89 @@ def action_executor(o, actions, defactions, templater, post=False): return execute +def _dotfile_install(o, dotfile, tmpdir=None): + """install a dotfile""" + # installer + inst = _get_install_installer(o, tmpdir=tmpdir) + + # templater + t = _get_templater(o) + + # add dotfile variables + newvars = dotfile.get_dotfile_variables() + t.add_tmp_vars(newvars=newvars) + + preactions = [] + if not o.install_temporary: + preactions.extend(dotfile.get_pre_actions()) + defactions = o.install_default_actions_pre + pre_actions_exec = action_executor(o, preactions, defactions, + t, post=False) + + if o.debug: + LOG.dbg('installing dotfile: \"{}\"'.format(dotfile.key)) + LOG.dbg(dotfile.prt()) + + if hasattr(dotfile, 'link') and dotfile.link == LinkTypes.LINK: + # link + r, err = inst.link(t, dotfile.src, dotfile.dst, + actionexec=pre_actions_exec, + template=dotfile.template) + elif hasattr(dotfile, 'link') and \ + dotfile.link == LinkTypes.LINK_CHILDREN: + # link_children + r, err = inst.link_children(t, dotfile.src, dotfile.dst, + actionexec=pre_actions_exec, + template=dotfile.template) + else: + # nolink + src = dotfile.src + tmp = None + if dotfile.trans_r: + tmp = apply_trans(o.dotpath, dotfile, t, debug=o.debug) + if not tmp: + return False, dotfile.key, None + src = tmp + ignores = list(set(o.install_ignore + dotfile.instignore)) + ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) + r, err = inst.install(t, src, dotfile.dst, + actionexec=pre_actions_exec, + noempty=dotfile.noempty, + ignore=ignores, + template=dotfile.template) + if tmp: + tmp = os.path.join(o.dotpath, tmp) + if os.path.exists(tmp): + removepath(tmp, LOG) + + # check result of installation + if r: + # dotfile was installed + if not o.install_temporary: + defactions = o.install_default_actions_post + postactions = dotfile.get_post_actions() + post_actions_exec = action_executor(o, postactions, defactions, + t, post=True) + post_actions_exec() + else: + # dotfile was NOT installed + if o.install_force_action: + # pre-actions + if o.debug: + LOG.dbg('force pre action execution ...') + pre_actions_exec() + # post-actions + if o.debug: + LOG.dbg('force post action execution ...') + defactions = o.install_default_actions_post + postactions = dotfile.get_post_actions() + post_actions_exec = action_executor(o, postactions, defactions, + t, post=True) + post_actions_exec() + + return r, dotfile.key, err + + def cmd_install(o): """install dotfiles for this profile""" dotfiles = o.dotfiles @@ -86,100 +170,44 @@ def cmd_install(o): LOG.warn(msg.format(o.profile)) return False - t = Templategen(base=o.dotpath, variables=o.variables, - func_file=o.func_file, filter_file=o.filter_file, - debug=o.debug) + # the installer tmpdir = None if o.install_temporary: tmpdir = get_tmpdir() - inst = Installer(create=o.create, backup=o.backup, - dry=o.dry, safe=o.safe, - base=o.dotpath, workdir=o.workdir, - diff=o.install_diff, debug=o.debug, - totemp=tmpdir, - showdiff=o.install_showdiff, - backup_suffix=o.install_backup_suffix, - diff_cmd=o.diff_command) + installed = 0 - tvars = t.add_tmp_vars() # execute profile pre-action if o.debug: LOG.dbg('run {} profile pre actions'.format(len(pro_pre_actions))) + t = _get_templater(o) ret, err = action_executor(o, pro_pre_actions, [], t, post=False)() if not ret: return False # install each dotfile - for dotfile in dotfiles: - # add dotfile variables - t.restore_vars(tvars) - newvars = dotfile.get_dotfile_variables() - t.add_tmp_vars(newvars=newvars) + if o.install_parallel: + # in parallel + ex = futures.ThreadPoolExecutor(max_workers=INST_WORKERS) - preactions = [] - if not o.install_temporary: - preactions.extend(dotfile.get_pre_actions()) - defactions = o.install_default_actions_pre - pre_actions_exec = action_executor(o, preactions, defactions, - t, post=False) - - if o.debug: - LOG.dbg('installing dotfile: \"{}\"'.format(dotfile.key)) - LOG.dbg(dotfile.prt()) - if hasattr(dotfile, 'link') and dotfile.link == LinkTypes.LINK: - r, err = inst.link(t, dotfile.src, dotfile.dst, - actionexec=pre_actions_exec, - template=dotfile.template) - elif hasattr(dotfile, 'link') and \ - dotfile.link == LinkTypes.LINK_CHILDREN: - r, err = inst.link_children(t, dotfile.src, dotfile.dst, - actionexec=pre_actions_exec, - template=dotfile.template) - else: - src = dotfile.src - tmp = None - if dotfile.trans_r: - tmp = apply_trans(o.dotpath, dotfile, t, debug=o.debug) - if not tmp: - continue - src = tmp - ignores = list(set(o.install_ignore + dotfile.instignore)) - ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) - r, err = inst.install(t, src, dotfile.dst, - actionexec=pre_actions_exec, - noempty=dotfile.noempty, - ignore=ignores, - template=dotfile.template) - if tmp: - tmp = os.path.join(o.dotpath, tmp) - if os.path.exists(tmp): - removepath(tmp, LOG) - if r: - # dotfile was installed - if not o.install_temporary: - defactions = o.install_default_actions_post - postactions = dotfile.get_post_actions() - post_actions_exec = action_executor(o, postactions, defactions, - t, post=True) - post_actions_exec() - installed += 1 - elif not r: - # dotfile was NOT installed - if o.install_force_action: - # pre-actions - if o.debug: - LOG.dbg('force pre action execution ...') - pre_actions_exec() - # post-actions - if o.debug: - LOG.dbg('force post action execution ...') - defactions = o.install_default_actions_post - postactions = dotfile.get_post_actions() - post_actions_exec = action_executor(o, postactions, defactions, - t, post=True) - post_actions_exec() - if err: + wait_for = [ + ex.submit(_dotfile_install, o, dotfile, tmpdir=tmpdir) + for dotfile in dotfiles + ] + for f in futures.as_completed(wait_for): + r, key, err = f.result() + if r: + installed += 1 + elif err: + LOG.err('installing \"{}\" failed: {}'.format(key, + err)) + else: + # sequentially + for dotfile in dotfiles: + r, err = _dotfile_install(o, dotfile, tmpdir=tmpdir) + if r: + installed += 1 + elif err: LOG.err('installing \"{}\" failed: {}'.format(dotfile.key, err)) @@ -217,9 +245,7 @@ def cmd_compare(o, tmp): if len(selected) < 1: return False - t = Templategen(base=o.dotpath, variables=o.variables, - func_file=o.func_file, filter_file=o.filter_file, - debug=o.debug) + t = _get_templater(o) tvars = t.add_tmp_vars() inst = Installer(create=o.create, backup=o.backup, dry=o.dry, base=o.dotpath, @@ -627,6 +653,27 @@ def cmd_remove(o): ########################################################### +def _get_install_installer(o, tmpdir=None): + """get an installer instance for cmd_install""" + inst = Installer(create=o.create, backup=o.backup, + dry=o.dry, safe=o.safe, + base=o.dotpath, workdir=o.workdir, + diff=o.install_diff, debug=o.debug, + totemp=tmpdir, + showdiff=o.install_showdiff, + backup_suffix=o.install_backup_suffix, + diff_cmd=o.diff_command) + return inst + + +def _get_templater(o): + """get an templater instance""" + t = Templategen(base=o.dotpath, variables=o.variables, + func_file=o.func_file, filter_file=o.filter_file, + debug=o.debug) + return t + + def _detail(dotpath, dotfile): """display details on all files under a dotfile entry""" LOG.log('{} (dst: \"{}\", link: {})'.format(dotfile.key, dotfile.dst, diff --git a/dotdrop/options.py b/dotdrop/options.py index 6651982..0815ae8 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -68,21 +68,21 @@ Usage: Options: -a --force-actions Execute all actions even if no dotfile is installed. + -b --no-banner Do not display the banner. -c --cfg= Path to the config. -C --file= Path of dotfile to compare. - -i --ignore= Pattern to ignore. + -d --dry Dry run. -l --link= Link option (nolink|link|link_children). -L --file-only Do not show diff but only the files that differ. -p --profile= Specify the profile to use [default: {}]. - -s --as= Import as a different path from actual path. - -b --no-banner Do not display the banner. - -d --dry Dry run. -D --showdiff Show a diff before overwriting. -f --force Do not ask user confirmation for anything. -G --grepable Grepable output. + -i --ignore= Pattern to ignore. -k --key Treat as a dotfile key. -n --nodiff Do not diff when installing. -P --show-patch Provide a one-liner to manually patch template. + -s --as= Import as a different path from actual path. -t --temp Install to a temporary directory for review. -T --template Only template dotfiles. -V --verbose Be verbose. @@ -237,6 +237,8 @@ class Options(AttrMonitor): self.install_default_actions_post = [a for a in self.default_actions if a.kind == Action.post] self.install_ignore = self.instignore + # TODO add as option + self.install_parallel = True # "compare" specifics self.compare_focus = self.args['--file'] self.compare_ignore = self.args['--ignore'] From e3c9a486d978e342555387c1b5ff7900b32e2c88 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 8 Nov 2020 14:12:52 +0100 Subject: [PATCH 3/9] fix test script --- tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests.sh b/tests.sh index 62167a8..1a18386 100755 --- a/tests.sh +++ b/tests.sh @@ -52,13 +52,13 @@ unset DOTDROP_FORCE_NODEBUG set +e ${scr} > "${logfile}" 2>&1 if [ "$?" -ne 0 ]; then - echo "test ${scr} finished with error" cat ${logfile} + echo "test ${scr} finished with error" rm -rf ${logdir} exit 1 elif grep Traceback ${logfile}; then - echo "test ${scr} crashed" cat ${logfile} + echo "test ${scr} crashed" rm -rf ${logdir} exit 1 fi From a76687465388e4195a95d05e365bb788998d1466 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 8 Nov 2020 14:13:06 +0100 Subject: [PATCH 4/9] add -w --workers cli switch --- dotdrop/options.py | 6 +++--- tests/helpers.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dotdrop/options.py b/dotdrop/options.py index 0815ae8..7825944 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -54,7 +54,7 @@ USAGE = """ Usage: dotdrop install [-VbtfndDa] [-c ] [-p ] [...] dotdrop import [-Vbdf] [-c ] [-p ] [-s ] - [-l ] ... + [-w ] [-l ] ... dotdrop compare [-LVb] [-c ] [-p ] [-C ...] [-i ...] dotdrop update [-VbfdkP] [-c ] [-p ] @@ -86,6 +86,7 @@ Options: -t --temp Install to a temporary directory for review. -T --template Only template dotfiles. -V --verbose Be verbose. + -w --workers= Number of concurrent workers [default: 1]. -v --version Show version. -h --help Show this screen. """.format(BANNER, PROFILE) @@ -237,8 +238,7 @@ class Options(AttrMonitor): self.install_default_actions_post = [a for a in self.default_actions if a.kind == Action.post] self.install_ignore = self.instignore - # TODO add as option - self.install_parallel = True + self.install_parallel = int(self.args['--workers']) # "compare" specifics self.compare_focus = self.args['--file'] self.compare_ignore = self.args['--ignore'] diff --git a/tests/helpers.py b/tests/helpers.py index d6347c2..8abb198 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -131,6 +131,7 @@ def _fake_args(): args['--grepable'] = False args['--as'] = None args['--file-only'] = False + args['--workers'] = 1 # cmds args['profiles'] = False args['files'] = False From ce98e75bf1f03464e237380b778a2315ca253667 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 8 Nov 2020 14:13:19 +0100 Subject: [PATCH 5/9] fix concurrent --- dotdrop/dotdrop.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 9cb1daf..ef6f379 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -72,7 +72,10 @@ def action_executor(o, actions, defactions, templater, post=False): def _dotfile_install(o, dotfile, tmpdir=None): - """install a dotfile""" + """ + install a dotfile + returns + """ # installer inst = _get_install_installer(o, tmpdir=tmpdir) @@ -186,7 +189,7 @@ def cmd_install(o): return False # install each dotfile - if o.install_parallel: + if o.install_parallel > 1: # in parallel ex = futures.ThreadPoolExecutor(max_workers=INST_WORKERS) @@ -204,11 +207,11 @@ def cmd_install(o): else: # sequentially for dotfile in dotfiles: - r, err = _dotfile_install(o, dotfile, tmpdir=tmpdir) + r, key, err = _dotfile_install(o, dotfile, tmpdir=tmpdir) if r: installed += 1 elif err: - LOG.err('installing \"{}\" failed: {}'.format(dotfile.key, + LOG.err('installing \"{}\" failed: {}'.format(key, err)) # execute profile post-action @@ -221,7 +224,7 @@ def cmd_install(o): return False if o.debug: - LOG.dbg('install done') + LOG.dbg('install done - {} installed'.format(installed)) if o.install_temporary: LOG.log('\ninstalled to tmp \"{}\".'.format(tmpdir)) From 80c553e1bb00a6d76fc15ac23b6850cb41e1fb23 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 8 Nov 2020 14:27:58 +0100 Subject: [PATCH 6/9] force single worker --- tests-ng/include-order.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests-ng/include-order.sh b/tests-ng/include-order.sh index c6da500..01c295a 100755 --- a/tests-ng/include-order.sh +++ b/tests-ng/include-order.sh @@ -108,7 +108,7 @@ echo "third" > ${tmps}/dotfiles/third attempts="3" for ((i=0;i<${attempts};i++)); do # install - cd ${ddpath} | ${bin} install -f -c ${cfg} -p p0 -V + cd ${ddpath} | ${bin} install -w 1 -f -c ${cfg} -p p0 -V # checks timestamp echo "first timestamp: `stat -c %y ${tmpd}/first`" From 455f5dd5ffde065c0f5173cc32cd52707540107b Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 8 Nov 2020 14:28:07 +0100 Subject: [PATCH 7/9] more logs --- tests-ng/global-update-ignore.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests-ng/global-update-ignore.sh b/tests-ng/global-update-ignore.sh index 4983d96..5b4c302 100755 --- a/tests-ng/global-update-ignore.sh +++ b/tests-ng/global-update-ignore.sh @@ -98,8 +98,10 @@ cd ${ddpath} | ${bin} update -f -c ${cfg} --verbose --profile=p1 --key f_abc #tree ${dt} # check files haven't been updated +[ ! -e ${dt}/a/c/acfile ] && echo "acfile not found" && exit 1 +cat ${dt}/a/c/acfile grep 'b' ${dt}/a/c/acfile >/dev/null -[ -e ${dt}/a/newfile ] && exit 1 +[ -e ${dt}/a/newfile ] && echo "newfile found" && exit 1 ## CLEANING rm -rf ${tmps} ${tmpd} From 2916dadcaa435b59aed1f1009f86d810b55f1dcc Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 8 Nov 2020 14:28:24 +0100 Subject: [PATCH 8/9] move -w --workers to install --- dotdrop/options.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dotdrop/options.py b/dotdrop/options.py index 7825944..f53ca3f 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -52,9 +52,10 @@ USAGE = """ {} Usage: - dotdrop install [-VbtfndDa] [-c ] [-p ] [...] + dotdrop install [-VbtfndDa] [-c ] [-p ] + [-w ] [...] dotdrop import [-Vbdf] [-c ] [-p ] [-s ] - [-w ] [-l ] ... + [-l ] ... dotdrop compare [-LVb] [-c ] [-p ] [-C ...] [-i ...] dotdrop update [-VbfdkP] [-c ] [-p ] From cf93ac9b219df5f41e1528e05c1e4a7600a0264d Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 8 Nov 2020 15:20:55 +0100 Subject: [PATCH 9/9] actually use provided number of workers --- dotdrop/dotdrop.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index ef6f379..a517ade 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -24,7 +24,6 @@ from dotdrop.exceptions import YamlException, UndefinedException LOG = Logger() TRANS_SUFFIX = 'trans' -INST_WORKERS = 10 ########################################################### # entry point @@ -191,7 +190,7 @@ def cmd_install(o): # install each dotfile if o.install_parallel > 1: # in parallel - ex = futures.ThreadPoolExecutor(max_workers=INST_WORKERS) + ex = futures.ThreadPoolExecutor(max_workers=o.install_parallel) wait_for = [ ex.submit(_dotfile_install, o, dotfile, tmpdir=tmpdir)