From 4d12859714112872a2228bc311b2c5f0ad2a0782 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 16 Nov 2020 21:47:14 +0100 Subject: [PATCH] parallel compare --- dotdrop/dotdrop.py | 222 ++++++++++++++++++++++---------------- dotdrop/installer.py | 2 +- dotdrop/options.py | 30 ++---- tests-ng/chmod-compare.sh | 2 +- 4 files changed, 140 insertions(+), 116 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 536dc03..da4a879 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -72,6 +72,10 @@ def action_executor(o, actions, defactions, templater, post=False): def _dotfile_update(o, path, key=False): + """ + update a dotfile pointed by path + if key is false or by key (in path) + """ updater = Updater(o.dotpath, o.variables, o.conf, dry=o.dry, safe=o.safe, debug=o.debug, ignore=o.update_ignore, @@ -81,6 +85,91 @@ def _dotfile_update(o, path, key=False): return updater.update_path(path) +def _dotfile_compare(o, dotfile, tmp): + """ + compare a dotfile + returns True if same + """ + t = _get_templater(o) + inst = Installer(create=o.create, backup=o.backup, + dry=o.dry, base=o.dotpath, + workdir=o.workdir, debug=o.debug, + backup_suffix=o.install_backup_suffix, + diff_cmd=o.diff_command) + comp = Comparator(diff_cmd=o.diff_command, debug=o.debug) + + # add dotfile variables + newvars = dotfile.get_dotfile_variables() + t.add_tmp_vars(newvars=newvars) + + # dotfiles does not exist / not installed + if o.debug: + LOG.dbg('comparing {}'.format(dotfile)) + + src = dotfile.src + if not os.path.lexists(os.path.expanduser(dotfile.dst)): + line = '=> compare {}: \"{}\" does not exist on destination' + LOG.log(line.format(dotfile.key, dotfile.dst)) + return False + + # apply transformation + tmpsrc = None + if dotfile.trans_r: + if o.debug: + LOG.dbg('applying transformation before comparing') + tmpsrc = apply_trans(o.dotpath, dotfile, t, debug=o.debug) + if not tmpsrc: + # could not apply trans + return False + src = tmpsrc + + # is a symlink pointing to itself + asrc = os.path.join(o.dotpath, os.path.expanduser(src)) + adst = os.path.expanduser(dotfile.dst) + if os.path.samefile(asrc, adst): + if o.debug: + line = '=> compare {}: diffing with \"{}\"' + LOG.dbg(line.format(dotfile.key, dotfile.dst)) + LOG.dbg('points to itself') + return True + + # install dotfile to temporary dir and compare + ret, err, insttmp = inst.install_to_temp(t, tmp, src, dotfile.dst, + template=dotfile.template, + chmod=dotfile.chmod) + if not ret: + # failed to install to tmp + line = '=> compare {}: error' + LOG.log(line.format(dotfile.key, err)) + LOG.err(err) + return False + ignores = list(set(o.compare_ignore + dotfile.cmpignore)) + ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) + diff = comp.compare(insttmp, dotfile.dst, ignore=ignores) + + # clean tmp transformed dotfile if any + if tmpsrc: + tmpsrc = os.path.join(o.dotpath, tmpsrc) + if os.path.exists(tmpsrc): + removepath(tmpsrc, LOG) + + if diff != '': + # print diff results + line = '=> compare {}: diffing with \"{}\"' + LOG.log(line.format(dotfile.key, dotfile.dst)) + if o.compare_fileonly: + LOG.raw('') + else: + LOG.emph(diff) + return False + # no difference + if o.debug: + line = '=> compare {}: diffing with \"{}\"' + LOG.dbg(line.format(dotfile.key, dotfile.dst)) + LOG.dbg('same file') + return True + + def _dotfile_install(o, dotfile, tmpdir=None): """ install a dotfile @@ -175,8 +264,8 @@ def cmd_install(o): dotfiles = o.dotfiles prof = o.conf.get_profile() - # ensure parallel install is unattended - if o.install_parallel > 1 and o.safe: + # ensure parallel is unattended + if o.workers > 1 and o.safe: LOG.err('\"-w --workers\" must be used with \"-f --force\"') return False @@ -208,9 +297,9 @@ def cmd_install(o): return False # install each dotfile - if o.install_parallel > 1: + if o.workers > 1: # in parallel - ex = futures.ThreadPoolExecutor(max_workers=o.install_parallel) + ex = futures.ThreadPoolExecutor(max_workers=o.workers) wait_for = [] for dotfile in dotfiles: @@ -269,14 +358,14 @@ def cmd_install(o): def cmd_compare(o, tmp): """compare dotfiles and return True if all identical""" - cnt = 0 + # ensure parallel is unattended dotfiles = o.dotfiles if not dotfiles: msg = 'no dotfile defined for this profile (\"{}\")' LOG.warn(msg.format(o.profile)) return True + # compare only specific files - same = True selected = dotfiles if o.compare_focus: selected = _select(o.compare_focus, dotfiles) @@ -285,94 +374,32 @@ def cmd_compare(o, tmp): LOG.log('\nno dotfile to compare') return False - t = _get_templater(o) - tvars = t.add_tmp_vars() - inst = Installer(create=o.create, backup=o.backup, - dry=o.dry, base=o.dotpath, - workdir=o.workdir, debug=o.debug, - backup_suffix=o.install_backup_suffix, - diff_cmd=o.diff_command) - comp = Comparator(diff_cmd=o.diff_command, debug=o.debug) - - for dotfile in selected: - if not dotfile.src and not dotfile.dst: - # ignore fake dotfile - continue - cnt += 1 - - # add dotfile variables - t.restore_vars(tvars) - newvars = dotfile.get_dotfile_variables() - t.add_tmp_vars(newvars=newvars) - - # dotfiles does not exist / not installed - if o.debug: - LOG.dbg('comparing {}'.format(dotfile)) - src = dotfile.src - if not os.path.lexists(os.path.expanduser(dotfile.dst)): - line = '=> compare {}: \"{}\" does not exist on destination' - LOG.log(line.format(dotfile.key, dotfile.dst)) - same = False - continue - - # apply transformation - tmpsrc = None - if dotfile.trans_r: - if o.debug: - LOG.dbg('applying transformation before comparing') - tmpsrc = apply_trans(o.dotpath, dotfile, t, debug=o.debug) - if not tmpsrc: - # could not apply trans - same = False + same = True + cnt = 0 + if o.workers > 1: + # in parallel + ex = futures.ThreadPoolExecutor(max_workers=o.workers) + wait_for = [] + for dotfile in selected: + j = ex.submit(_dotfile_compare, o, dotfile, tmp) + wait_for.append(j) + # check result + for f in futures.as_completed(wait_for): + if not dotfile.src and not dotfile.dst: + # ignore fake dotfile continue - src = tmpsrc - - # is a symlink pointing to itself - asrc = os.path.join(o.dotpath, os.path.expanduser(src)) - adst = os.path.expanduser(dotfile.dst) - if os.path.samefile(asrc, adst): - if o.debug: - line = '=> compare {}: diffing with \"{}\"' - LOG.dbg(line.format(dotfile.key, dotfile.dst)) - LOG.dbg('points to itself') - continue - - # install dotfile to temporary dir and compare - ret, err, insttmp = inst.install_to_temp(t, tmp, src, dotfile.dst, - template=dotfile.template, - chmod=dotfile.chmod) - if not ret: - # failed to install to tmp - line = '=> compare {}: error' - LOG.log(line.format(dotfile.key, err)) - LOG.err(err) - same = False - continue - ignores = list(set(o.compare_ignore + dotfile.cmpignore)) - ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) - diff = comp.compare(insttmp, dotfile.dst, ignore=ignores) - - # clean tmp transformed dotfile if any - if tmpsrc: - tmpsrc = os.path.join(o.dotpath, tmpsrc) - if os.path.exists(tmpsrc): - removepath(tmpsrc, LOG) - - if diff == '': - # no difference - if o.debug: - line = '=> compare {}: diffing with \"{}\"' - LOG.dbg(line.format(dotfile.key, dotfile.dst)) - LOG.dbg('same file') - else: - # print diff results - line = '=> compare {}: diffing with \"{}\"' - LOG.log(line.format(dotfile.key, dotfile.dst)) - if o.compare_fileonly: - LOG.raw('') - else: - LOG.emph(diff) - same = False + if not f.result(): + same = False + cnt += 1 + else: + # sequentially + for dotfile in selected: + if not dotfile.src and not dotfile.dst: + # ignore fake dotfile + continue + if not _dotfile_compare(o, dotfile, tmp): + same = False + cnt += 1 LOG.log('\n{} dotfile(s) compared.'.format(cnt)) return same @@ -380,6 +407,11 @@ def cmd_compare(o, tmp): def cmd_update(o): """update the dotfile(s) from path(s) or key(s)""" + # ensure parallel is unattended + if o.workers > 1 and o.safe: + LOG.err('\"-w --workers\" must be used with \"-f --force\"') + return False + cnt = 0 paths = o.update_path iskey = o.update_iskey @@ -407,9 +439,9 @@ def cmd_update(o): LOG.dbg('dotfile to update: {}'.format(paths)) # update each dotfile - if o.update_parallel > 1: + if o.workers > 1: # in parallel - ex = futures.ThreadPoolExecutor(max_workers=o.update_parallel) + ex = futures.ThreadPoolExecutor(max_workers=o.workers) wait_for = [] for path in paths: diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 55588fe..1dee366 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -715,7 +715,7 @@ class Installer: if self.debug: self.log.dbg('mkdir -p {}'.format(directory)) - os.makedirs(directory) + os.makedirs(directory, exist_ok=True) return os.path.exists(directory) def _backup(self, path): diff --git a/dotdrop/options.py b/dotdrop/options.py index efd67c2..7fc7680 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -58,7 +58,7 @@ Usage: dotdrop import [-Vbdfm] [-c ] [-p ] [-s ] [-l ] ... dotdrop compare [-LVb] [-c ] [-p ] - [-C ...] [-i ...] + [-w ] [-C ...] [-i ...] dotdrop update [-VbfdkP] [-c ] [-p ] [-w ] [-i ...] [...] dotdrop remove [-Vbfdk] [-c ] [-p ] [...] @@ -217,6 +217,16 @@ class Options(AttrMonitor): # adapt attributes based on arguments self.safe = not self.args['--force'] + try: + if ENV_WORKERS in os.environ: + workers = int(os.environ[ENV_WORKERS]) + else: + workers = int(self.args['--workers']) + self.workers = workers + except ValueError: + self.log.err('bad option for --workers') + sys.exit(USAGE) + # import link default value self.import_link = self.link_on_import if self.args['--link']: @@ -246,15 +256,6 @@ class Options(AttrMonitor): self.install_default_actions_post = [a for a in self.default_actions if a.kind == Action.post] self.install_ignore = self.instignore - try: - if ENV_WORKERS in os.environ: - workers = int(os.environ[ENV_WORKERS]) - else: - workers = int(self.args['--workers']) - self.install_parallel = workers - except ValueError: - self.log.err('bad option for --workers') - sys.exit(USAGE) # "compare" specifics self.compare_focus = self.args['--file'] @@ -277,15 +278,6 @@ class Options(AttrMonitor): self.update_ignore.append('*{}'.format(self.install_backup_suffix)) self.update_ignore = uniq_list(self.update_ignore) self.update_showpatch = self.args['--show-patch'] - try: - if ENV_WORKERS in os.environ: - workers = int(os.environ[ENV_WORKERS]) - else: - workers = int(self.args['--workers']) - self.update_parallel = workers - except ValueError: - self.log.err('bad option for --workers') - sys.exit(USAGE) # "detail" specifics self.detail_keys = self.args[''] diff --git a/tests-ng/chmod-compare.sh b/tests-ng/chmod-compare.sh index c92be03..6a2b909 100755 --- a/tests-ng/chmod-compare.sh +++ b/tests-ng/chmod-compare.sh @@ -111,7 +111,7 @@ chmod 700 ${flink} set +e cnt=`cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 2>&1 | grep 'modes differ' | wc -l` set -e -[ "${cnt}" != "5" ] && echo "compare modes failed" && exit 1 +[ "${cnt}" != "5" ] && echo "compare modes failed (${cnt})" && exit 1 ## CLEANING rm -rf ${tmps} ${tmpd}