From fc9c12c1b03447070a8a02ebf8936f0d817e12cc Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 9 Nov 2020 13:36:48 +0100 Subject: [PATCH 01/82] add chmod to import command --- dotdrop/cfg_aggregator.py | 9 +++++---- dotdrop/cfg_yaml.py | 31 ++++++++++++++++++++++++++++++- dotdrop/dotdrop.py | 14 ++++++++++++-- dotdrop/dotfile.py | 6 +++++- dotdrop/options.py | 13 ++++++++----- dotdrop/utils.py | 12 ++++++++++++ 6 files changed, 72 insertions(+), 13 deletions(-) diff --git a/dotdrop/cfg_aggregator.py b/dotdrop/cfg_aggregator.py index 230e264..c83afb6 100644 --- a/dotdrop/cfg_aggregator.py +++ b/dotdrop/cfg_aggregator.py @@ -149,27 +149,28 @@ class CfgAggregator: """remove this dotfile from this profile""" return self.cfgyaml.del_dotfile_from_profile(dotfile.key, profile.key) - def _create_new_dotfile(self, src, dst, link): + def _create_new_dotfile(self, src, dst, link, chmod=None): """create a new dotfile""" # get a new dotfile with a unique key key = self._get_new_dotfile_key(dst) if self.debug: self.log.dbg('new dotfile key: {}'.format(key)) # add the dotfile - self.cfgyaml.add_dotfile(key, src, dst, link) + self.cfgyaml.add_dotfile(key, src, dst, link, chmod=chmod) return Dotfile(key, dst, src) - def new(self, src, dst, link): + def new(self, src, dst, link, chmod=None): """ import a new dotfile @src: path in dotpath @dst: path in FS @link: LinkType + @chmod: file permission """ dst = self.path_to_dotfile_dst(dst) dotfile = self.get_dotfile_by_src_dst(src, dst) if not dotfile: - dotfile = self._create_new_dotfile(src, dst, link) + dotfile = self._create_new_dotfile(src, dst, link, chmod=chmod) key = dotfile.key ret = self.cfgyaml.add_dotfile_to_profile(key, self.profile_key) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index c5ccfab..c3c5e7a 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -58,6 +58,7 @@ class CfgYaml: key_dotfile_actions = 'actions' key_dotfile_noempty = 'ignoreempty' key_dotfile_template = 'template' + key_dotfile_chmod = 'chmod' # profile key_profile_dotfiles = 'dotfiles' @@ -316,7 +317,7 @@ class CfgYaml: """return all existing dotfile keys""" return self.dotfiles.keys() - def add_dotfile(self, key, src, dst, link): + def add_dotfile(self, key, src, dst, link, chmod=None): """add a new dotfile""" if key in self.dotfiles.keys(): return False @@ -324,14 +325,23 @@ class CfgYaml: self._dbg('adding new dotfile: {}'.format(key)) self._dbg('new dotfile src: {}'.format(src)) self._dbg('new dotfile dst: {}'.format(dst)) + self._dbg('new dotfile link: {}'.format(link)) + if chmod: + self._dbg('new dotfile chmod: {}'.format(chmod)) df_dict = { self.key_dotfile_src: src, self.key_dotfile_dst: dst, } + # link dfl = self.settings[self.key_settings_link_dotfile_default] if str(link) != dfl: df_dict[self.key_dotfile_link] = str(link) + # chmod + if chmod: + df_dict[self.key_dotfile_chmod] = format(chmod, 'o') + + # add to global dict self._yaml_dict[self.key_dotfiles][key] = df_dict self._dirty = True @@ -623,6 +633,25 @@ class CfgYaml: if self.key_dotfile_template not in v: val = self.settings.get(self.key_settings_template, True) v[self.key_dotfile_template] = val + # validate value of chmod if defined + if self.key_dotfile_chmod in v: + val = v[self.key_dotfile_chmod] + if len(val) < 3: + err = 'bad format for chmod: {}'.format(val) + self._log.err(err) + raise YamlException('dotfile chmod error: {}'.format(err)) + try: + int(val) + except Exception: + err = 'bad format for chmod: {}'.format(val) + self._log.err(err) + raise YamlException('dotfile chmod error: {}'.format(err)) + for x in val: + y = int(x) + if y < 0 or y > 7: + err = 'bad format for chmod: {}'.format(val) + self._log.err(err) + raise YamlException('dotfile chmod error: {}'.format(err)) return new diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 6c06eee..6b87ff6 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -19,7 +19,7 @@ from dotdrop.installer import Installer from dotdrop.updater import Updater from dotdrop.comparator import Comparator from dotdrop.utils import get_tmpdir, removepath, strip_home, \ - uniq_list, patch_ignores, dependencies_met + uniq_list, patch_ignores, dependencies_met, get_file_perm from dotdrop.linktypes import LinkTypes from dotdrop.exceptions import YamlException, UndefinedException @@ -426,6 +426,9 @@ def cmd_importer(o): strip = os.sep src = src.lstrip(strip) + # get the permission + perm = get_file_perm(dst) + # set the link attribute linktype = o.import_link if linktype == LinkTypes.LINK_CHILDREN and \ @@ -485,16 +488,23 @@ def cmd_importer(o): LOG.err('importing \"{}\" failed!'.format(path)) ret = False continue + if o.dry: LOG.dry('would copy {} to {}'.format(dst, srcf)) else: + # copy the file to the dotpath if os.path.isdir(dst): if os.path.exists(srcf): shutil.rmtree(srcf) shutil.copytree(dst, srcf) else: shutil.copy2(dst, srcf) - retconf = o.conf.new(src, dst, linktype) + + chmod = None + if o.import_mode or perm&o.umask: + # insert chmod + chmod = perm + retconf = o.conf.new(src, dst, linktype, chmod=chmod) if retconf: LOG.sub('\"{}\" imported'.format(path)) cnt += 1 diff --git a/dotdrop/dotfile.py b/dotdrop/dotfile.py index da7890d..fee2727 100644 --- a/dotdrop/dotfile.py +++ b/dotdrop/dotfile.py @@ -22,7 +22,7 @@ class Dotfile(DictParser): actions=[], trans_r=None, trans_w=None, link=LinkTypes.NOLINK, noempty=False, cmpignore=[], upignore=[], - instignore=[], template=True): + instignore=[], template=True, chmod=None): """ constructor @key: dotfile key @@ -37,6 +37,7 @@ class Dotfile(DictParser): @cmpignore: patterns to ignore when comparing @instignore: patterns to ignore when installing @template: template this dotfile + @chmod: file permission """ self.actions = actions self.dst = dst @@ -50,6 +51,7 @@ class Dotfile(DictParser): self.cmpignore = cmpignore self.instignore = instignore self.template = template + self.chmod = chmod if self.link != LinkTypes.NOLINK and \ ( @@ -113,6 +115,7 @@ class Dotfile(DictParser): msg += ', dst:\"{}\"'.format(self.dst) msg += ', link:\"{}\"'.format(str(self.link)) msg += ', template:{}'.format(self.template) + msg += ', chmod:{}'.format(self.chmod) return msg def prt(self): @@ -123,6 +126,7 @@ class Dotfile(DictParser): out += '\n{}dst: \"{}\"'.format(indent, self.dst) out += '\n{}link: \"{}\"'.format(indent, str(self.link)) out += '\n{}template: \"{}\"'.format(indent, str(self.template)) + out += '\n{}chmod: \"{}\"'.format(indent, str(self.chmod)) out += '\n{}pre-action:'.format(indent) some = self.get_pre_actions() diff --git a/dotdrop/options.py b/dotdrop/options.py index 8c549a1..3531b54 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -16,7 +16,7 @@ from dotdrop.linktypes import LinkTypes from dotdrop.logger import Logger from dotdrop.cfg_aggregator import CfgAggregator as Cfg from dotdrop.action import Action -from dotdrop.utils import uniq_list +from dotdrop.utils import uniq_list, get_umask from dotdrop.exceptions import YamlException ENV_PROFILE = 'DOTDROP_PROFILE' @@ -54,7 +54,7 @@ USAGE = """ Usage: dotdrop install [-VbtfndDa] [-c ] [-p ] [-w ] [...] - dotdrop import [-Vbdf] [-c ] [-p ] [-s ] + dotdrop import [-Vbdfm] [-c ] [-p ] [-s ] [-l ] ... dotdrop compare [-LVb] [-c ] [-p ] [-C ...] [-i ...] @@ -73,15 +73,16 @@ Options: -c --cfg= Path to the config. -C --file= Path of dotfile to compare. -d --dry Dry run. - -l --link= Link option (nolink|link|link_children). - -L --file-only Do not show diff but only the files that differ. - -p --profile= Specify the profile to use [default: {}]. -D --showdiff Show a diff before overwriting. -f --force Do not ask user confirmation for anything. -G --grepable Grepable output. -i --ignore= Pattern to ignore. -k --key Treat as a dotfile key. + -l --link= Link option (nolink|link|link_children). + -L --file-only Do not show diff but only the files that differ. + -m --preserve-mode Insert a chmod entry in the dotfile with its permissions. -n --nodiff Do not diff when installing. + -p --profile= Specify the profile to use [default: {}]. -P --show-patch Provide a one-liner to manually patch template. -s --as= Import as a different path from actual path. -t --temp Install to a temporary directory for review. @@ -121,6 +122,7 @@ class Options(AttrMonitor): self.log = Logger() self.debug = self.args['--verbose'] or ENV_DEBUG in os.environ self.dry = self.args['--dry'] + self.umask = get_umask() if ENV_NODEBUG in os.environ: # force disabling debugs self.debug = False @@ -261,6 +263,7 @@ class Options(AttrMonitor): # "import" specifics self.import_path = self.args[''] self.import_as = self.args['--as'] + self.import_mode = self.args['--preserve-mode'] # "update" specifics self.update_path = self.args[''] diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 7a67ac8..47acc9f 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -302,3 +302,15 @@ def mirror_file_rights(src, dst): """mirror file rights of src to dst (can rise exc)""" rights = os.stat(src).st_mode os.chmod(dst, rights) + + +def get_umask(): + """return current umask value""" + cur = os.umask(0) + os.umask(cur) + return 0o777-cur + + +def get_file_perm(path): + """return file permission""" + return os.stat(path).st_mode & 0o777 From 4517c3902cd56df18919ddc7d253ac9cf9a0ffd3 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 9 Nov 2020 15:16:53 +0100 Subject: [PATCH 02/82] fix cicd --- dotdrop/cfg_yaml.py | 6 +++--- dotdrop/dotdrop.py | 2 +- dotdrop/options.py | 2 +- dotdrop/utils.py | 2 +- tests/helpers.py | 1 + 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index c3c5e7a..a25ad7a 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -639,19 +639,19 @@ class CfgYaml: if len(val) < 3: err = 'bad format for chmod: {}'.format(val) self._log.err(err) - raise YamlException('dotfile chmod error: {}'.format(err)) + raise YamlException(err) try: int(val) except Exception: err = 'bad format for chmod: {}'.format(val) self._log.err(err) - raise YamlException('dotfile chmod error: {}'.format(err)) + raise YamlException(err) for x in val: y = int(x) if y < 0 or y > 7: err = 'bad format for chmod: {}'.format(val) self._log.err(err) - raise YamlException('dotfile chmod error: {}'.format(err)) + raise YamlException(err) return new diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 6b87ff6..0e22968 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -501,7 +501,7 @@ def cmd_importer(o): shutil.copy2(dst, srcf) chmod = None - if o.import_mode or perm&o.umask: + if o.import_mode or perm & o.umask: # insert chmod chmod = perm retconf = o.conf.new(src, dst, linktype, chmod=chmod) diff --git a/dotdrop/options.py b/dotdrop/options.py index 3531b54..0cbe1c6 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -80,7 +80,7 @@ Options: -k --key Treat as a dotfile key. -l --link= Link option (nolink|link|link_children). -L --file-only Do not show diff but only the files that differ. - -m --preserve-mode Insert a chmod entry in the dotfile with its permissions. + -m --preserve-mode Insert a chmod entry in the dotfile with its mode. -n --nodiff Do not diff when installing. -p --profile= Specify the profile to use [default: {}]. -P --show-patch Provide a one-liner to manually patch template. diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 47acc9f..bc32990 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -308,7 +308,7 @@ def get_umask(): """return current umask value""" cur = os.umask(0) os.umask(cur) - return 0o777-cur + return 0o777 - cur def get_file_perm(path): diff --git a/tests/helpers.py b/tests/helpers.py index 8abb198..83c4800 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -132,6 +132,7 @@ def _fake_args(): args['--as'] = None args['--file-only'] = False args['--workers'] = 1 + args['--preserve-mode'] = False # cmds args['profiles'] = False args['files'] = False From b5d8745fea1d996059dcb6108a10d324422e79ff Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 9 Nov 2020 21:08:27 +0100 Subject: [PATCH 03/82] adding chmod doc --- docs/config-format.md | 1 + docs/config.md | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/docs/config-format.md b/docs/config-format.md index 666e5a3..3cc1649 100644 --- a/docs/config-format.md +++ b/docs/config-format.md @@ -46,6 +46,7 @@ Entry | Description `src` | dotfile path within the `dotpath` (dotfile with empty `src` are ignored and considered installed, can use `variables` and `dynvariables`, make sure to quote) `link` | define how this dotfile is installed. Possible values: *nolink*, *link*, *link_children* (see [Symlinking dotfiles](config.md#symlink-dotfiles)) (defaults to value of `link_dotfile_default`) `actions` | list of action keys that need to be defined in the **actions** entry below (see [actions](config-details.md#entry-actions)) +`chmod` | defines the file permission in octal notation to apply during installation (see [permissions](config.md#permissions)) `cmpignore` | list of patterns to ignore when comparing (enclose in quotes when using wildcards, see [ignore patterns](config.md#ignore-patterns)) `ignoreempty` | if true empty template will not be deployed (defaults to value of `ignoreempty`) `instignore` | list of patterns to ignore when installing (enclose in quotes when using wildcards, see [ignore patterns](config.md#ignore-patterns)) diff --git a/docs/config.md b/docs/config.md index 46b7a0f..7cd2366 100644 --- a/docs/config.md +++ b/docs/config.md @@ -59,6 +59,42 @@ Here are some rules on the use of variables in configs: * external/imported `(dyn)variables` take precedence over `(dyn)variables` defined inside the main config file +## Permissions + +Dotfile allows to control the permission applied to a dotfile using the +config dotfile entry `chmod`. +A `chmod` entry on a directory is applied to the directory only, not recursively. + +On `import` the following rules are applied: + +* if the `-m --preserve-mode` switch is provided the imported file permissions are + stored in the `chmod` entry +* if imported file permissions differ from umask its permissions are automatically + stored in the `chmod` entry + +On `install` the following rules are applied: + +* if `chmod` is specified, it will be applied to the installed dotfile +* if file exists and its permissions differ from `umask` and no `chmod` is specified user needs + to confirm installation (unless `-f --force` is used) **TODO unless permissions match existing file** + +On `update`: + +* **TODO** + +One `compare`: + +* **TODO** + +Make sure to quote the `chmod` value in the config file: +```yaml +dotfiles: + f_xinitrc: + dst: ~/.xinitrc + src: xinitrc + chmod: '777' +``` + ## Symlink dotfiles Dotdrop is able to install dotfiles in three different ways From 2ac55f3f18adbeb9626e5791ebff2273977b2939 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 9 Nov 2020 21:08:53 +0100 Subject: [PATCH 04/82] add chmod --- dotdrop/cfg_aggregator.py | 6 +- dotdrop/cfg_yaml.py | 36 +++++++--- dotdrop/dotdrop.py | 19 ++++-- dotdrop/installer.py | 139 ++++++++++++++++++++++++++++---------- dotdrop/utils.py | 3 +- 5 files changed, 149 insertions(+), 54 deletions(-) diff --git a/dotdrop/cfg_aggregator.py b/dotdrop/cfg_aggregator.py index c83afb6..779f281 100644 --- a/dotdrop/cfg_aggregator.py +++ b/dotdrop/cfg_aggregator.py @@ -156,7 +156,8 @@ class CfgAggregator: if self.debug: self.log.dbg('new dotfile key: {}'.format(key)) # add the dotfile - self.cfgyaml.add_dotfile(key, src, dst, link, chmod=chmod) + if not self.cfgyaml.add_dotfile(key, src, dst, link, chmod=chmod): + return None return Dotfile(key, dst, src) def new(self, src, dst, link, chmod=None): @@ -172,6 +173,9 @@ class CfgAggregator: if not dotfile: dotfile = self._create_new_dotfile(src, dst, link, chmod=chmod) + if not dotfile: + return False + key = dotfile.key ret = self.cfgyaml.add_dotfile_to_profile(key, self.profile_key) if ret and self.debug: diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index a25ad7a..35e3551 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -337,13 +337,17 @@ class CfgYaml: dfl = self.settings[self.key_settings_link_dotfile_default] if str(link) != dfl: df_dict[self.key_dotfile_link] = str(link) + # chmod if chmod: - df_dict[self.key_dotfile_chmod] = format(chmod, 'o') + lnkval = df_dict.get(self.key_dotfile_link, None) + if lnkval != self.lnk_children: + df_dict[self.key_dotfile_chmod] = format(chmod, 'o') # add to global dict self._yaml_dict[self.key_dotfiles][key] = df_dict self._dirty = True + return True def del_dotfile(self, key): """remove this dotfile from config""" @@ -603,7 +607,7 @@ class CfgYaml: return new def _norm_dotfiles(self, dotfiles): - """normalize dotfiles entries""" + """normalize and check dotfiles entries""" if not dotfiles: return dotfiles new = {} @@ -621,7 +625,14 @@ class CfgYaml: v[self.key_trans_r] = v[self.old_key_trans_r] del v[self.old_key_trans_r] new[k] = v - if self.key_dotfile_link not in v: + if self.key_dotfile_link in v: + # validate link value + val = v[self.key_dotfile_link] + if val not in self.allowed_link_val: + err = 'bad link value: {}'.format(val) + self._log.err(err) + raise YamlException('config content error: {}'.format(err)) + else: # apply link value if undefined val = self.settings[self.key_settings_link_dotfile_default] v[self.key_dotfile_link] = val @@ -639,19 +650,24 @@ class CfgYaml: if len(val) < 3: err = 'bad format for chmod: {}'.format(val) self._log.err(err) - raise YamlException(err) + raise YamlException('config content error: {}'.format(err)) try: int(val) except Exception: err = 'bad format for chmod: {}'.format(val) self._log.err(err) - raise YamlException(err) - for x in val: + raise YamlException('config content error: {}'.format(err)) + for x in list(val): y = int(x) - if y < 0 or y > 7: - err = 'bad format for chmod: {}'.format(val) - self._log.err(err) - raise YamlException(err) + if y >= 0 or y <= 7: + continue + err = 'bad format for chmod: {}'.format(val) + self._log.err(err) + raise YamlException('config content error: {}'.format(err)) + if v[self.key_dotfile_link] == self.lnk_children: + err = 'incompatible use of chmod and link_children' + self._log.err(err) + raise YamlException('config content error: {}'.format(err)) return new diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 0e22968..b956f4b 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -101,7 +101,8 @@ def _dotfile_install(o, dotfile, tmpdir=None): # link r, err = inst.link(t, dotfile.src, dotfile.dst, actionexec=pre_actions_exec, - template=dotfile.template) + template=dotfile.template, + chmod=dotfile.chmod) elif hasattr(dotfile, 'link') and \ dotfile.link == LinkTypes.LINK_CHILDREN: # link_children @@ -123,7 +124,8 @@ def _dotfile_install(o, dotfile, tmpdir=None): actionexec=pre_actions_exec, noempty=dotfile.noempty, ignore=ignores, - template=dotfile.template) + template=dotfile.template, + chmod=dotfile.chmod) if tmp: tmp = os.path.join(o.dotpath, tmp) if os.path.exists(tmp): @@ -300,7 +302,8 @@ def cmd_compare(o, tmp): # install dotfile to temporary dir and compare ret, err, insttmp = inst.install_to_temp(t, tmp, src, dotfile.dst, - template=dotfile.template) + template=dotfile.template, + chmod=dotfile.chmod) if not ret: # failed to install to tmp line = '=> compare {}: error' @@ -339,6 +342,7 @@ def cmd_compare(o, tmp): def cmd_update(o): """update the dotfile(s) from path(s) or key(s)""" + # TODO chmod ret = True paths = o.update_path iskey = o.update_iskey @@ -689,8 +693,13 @@ def _get_templater(o): def _detail(dotpath, dotfile): """display details on all files under a dotfile entry""" - LOG.log('{} (dst: \"{}\", link: {})'.format(dotfile.key, dotfile.dst, - dotfile.link.name.lower())) + entry = '{}'.format(dotfile.key) + attribs = [] + attribs.append('dst: \"{}\"'.format(dotfile.dst)) + attribs.append('link: \"{}\"'.format(dotfile.link.name.lower())) + if dotfile.chmod: + attribs.append('chmod: \"{}\"'.format(dotfile.chmod)) + LOG.log('{} ({})'.format(entry, ', '.join(attribs))) path = os.path.join(dotpath, os.path.expanduser(dotfile.src)) if not os.path.isdir(path): template = 'no' diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 663ac6f..8f670dc 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -50,23 +50,13 @@ class Installer: self.diff_cmd = diff_cmd self.comparing = False self.action_executed = False + self.umask = utils.get_umask() self.log = Logger() - def _log_install(self, boolean, err): - if not self.debug: - return boolean, err - if boolean: - self.log.dbg('install: SUCCESS') - else: - if err: - self.log.dbg('install: ERROR: {}'.format(err)) - else: - self.log.dbg('install: IGNORED') - return boolean, err - def install(self, templater, src, dst, actionexec=None, noempty=False, - ignore=[], template=True): + ignore=[], template=True, + chmod=None): """ install src to dst using a template @templater: the templater object @@ -76,30 +66,50 @@ class Installer: @noempty: render empty template flag @ignore: pattern to ignore when installing @template: template this dotfile + @chmod: rights to apply if any return - True, None : success - False, error_msg : error - False, None : ignored """ + if not self._chmod_file(dst, chmod): + err = 'ignoring "{}", nothing installed'.format(dst) + return False, err + + r, err = self._install(templater, src, dst, + actionexec=actionexec, + noempty=noempty, + ignore=ignore, + template=template) + + if chmod: + os.chmod(dst, chmod) + + return self._log_install(r, err) + + def _install(self, templater, src, dst, + actionexec=None, noempty=False, + ignore=[], template=True): + """install link:nolink""" if self.debug: self.log.dbg('installing \"{}\" to \"{}\"'.format(src, dst)) if not dst or not src: if self.debug: self.log.dbg('empty dst for {}'.format(src)) - return self._log_install(True, None) + return True, None self.action_executed = False src = os.path.join(self.base, os.path.expanduser(src)) if not os.path.exists(src): err = 'source dotfile does not exist: {}'.format(src) - return self._log_install(False, err) + return False, err dst = os.path.expanduser(dst) if self.totemp: dst = self._pivot_path(dst, self.totemp) if utils.samefile(src, dst): # symlink loop err = 'dotfile points to itself: {}'.format(dst) - return self._log_install(False, err) + return False, err isdir = os.path.isdir(src) if self.debug: self.log.dbg('install {} to {}'.format(src, dst)) @@ -109,14 +119,15 @@ class Installer: actionexec=actionexec, noempty=noempty, ignore=ignore, template=template) - return self._log_install(b, e) + return b, e b, e = self._install_file(templater, src, dst, actionexec=actionexec, noempty=noempty, ignore=ignore, template=template) - return self._log_install(b, e) + return b, e - def link(self, templater, src, dst, actionexec=None, template=True): + def link(self, templater, src, dst, actionexec=None, + template=True, chmod=None): """ set src as the link target of dst @templater: the templater @@ -124,30 +135,45 @@ class Installer: @dst: dotfile destination path in the FS @actionexec: action executor callback @template: template this dotfile + @chmod: rights to apply if any return - True, None : success - False, error_msg : error - False, None : ignored """ + if not self._chmod_file(dst, chmod): + err = 'ignoring "{}", nothing installed'.format(dst) + return False, err + + r, err = self._link(templater, src, dst, + actionexec=actionexec, + template=template) + if chmod: + os.chmod(dst, chmod) + return self._log_install(r, err) + + def _link(self, templater, src, dst, actionexec=None, + template=True): + """install link:link""" if self.debug: self.log.dbg('link \"{}\" to \"{}\"'.format(src, dst)) if not dst or not src: if self.debug: self.log.dbg('empty dst for {}'.format(src)) - return self._log_install(True, None) + return True, None self.action_executed = False src = os.path.normpath(os.path.join(self.base, os.path.expanduser(src))) if not os.path.exists(src): err = 'source dotfile does not exist: {}'.format(src) - return self._log_install(False, err) + return False, err dst = os.path.normpath(os.path.expanduser(dst)) if self.totemp: # ignore actions b, e = self.install(templater, src, dst, actionexec=None, template=template) - return self._log_install(b, e) + return b, e if template and Templategen.is_template(src): if self.debug: @@ -157,10 +183,10 @@ class Installer: i, err = self.install(templater, src, tmp, actionexec=actionexec, template=template) if not i and not os.path.exists(tmp): - return self._log_install(i, err) + return i, err src = tmp - b, e = self._link(src, dst, actionexec=actionexec) - return self._log_install(b, e) + b, e = self._symlink(src, dst, actionexec=actionexec) + return b, e def link_children(self, templater, src, dst, actionexec=None, template=True): @@ -177,19 +203,27 @@ class Installer: - False, error_msg: error - False, None, ignored """ + r, err = self._link_children(templater, src, dst, + actionexec=actionexec, + template=template) + return self._log_install(r, err) + + def _link_children(self, templater, src, dst, actionexec=None, + template=True): + """install link:link_children""" if self.debug: self.log.dbg('link_children \"{}\" to \"{}\"'.format(src, dst)) if not dst or not src: if self.debug: self.log.dbg('empty dst for {}'.format(src)) - return self._log_install(True, None) + return True, None self.action_executed = False parent = os.path.join(self.base, os.path.expanduser(src)) # Fail if source doesn't exist if not os.path.exists(parent): err = 'source dotfile does not exist: {}'.format(parent) - return self._log_install(False, err) + return False, err # Fail if source not a directory if not os.path.isdir(parent): @@ -197,7 +231,7 @@ class Installer: self.log.dbg('symlink children of {} to {}'.format(src, dst)) err = 'source dotfile is not a directory: {}'.format(parent) - return self._log_install(False, err) + return False, err dst = os.path.normpath(os.path.expanduser(dst)) if not os.path.lexists(dst): @@ -212,7 +246,7 @@ class Installer: if self.safe and not self.log.ask(msg): err = 'ignoring "{}", nothing installed'.format(dst) - return self._log_install(False, err) + return False, err os.unlink(dst) os.mkdir(dst) @@ -242,7 +276,7 @@ class Installer: continue src = tmp - ret, err = self._link(src, dst, actionexec=actionexec) + ret, err = self._symlink(src, dst, actionexec=actionexec) if ret: installed += 1 # void actionexec if dotfile installed @@ -250,11 +284,11 @@ class Installer: actionexec = None else: if err: - return self._log_install(ret, err) + return ret, err - return self._log_install(installed > 0, None) + return installed > 0, None - def _link(self, src, dst, actionexec=None): + def _symlink(self, src, dst, actionexec=None): """ set src as a link target of dst @@ -595,13 +629,16 @@ class Installer: self.action_executed = True return ret, err - def _install_to_temp(self, templater, src, dst, tmpdir, template=True): + def _install_to_temp(self, templater, src, dst, tmpdir, + template=True, chmod=None): """install a dotfile to a tempdir""" tmpdst = self._pivot_path(dst, tmpdir) - r = self.install(templater, src, tmpdst, template=template) + r = self.install(templater, src, tmpdst, + template=template, chmod=chmod) return r, tmpdst - def install_to_temp(self, templater, tmpdir, src, dst, template=True): + def install_to_temp(self, templater, tmpdir, src, dst, + template=True, chmod=None): """install a dotfile to a tempdir""" ret = False tmpdst = '' @@ -620,7 +657,7 @@ class Installer: self.log.dbg('tmp install {} (defined dst: {})'.format(src, dst)) # install the dotfile to a temp directory for comparing r, tmpdst = self._install_to_temp(templater, src, dst, tmpdir, - template=template) + template=template, chmod=chmod) ret, err = r if self.debug: self.log.dbg('tmp installed in {}'.format(tmpdst)) @@ -630,3 +667,31 @@ class Installer: self.comparing = False self.create = createsaved return ret, err, tmpdst + + def _log_install(self, boolean, err): + """log installation process""" + if not self.debug: + return boolean, err + if boolean: + self.log.dbg('install: SUCCESS') + else: + if err: + self.log.dbg('install: ERROR: {}'.format(err)) + else: + self.log.dbg('install: IGNORED') + return boolean, err + + def _chmod_file(self, path, chmod): + """chmod file if needed and return True to continue""" + if chmod: + return True + if os.path.exists(path) and self.safe: + perms = utils.get_file_perm(path) + if perms & self.umask: + # perms and umask differ + msg = 'File mode ({}) differs from umask ({})' + msg.format(perms, self.umask) + msg += ', continue' + if not self.log.ask(msg): + return False + return True diff --git a/dotdrop/utils.py b/dotdrop/utils.py index bc32990..95e1539 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -308,7 +308,8 @@ def get_umask(): """return current umask value""" cur = os.umask(0) os.umask(cur) - return 0o777 - cur + # return 0o777 - cur + return cur def get_file_perm(path): From 8fd06937215813b07e891fdef73e9e01a17f8e24 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 9 Nov 2020 21:09:00 +0100 Subject: [PATCH 05/82] fix test --- tests-ng/corner-case.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests-ng/corner-case.sh b/tests-ng/corner-case.sh index bea39c1..9c4b037 100755 --- a/tests-ng/corner-case.sh +++ b/tests-ng/corner-case.sh @@ -89,7 +89,7 @@ cd ${ddpath} | ${bin} install -D -c ${cfg} -p p1 --verbose f_x [ "$?" != "0" ] && exit 1 echo "[+] test install not existing src" -cd ${ddpath} | ${bin} install -c ${cfg} --dry -p p1 --verbose f_y +cd ${ddpath} | ${bin} install -c ${cfg} -f --dry -p p1 --verbose f_y echo "[+] test install to temp" cd ${ddpath} | ${bin} install -t -c ${cfg} -p p1 --verbose f_x From 4e69f47ec8c3200a7db8161d1530f3ee85f4d23e Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 9 Nov 2020 21:09:08 +0100 Subject: [PATCH 06/82] add chmod import tests --- tests-ng/chmod-import.sh | 203 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100755 tests-ng/chmod-import.sh diff --git a/tests-ng/chmod-import.sh b/tests-ng/chmod-import.sh new file mode 100755 index 0000000..5845f90 --- /dev/null +++ b/tests-ng/chmod-import.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2020, deadc0de6 +# +# test chmod on import +# with files and directories +# with different link +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +#echo "dotfile destination: ${tmpd}" + +# create the dotfile +dnormal="${tmpd}/dir_normal" +mkdir -p ${dnormal} +echo "dir_normal/f1" > ${dnormal}/file1 +echo "dir_normal/f2" > ${dnormal}/file2 +chmod 777 ${dnormal} + +dlink="${tmpd}/dir_link" +mkdir -p ${dlink} +echo "dir_link/f1" > ${dlink}/file1 +echo "dir_link/f2" > ${dlink}/file2 +chmod 777 ${dlink} + +dlinkchildren="${tmpd}/dir_link_children" +mkdir -p ${dlinkchildren} +echo "dir_linkchildren/f1" > ${dlinkchildren}/file1 +echo "dir_linkchildren/f2" > ${dlinkchildren}/file2 +chmod 777 ${dlinkchildren} + +fnormal="${tmpd}/filenormal" +echo "filenormal" > ${fnormal} +chmod 777 ${fnormal} + +flink="${tmpd}/filelink" +echo "filelink" > ${flink} +chmod 777 ${flink} + +toimport="${dnormal} ${dlink} ${dlinkchildren} ${fnormal} ${flink}" + +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: +profiles: +_EOF +#cat ${cfg} + +# import without --preserve-mode +for i in ${toimport}; do + cd ${ddpath} | ${bin} import -c ${cfg} -f -p p1 -V ${i} +done + +cat ${cfg} + +# list files +cd ${ddpath} | ${bin} detail -c ${cfg} -p p1 -V + +tot=`echo ${toimport} | wc -w` +cnt=`cat ${cfg} | grep chmod | wc -l` +[ "${cnt}" != "${tot}" ] && echo "not all chmod inserted" && exit 1 + +## with link +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: +profiles: +_EOF + +# clean +rm -rf ${tmps}/dotfiles +mkdir -p ${tmps}/dotfiles + +# import without --preserve-mode and link + +for i in ${toimport}; do + cd ${ddpath} | ${bin} import -c ${cfg} -l link -f -p p1 -V ${i} +done + +cat ${cfg} + +# list files +cd ${ddpath} | ${bin} detail -c ${cfg} -p p1 -V + +tot=`echo ${toimport} | wc -w` +cnt=`cat ${cfg} | grep chmod | wc -l` +[ "${cnt}" != "${tot}" ] && echo "not all chmod inserted" && exit 1 + +## --preserve-mode +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: +profiles: +_EOF + +# clean +rm -rf ${tmps}/dotfiles +mkdir -p ${tmps}/dotfiles + +# import with --preserve-mode +for i in ${toimport}; do + chmod `umask -S` ${i} + cd ${ddpath} | ${bin} import -c ${cfg} -m -f -p p1 -V ${i} +done + +cat ${cfg} + +# list files +cd ${ddpath} | ${bin} detail -c ${cfg} -p p1 -V + +tot=`echo ${toimport} | wc -w` +cnt=`cat ${cfg} | grep chmod | wc -l` +[ "${cnt}" != "${tot}" ] && echo "not all chmod inserted" && exit 1 + +## import normal +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: +profiles: +_EOF + +# clean +rm -rf ${tmps}/dotfiles +mkdir -p ${tmps}/dotfiles + +# import with --preserve-mode +for i in ${toimport}; do + chmod `umask -S` ${i} + cd ${ddpath} | ${bin} import -c ${cfg} -f -p p1 -V ${i} +done + +cat ${cfg} + +# list files +cd ${ddpath} | ${bin} detail -c ${cfg} -p p1 -V + +cnt=`cat ${cfg} | grep chmod | wc -l` +[ "${cnt}" != "0" ] && echo "chmod inserted but not needed" && exit 1 + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 From 73bbfeef04af9b26286275eb7c4974d3f507da1c Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 9 Nov 2020 21:20:36 +0100 Subject: [PATCH 07/82] fix link value --- dotdrop/cfg_yaml.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 35e3551..75b8cbc 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -625,14 +625,7 @@ class CfgYaml: v[self.key_trans_r] = v[self.old_key_trans_r] del v[self.old_key_trans_r] new[k] = v - if self.key_dotfile_link in v: - # validate link value - val = v[self.key_dotfile_link] - if val not in self.allowed_link_val: - err = 'bad link value: {}'.format(val) - self._log.err(err) - raise YamlException('config content error: {}'.format(err)) - else: + if self.key_dotfile_link not in v: # apply link value if undefined val = self.settings[self.key_settings_link_dotfile_default] v[self.key_dotfile_link] = val From 1f61699ca437ae374b6098941e172e3fa998fa21 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 9 Nov 2020 21:56:13 +0100 Subject: [PATCH 08/82] fix chmod install --- dotdrop/cfg_yaml.py | 5 +++-- dotdrop/installer.py | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 75b8cbc..432f3a4 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -342,7 +342,7 @@ class CfgYaml: if chmod: lnkval = df_dict.get(self.key_dotfile_link, None) if lnkval != self.lnk_children: - df_dict[self.key_dotfile_chmod] = format(chmod, 'o') + df_dict[self.key_dotfile_chmod] = str(format(chmod, 'o')) # add to global dict self._yaml_dict[self.key_dotfiles][key] = df_dict @@ -639,7 +639,7 @@ class CfgYaml: v[self.key_dotfile_template] = val # validate value of chmod if defined if self.key_dotfile_chmod in v: - val = v[self.key_dotfile_chmod] + val = str(v[self.key_dotfile_chmod]) if len(val) < 3: err = 'bad format for chmod: {}'.format(val) self._log.err(err) @@ -661,6 +661,7 @@ class CfgYaml: err = 'incompatible use of chmod and link_children' self._log.err(err) raise YamlException('config content error: {}'.format(err)) + v[self.key_dotfile_chmod] = val return new diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 8f670dc..5c520e1 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -73,7 +73,7 @@ class Installer: - False, error_msg : error - False, None : ignored """ - if not self._chmod_file(dst, chmod): + if not self._check_chmod(dst, chmod): err = 'ignoring "{}", nothing installed'.format(dst) return False, err @@ -83,8 +83,8 @@ class Installer: ignore=ignore, template=template) - if chmod: - os.chmod(dst, chmod) + if r and chmod: + os.chmod(dst, int(chmod, 8)) return self._log_install(r, err) @@ -142,7 +142,7 @@ class Installer: - False, error_msg : error - False, None : ignored """ - if not self._chmod_file(dst, chmod): + if not self._check_chmod(dst, chmod): err = 'ignoring "{}", nothing installed'.format(dst) return False, err @@ -150,7 +150,7 @@ class Installer: actionexec=actionexec, template=template) if chmod: - os.chmod(dst, chmod) + os.chmod(dst, int(chmod, 8)) return self._log_install(r, err) def _link(self, templater, src, dst, actionexec=None, @@ -681,7 +681,7 @@ class Installer: self.log.dbg('install: IGNORED') return boolean, err - def _chmod_file(self, path, chmod): + def _check_chmod(self, path, chmod): """chmod file if needed and return True to continue""" if chmod: return True From c608a8e868733bff3735f566fa85d40ebe8208d8 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 9 Nov 2020 21:56:21 +0100 Subject: [PATCH 09/82] add chmod install tests --- tests-ng/chmod-install.sh | 109 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100755 tests-ng/chmod-install.sh diff --git a/tests-ng/chmod-install.sh b/tests-ng/chmod-install.sh new file mode 100755 index 0000000..71782d5 --- /dev/null +++ b/tests-ng/chmod-install.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2020, deadc0de6 +# +# test chmod on install +# with files and directories +# with different link +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +#echo "dotfile destination: ${tmpd}" + +# create the config file +cfg="${tmps}/config.yaml" + +echo 'f777' > ${tmps}/dotfiles/f777 +echo 'link' > ${tmps}/dotfiles/link +mkdir -p ${tmps}/dotfiles/dir +echo "f1" > ${tmps}/dotfiles/dir/f1 + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: + f_f777: + src: f777 + dst: ${tmpd}/f777 + chmod: 777 + f_link: + src: link + dst: ${tmpd}/link + chmod: 777 + link: link + d_dir: + src: dir + dst: ${tmpd}/dir + chmod: 777 +profiles: + p1: + dotfiles: + - f_f777 + - f_link + - d_dir +_EOF +#cat ${cfg} + +# install +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 -V ${i} + +mode=`stat -c '%a' "${tmpd}/f777"` +[ "${mode}" != "777" ] && echo "bad mode for f777" && exit 1 + +mode=`stat -c '%a' "${tmpd}/link"` +[ "${mode}" != "777" ] && echo "bad mode for link" && exit 1 + +mode=`stat -c '%a' "${tmpd}/dir"` +[ "${mode}" != "777" ] && echo "bad mode for dir" && exit 1 + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 From 500db8d0db4919d0201d1aab7ef8926a905535ed Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 9 Nov 2020 22:24:21 +0100 Subject: [PATCH 10/82] more tests --- tests-ng/chmod-install.sh | 47 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests-ng/chmod-install.sh b/tests-ng/chmod-install.sh index 71782d5..c4b94b2 100755 --- a/tests-ng/chmod-install.sh +++ b/tests-ng/chmod-install.sh @@ -6,6 +6,9 @@ # with files and directories # with different link # +# TODO +# - test for symlink templates +# - check for mode difference when install # exit on first error set -e @@ -62,6 +65,14 @@ echo 'link' > ${tmps}/dotfiles/link mkdir -p ${tmps}/dotfiles/dir echo "f1" > ${tmps}/dotfiles/dir/f1 +echo "exists" > ${tmps}/dotfiles/exists +chmod 644 ${tmps}/dotfiles/exists +echo "exists" > ${tmpd}/exists +chmod 644 ${tmpd}/exists + +echo "existslink" > ${tmps}/dotfiles/existslink +chmod 644 ${tmpd}/exists + cat > ${cfg} << _EOF config: backup: true @@ -81,12 +92,27 @@ dotfiles: src: dir dst: ${tmpd}/dir chmod: 777 + f_exists: + src: exists + dst: ${tmpd}/exists + chmod: 777 + f_existslink: + src: existslink + dst: ${tmpd}/existslink + chmod: 777 + link: link profiles: p1: dotfiles: - f_f777 - f_link - d_dir + - f_exists + - f_existslink + p2: + dotfiles: + - f_exists + - f_existslink _EOF #cat ${cfg} @@ -102,6 +128,27 @@ mode=`stat -c '%a' "${tmpd}/link"` mode=`stat -c '%a' "${tmpd}/dir"` [ "${mode}" != "777" ] && echo "bad mode for dir" && exit 1 +mode=`stat -c '%a' "${tmpd}/exists"` +[ "${mode}" != "777" ] && echo "bad mode for exists" && exit 1 + +mode=`stat -c '%a' "${tmpd}/existslink"` +[ "${mode}" != "777" ] && echo "bad mode for existslink" && exit 1 + +echo "exists" > ${tmps}/dotfiles/exists +chmod 644 ${tmps}/dotfiles/exists +echo "exists" > ${tmpd}/exists +chmod 644 ${tmpd}/exists + +chmod 644 ${tmpd}/existslink + +cd ${ddpath} | ${bin} install -c ${cfg} -p p2 -V ${i} + +mode=`stat -c '%a' "${tmpd}/exists"` +[ "${mode}" != "777" ] && echo "bad mode for exists" && exit 1 + +mode=`stat -c '%a' "${tmpd}/existslink"` +[ "${mode}" != "777" ] && echo "bad mode for existslink" && exit 1 + ## CLEANING rm -rf ${tmps} ${tmpd} From c07a865809cef0d8fb35cd10f91e99d2f81dc557 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 9 Nov 2020 22:25:57 +0100 Subject: [PATCH 11/82] update doc --- docs/config.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docs/config.md b/docs/config.md index 7cd2366..9372285 100644 --- a/docs/config.md +++ b/docs/config.md @@ -86,15 +86,6 @@ One `compare`: * **TODO** -Make sure to quote the `chmod` value in the config file: -```yaml -dotfiles: - f_xinitrc: - dst: ~/.xinitrc - src: xinitrc - chmod: '777' -``` - ## Symlink dotfiles Dotdrop is able to install dotfiles in three different ways From 2bb684d3da1643526309e4a1aacad2839338220d Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 12 Nov 2020 22:24:09 +0100 Subject: [PATCH 12/82] modes install and compare --- docs/config.md | 6 +- dotdrop/cfg_yaml.py | 4 - dotdrop/comparator.py | 26 +++++- dotdrop/dotdrop.py | 4 +- dotdrop/installer.py | 182 +++++++++++++++++++++++--------------- dotdrop/utils.py | 7 ++ tests-ng/chmod-compare.sh | 121 +++++++++++++++++++++++++ tests-ng/chmod-install.sh | 107 +++++++++++++++++----- 8 files changed, 349 insertions(+), 108 deletions(-) create mode 100755 tests-ng/chmod-compare.sh diff --git a/docs/config.md b/docs/config.md index 9372285..a5f0c94 100644 --- a/docs/config.md +++ b/docs/config.md @@ -76,16 +76,12 @@ On `install` the following rules are applied: * if `chmod` is specified, it will be applied to the installed dotfile * if file exists and its permissions differ from `umask` and no `chmod` is specified user needs - to confirm installation (unless `-f --force` is used) **TODO unless permissions match existing file** + to confirm installation (unless `-f --force` is used or the permission match the dotfile in the `dotpath`) On `update`: * **TODO** -One `compare`: - -* **TODO** - ## Symlink dotfiles Dotdrop is able to install dotfiles in three different ways diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 432f3a4..663d06e 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -657,10 +657,6 @@ class CfgYaml: err = 'bad format for chmod: {}'.format(val) self._log.err(err) raise YamlException('config content error: {}'.format(err)) - if v[self.key_dotfile_link] == self.lnk_children: - err = 'incompatible use of chmod and link_children' - self._log.err(err) - raise YamlException('config content error: {}'.format(err)) v[self.key_dotfile_chmod] = val return new diff --git a/dotdrop/comparator.py b/dotdrop/comparator.py index f29351f..9af3054 100644 --- a/dotdrop/comparator.py +++ b/dotdrop/comparator.py @@ -10,7 +10,8 @@ import filecmp # local imports from dotdrop.logger import Logger -from dotdrop.utils import must_ignore, uniq_list, diff +from dotdrop.utils import must_ignore, uniq_list, diff, \ + get_file_perm class Comparator: @@ -31,6 +32,7 @@ class Comparator: if self.debug: self.log.dbg('comparing {} and {}'.format(left, right)) self.log.dbg('ignore pattern(s): {}'.format(ignore)) + # test type of file if os.path.isdir(left) and not os.path.isdir(right): return '\"{}\" is a dir while \"{}\" is a file\n'.format(left, @@ -38,14 +40,32 @@ class Comparator: if not os.path.isdir(left) and os.path.isdir(right): return '\"{}\" is a file while \"{}\" is a dir\n'.format(left, right) + # test content if not os.path.isdir(left): if self.debug: self.log.dbg('is file') - return self._comp_file(left, right, ignore) + ret = self._comp_file(left, right, ignore) + if not ret: + ret = self._comp_mode(left, right) + return ret + if self.debug: self.log.dbg('is directory') - return self._comp_dir(left, right, ignore) + + ret = self._comp_dir(left, right, ignore) + if not ret: + ret = self._comp_mode(left, right) + return ret + + def _comp_mode(self, left, right): + """compare mode""" + left_mode = get_file_perm(left) + right_mode = get_file_perm(right) + if left_mode == right_mode: + return '' + ret = 'modes differ ({} vs {})\n'.format(left_mode, right_mode) + return ret def _comp_file(self, left, right, ignore): """compare a file""" diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index b956f4b..6bb509f 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -108,7 +108,8 @@ def _dotfile_install(o, dotfile, tmpdir=None): # link_children r, err = inst.link_children(t, dotfile.src, dotfile.dst, actionexec=pre_actions_exec, - template=dotfile.template) + template=dotfile.template, + chmod=dotfile.chmod) else: # nolink src = dotfile.src @@ -342,7 +343,6 @@ def cmd_compare(o, tmp): def cmd_update(o): """update the dotfile(s) from path(s) or key(s)""" - # TODO chmod ret = True paths = o.update_path iskey = o.update_iskey diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 5c520e1..10bed19 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -73,36 +73,41 @@ class Installer: - False, error_msg : error - False, None : ignored """ - if not self._check_chmod(dst, chmod): - err = 'ignoring "{}", nothing installed'.format(dst) + if not self._check_chmod(src, dst, chmod): + err = 'ignoring "{}", not installed'.format(dst) return False, err r, err = self._install(templater, src, dst, actionexec=actionexec, noempty=noempty, ignore=ignore, - template=template) - - if r and chmod: - os.chmod(dst, int(chmod, 8)) + template=template, + chmod=chmod) return self._log_install(r, err) def _install(self, templater, src, dst, actionexec=None, noempty=False, - ignore=[], template=True): + ignore=[], template=True, + chmod=None): """install link:nolink""" if self.debug: self.log.dbg('installing \"{}\" to \"{}\"'.format(src, dst)) + # check dst exists in config if not dst or not src: if self.debug: self.log.dbg('empty dst for {}'.format(src)) return True, None + self.action_executed = False + + # check source file exists src = os.path.join(self.base, os.path.expanduser(src)) if not os.path.exists(src): err = 'source dotfile does not exist: {}'.format(src) return False, err + + # check dst is valid dst = os.path.expanduser(dst) if self.totemp: dst = self._pivot_path(dst, self.totemp) @@ -110,20 +115,23 @@ class Installer: # symlink loop err = 'dotfile points to itself: {}'.format(dst) return False, err + isdir = os.path.isdir(src) if self.debug: self.log.dbg('install {} to {}'.format(src, dst)) - self.log.dbg('is a directory \"{}\": {}'.format(src, isdir)) + self.log.dbg('\"{}\" is a directory: {}'.format(src, isdir)) if isdir: - b, e = self._install_dir(templater, src, dst, - actionexec=actionexec, - noempty=noempty, ignore=ignore, - template=template) + b, e = self._deploy_dir(templater, src, dst, + actionexec=actionexec, + noempty=noempty, ignore=ignore, + template=template, + chmod=chmod) return b, e - b, e = self._install_file(templater, src, dst, - actionexec=actionexec, - noempty=noempty, ignore=ignore, - template=template) + b, e = self._deploy_file(templater, src, dst, + actionexec=actionexec, + noempty=noempty, ignore=ignore, + template=template, + chmod=chmod) return b, e def link(self, templater, src, dst, actionexec=None, @@ -142,15 +150,16 @@ class Installer: - False, error_msg : error - False, None : ignored """ - if not self._check_chmod(dst, chmod): + if not self._check_chmod(src, dst, chmod): err = 'ignoring "{}", nothing installed'.format(dst) return False, err r, err = self._link(templater, src, dst, actionexec=actionexec, template=template) - if chmod: - os.chmod(dst, int(chmod, 8)) + if chmod and not self.dry: + # apply mode + utils.chmod(dst, chmod, self.debug) return self._log_install(r, err) def _link(self, templater, src, dst, actionexec=None, @@ -189,7 +198,7 @@ class Installer: return b, e def link_children(self, templater, src, dst, actionexec=None, - template=True): + template=True, chmod=None): """ link all files under a given directory @templater: the templater @@ -197,19 +206,23 @@ class Installer: @dst: dotfile destination path in the FS @actionexec: action executor callback @template: template this dotfile + @chmod: file mode to apply return - True, None: success - False, error_msg: error - False, None, ignored """ + if not self._check_chmod(src, dst, chmod): + err = 'ignoring "{}", nothing installed'.format(dst) + return False, err r, err = self._link_children(templater, src, dst, actionexec=actionexec, - template=template) + template=template, chmod=chmod) return self._log_install(r, err) def _link_children(self, templater, src, dst, actionexec=None, - template=True): + template=True, chmod=None): """install link:link_children""" if self.debug: self.log.dbg('link_children \"{}\" to \"{}\"'.format(src, dst)) @@ -245,7 +258,7 @@ class Installer: ]).format(dst) if self.safe and not self.log.ask(msg): - err = 'ignoring "{}", nothing installed'.format(dst) + err = 'ignoring "{}", not installed'.format(dst) return False, err os.unlink(dst) os.mkdir(dst) @@ -258,25 +271,26 @@ class Installer: installed = 0 for i in range(len(children)): - src = srcs[i] - dst = dsts[i] + subsrc = srcs[i] + subdst = dsts[i] if self.debug: - self.log.dbg('symlink child {} to {}'.format(src, dst)) + self.log.dbg('symlink child {} to {}'.format(subsrc, subdst)) - if template and Templategen.is_template(src): + if template and Templategen.is_template(subsrc): if self.debug: self.log.dbg('dotfile is a template') self.log.dbg('install to {} and symlink' .format(self.workdir)) - tmp = self._pivot_path(dst, self.workdir, striphome=True) - r, e = self.install(templater, src, tmp, actionexec=actionexec, + tmp = self._pivot_path(subdst, self.workdir, striphome=True) + r, e = self.install(templater, subsrc, tmp, + actionexec=actionexec, template=template) if not r and e and not os.path.exists(tmp): continue - src = tmp + subsrc = tmp - ret, err = self._symlink(src, dst, actionexec=actionexec) + ret, err = self._symlink(subsrc, subdst, actionexec=actionexec) if ret: installed += 1 # void actionexec if dotfile installed @@ -286,6 +300,10 @@ class Installer: if err: return ret, err + if chmod and not self.dry: + # apply mode + utils.chmod(dst, chmod, debug=self.debug) + return installed > 0, None def _symlink(self, src, dst, actionexec=None): @@ -350,9 +368,10 @@ class Installer: tmp['_dotfile_sub_abs_dst'] = dst return tmp - def _install_file(self, templater, src, dst, - actionexec=None, noempty=False, - ignore=[], template=True): + def _deploy_file(self, templater, src, dst, + actionexec=None, noempty=False, + ignore=[], template=True, + chmod=None): """install src to dst when is a file""" if self.debug: self.log.dbg('deploy file: {}'.format(src)) @@ -367,7 +386,7 @@ class Installer: return False, None if utils.samefile(src, dst): - # symlink loop + # loop err = 'dotfile points to itself: {}'.format(dst) return False, err @@ -386,6 +405,7 @@ class Installer: return False, str(e) finally: templater.restore_vars(saved) + # test is empty if noempty and utils.content_empty(content): if self.debug: self.log.dbg('ignoring empty template: {}'.format(src)) @@ -393,10 +413,13 @@ class Installer: if content is None: err = 'empty template {}'.format(src) return False, err + + # write the file ret, err = self._write(src, dst, content=content, actionexec=actionexec, - template=template) + template=template, + chmod=chmod) # build return values if ret < 0: @@ -416,29 +439,36 @@ class Installer: err = 'installing {} to {}'.format(src, dst) return False, err - def _install_dir(self, templater, src, dst, - actionexec=None, noempty=False, - ignore=[], template=True): + def _deploy_dir(self, templater, src, dst, + actionexec=None, noempty=False, + ignore=[], template=True, chmod=None): """install src to dst when is a directory""" if self.debug: - self.log.dbg('install dir {}'.format(src)) - self.log.dbg('ignore empty: {}'.format(noempty)) + self.log.dbg('deploy dir {}'.format(src)) # default to nothing installed and no error ret = False, None + + # create the directory anyway + if self.debug: + self.log.dbg('mkdir -p {}'.format(dst)) if not self._create_dirs(dst): err = 'creating directory for {}'.format(dst) return False, err + # handle all files in dir for entry in os.listdir(src): f = os.path.join(src, entry) + if self.debug: + self.log.dbg('deploy sub from {}: {}'.format(dst, entry)) if not os.path.isdir(f): # is file - res, err = self._install_file(templater, f, - os.path.join(dst, entry), - actionexec=actionexec, - noempty=noempty, - ignore=ignore, - template=template) + res, err = self._deploy_file(templater, f, + os.path.join(dst, entry), + actionexec=actionexec, + noempty=noempty, + ignore=ignore, + template=template, + chmod=None) if not res and err: # error occured ret = res, err @@ -448,12 +478,13 @@ class Installer: ret = True, None else: # is directory - res, err = self._install_dir(templater, f, - os.path.join(dst, entry), - actionexec=actionexec, - noempty=noempty, - ignore=ignore, - template=template) + res, err = self._deploy_dir(templater, f, + os.path.join(dst, entry), + actionexec=actionexec, + noempty=noempty, + ignore=ignore, + template=template, + chmod=None) if not res and err: # error occured ret = res, err @@ -461,20 +492,15 @@ class Installer: elif res: # something got installed ret = True, None + + if chmod and not self.dry: + # apply mode + utils.chmod(dst, chmod, debug=self.debug) return ret - def _fake_diff(self, dst, content): - """ - fake diff by comparing file content with content - returns True if same - """ - cur = '' - with open(dst, 'br') as f: - cur = f.read() - return cur == content - def _write(self, src, dst, content=None, - actionexec=None, template=True): + actionexec=None, template=True, + chmod=None): """ copy dotfile / write content to file return 0, None: for success, @@ -487,11 +513,16 @@ class Installer: if self.dry: self.log.dry('would install {}'.format(dst)) return 0, None + if os.path.lexists(dst): - rights = os.stat(src).st_mode + if chmod: + rights = chmod + else: + rights = utils.get_file_perm(src) samerights = False try: - samerights = os.stat(dst).st_mode == rights + dstrights = utils.get_file_perm(dst) + samerights = dstrights == rights except OSError as e: if e.errno == errno.ENOENT: # broken symlink @@ -531,7 +562,7 @@ class Installer: if not r: return -1, e if self.debug: - self.log.dbg('install dotfile to \"{}\"'.format(dst)) + self.log.dbg('install file to \"{}\"'.format(dst)) # re-check in case action created the file if self.safe and not overwrite and os.path.lexists(dst): if not self.log.ask('Overwrite \"{}\"'.format(dst)): @@ -556,6 +587,9 @@ class Installer: shutil.copymode(src, dst) except Exception as e: return -1, str(e) + + if chmod: + utils.chmod(dst, chmod, debug=self.debug) return 0, None def _diff_before_write(self, src, dst, content=None, quiet=False): @@ -595,6 +629,7 @@ class Installer: return True if self.debug: self.log.dbg('mkdir -p {}'.format(directory)) + os.makedirs(directory) return os.path.exists(directory) @@ -681,12 +716,19 @@ class Installer: self.log.dbg('install: IGNORED') return boolean, err - def _check_chmod(self, path, chmod): + def _check_chmod(self, src, dst, chmod): """chmod file if needed and return True to continue""" if chmod: return True - if os.path.exists(path) and self.safe: - perms = utils.get_file_perm(path) + if not os.path.exists(dst): + return True + if not os.path.exists(src): + return True + cperms = utils.get_file_perm(src) + perms = utils.get_file_perm(dst) + if perms == cperms: + return True + if self.safe: if perms & self.umask: # perms and umask differ msg = 'File mode ({}) differs from umask ({})' diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 95e1539..a2fdba0 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -315,3 +315,10 @@ def get_umask(): def get_file_perm(path): """return file permission""" return os.stat(path).st_mode & 0o777 + + +def chmod(path, mode, debug=False): + cm = int(mode, 8) + if debug: + LOG.dbg('chmod {} {}'.format(oct(cm), path)) + os.chmod(path, cm) diff --git a/tests-ng/chmod-compare.sh b/tests-ng/chmod-compare.sh new file mode 100755 index 0000000..c8e77e2 --- /dev/null +++ b/tests-ng/chmod-compare.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2020, deadc0de6 +# +# test chmod on compare +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +#echo "dotfile destination: ${tmpd}" + +# create the dotfile +dnormal="${tmpd}/dir_normal" +mkdir -p ${dnormal} +echo "dir_normal/f1" > ${dnormal}/file1 +echo "dir_normal/f2" > ${dnormal}/file2 +chmod 777 ${dnormal} + +dlink="${tmpd}/dir_link" +mkdir -p ${dlink} +echo "dir_link/f1" > ${dlink}/file1 +echo "dir_link/f2" > ${dlink}/file2 +chmod 777 ${dlink} + +dlinkchildren="${tmpd}/dir_link_children" +mkdir -p ${dlinkchildren} +echo "dir_linkchildren/f1" > ${dlinkchildren}/file1 +echo "dir_linkchildren/f2" > ${dlinkchildren}/file2 +chmod 777 ${dlinkchildren} + +fnormal="${tmpd}/filenormal" +echo "filenormal" > ${fnormal} +chmod 777 ${fnormal} + +flink="${tmpd}/filelink" +echo "filelink" > ${flink} +chmod 777 ${flink} + +toimport="${dnormal} ${dlink} ${dlinkchildren} ${fnormal} ${flink}" + +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: +profiles: +_EOF +#cat ${cfg} + +# import without --preserve-mode +for i in ${toimport}; do + cd ${ddpath} | ${bin} import -c ${cfg} -f -p p1 ${i} +done + +#cat ${cfg} + +# patch rights +chmod 700 ${dnormal} +chmod 700 ${dlink} +chmod 700 ${dlinkchildren} +chmod 700 ${fnormal} +chmod 700 ${flink} + +set +e +cnt=`cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 2>&1 | grep 'modes differ' | wc -l` +set -e + +[ "${cnt}" != "5" ] && echo "compare modes failed" && exit 1 + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 diff --git a/tests-ng/chmod-install.sh b/tests-ng/chmod-install.sh index c4b94b2..05ff2a8 100755 --- a/tests-ng/chmod-install.sh +++ b/tests-ng/chmod-install.sh @@ -6,13 +6,21 @@ # with files and directories # with different link # -# TODO -# - test for symlink templates -# - check for mode difference when install # exit on first error set -e +# $1 path +# $2 rights +has_rights() +{ + echo "testing ${1} is ${2}" + [ ! -e "$1" ] && echo "`basename $1` does not exist" && exit 1 + local mode=`stat -L -c '%a' "$1"` + [ "${mode}" != "$2" ] && echo "bad mode for `basename $1`" && exit 1 + true +} + # all this crap to get current path rl="readlink -f" if ! ${rl} "${0}" >/dev/null 2>&1; then @@ -73,6 +81,23 @@ chmod 644 ${tmpd}/exists echo "existslink" > ${tmps}/dotfiles/existslink chmod 644 ${tmpd}/exists +mkdir -p ${tmps}/dotfiles/direxists +echo "f1" > ${tmps}/dotfiles/direxists/f1 +mkdir -p ${tmpd}/direxists +echo "f1" > ${tmpd}/direxists/f1 +chmod 644 ${tmpd}/direxists/f1 +chmod 744 ${tmpd}/direxists + +mkdir -p ${tmps}/dotfiles/linkchildren +echo "f1" > ${tmps}/dotfiles/linkchildren/f1 +mkdir -p ${tmps}/dotfiles/linkchildren/d1 +echo "f2" > ${tmps}/dotfiles/linkchildren/d1/f2 + +echo '{{@@ profile @@}}' > ${tmps}/dotfiles/symlinktemplate + +mkdir -p ${tmps}/dotfiles/symlinktemplatedir +echo "{{@@ profile @@}}" > ${tmps}/dotfiles/symlinktemplatedir/t + cat > ${cfg} << _EOF config: backup: true @@ -101,6 +126,25 @@ dotfiles: dst: ${tmpd}/existslink chmod: 777 link: link + d_direxists: + src: direxists + dst: ${tmpd}/direxists + chmod: 777 + d_linkchildren: + src: linkchildren + dst: ${tmpd}/linkchildren + chmod: 777 + link: link_children + f_symlinktemplate: + src: symlinktemplate + dst: ${tmpd}/symlinktemplate + chmod: 777 + link: link + d_symlinktemplatedir: + src: symlinktemplatedir + dst: ${tmpd}/symlinktemplatedir + chmod: 777 + link: link profiles: p1: dotfiles: @@ -109,45 +153,60 @@ profiles: - d_dir - f_exists - f_existslink + - d_direxists + - d_linkchildren + - f_symlinktemplate + - d_symlinktemplatedir p2: dotfiles: - f_exists - f_existslink + - d_linkchildren + - f_symlinktemplate _EOF #cat ${cfg} # install +echo "first install round" cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 -V ${i} -mode=`stat -c '%a' "${tmpd}/f777"` -[ "${mode}" != "777" ] && echo "bad mode for f777" && exit 1 +has_rights "${tmpd}/f777" "777" +has_rights "${tmpd}/link" "777" +has_rights "${tmpd}/dir" "777" +has_rights "${tmpd}/exists" "777" +has_rights "${tmpd}/existslink" "777" +has_rights "${tmpd}/direxists" "777" +has_rights "${tmpd}/direxists/f1" "644" +has_rights "${tmpd}/linkchildren" "777" +has_rights "${tmpd}/linkchildren/f1" "644" +has_rights "${tmpd}/linkchildren/d1" "755" +has_rights "${tmpd}/linkchildren/d1/f2" "644" +has_rights "${tmpd}/symlinktemplate" "777" -mode=`stat -c '%a' "${tmpd}/link"` -[ "${mode}" != "777" ] && echo "bad mode for link" && exit 1 - -mode=`stat -c '%a' "${tmpd}/dir"` -[ "${mode}" != "777" ] && echo "bad mode for dir" && exit 1 - -mode=`stat -c '%a' "${tmpd}/exists"` -[ "${mode}" != "777" ] && echo "bad mode for exists" && exit 1 - -mode=`stat -c '%a' "${tmpd}/existslink"` -[ "${mode}" != "777" ] && echo "bad mode for existslink" && exit 1 +grep 'p1' ${tmpd}/symlinktemplate +grep 'p1' ${tmpd}/symlinktemplatedir/t +## second round echo "exists" > ${tmps}/dotfiles/exists -chmod 644 ${tmps}/dotfiles/exists +chmod 600 ${tmps}/dotfiles/exists echo "exists" > ${tmpd}/exists -chmod 644 ${tmpd}/exists +chmod 600 ${tmpd}/exists -chmod 644 ${tmpd}/existslink +chmod 600 ${tmpd}/existslink -cd ${ddpath} | ${bin} install -c ${cfg} -p p2 -V ${i} +chmod 700 ${tmpd}/linkchildren -mode=`stat -c '%a' "${tmpd}/exists"` -[ "${mode}" != "777" ] && echo "bad mode for exists" && exit 1 +chmod 600 ${tmpd}/symlinktemplate -mode=`stat -c '%a' "${tmpd}/existslink"` -[ "${mode}" != "777" ] && echo "bad mode for existslink" && exit 1 +echo "second install round" +cd ${ddpath} | ${bin} install -c ${cfg} -p p2 -f -V ${i} + +has_rights "${tmpd}/exists" "777" +has_rights "${tmpd}/existslink" "777" +has_rights "${tmpd}/linkchildren/f1" "644" +has_rights "${tmpd}/linkchildren/d1" "755" +has_rights "${tmpd}/linkchildren/d1/f2" "644" +has_rights "${tmpd}/symlinktemplate" "777" ## CLEANING rm -rf ${tmps} ${tmpd} From 0f79a29ab26c89e734187d0cd3c78f0f07df42bc Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 13 Nov 2020 15:49:07 +0100 Subject: [PATCH 13/82] print mode in octal --- dotdrop/comparator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotdrop/comparator.py b/dotdrop/comparator.py index 9af3054..8c28266 100644 --- a/dotdrop/comparator.py +++ b/dotdrop/comparator.py @@ -64,7 +64,7 @@ class Comparator: right_mode = get_file_perm(right) if left_mode == right_mode: return '' - ret = 'modes differ ({} vs {})\n'.format(left_mode, right_mode) + ret = 'modes differ ({:o} vs {:o})\n'.format(left_mode, right_mode) return ret def _comp_file(self, left, right, ignore): From 4901e1b9691bc93386b76a8b44f41dfa570edc3b Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 13 Nov 2020 16:06:04 +0100 Subject: [PATCH 14/82] norm mkdtemp for chmod --- tests/helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/helpers.py b/tests/helpers.py index 83c4800..d080e9e 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -65,7 +65,9 @@ def get_string(length): def get_tempdir(): """Get a temporary directory""" - return tempfile.mkdtemp(suffix=TMPSUFFIX) + tmpdir = tempfile.mkdtemp(suffix=TMPSUFFIX) + os.chmod(tmpdir, 0o755) + return tmpdir def create_random_file(directory, content=None, From ca1956454b4ab0da79d9248158345988716f9998 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 13 Nov 2020 16:08:35 +0100 Subject: [PATCH 15/82] speed up tests on ci/cd --- tests.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests.sh b/tests.sh index 1a18386..a6c56e2 100755 --- a/tests.sh +++ b/tests.sh @@ -33,7 +33,12 @@ cur=`dirname $(readlink -f "${0}")` export COVERAGE_FILE="${cur}/.coverage" # execute tests with coverage -PYTHONPATH="dotdrop" ${nosebin} -s --with-coverage --cover-package=dotdrop +if [ -z ${GITHUB_WORKFLOW} ]; then + #PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop + PYTHONPATH="dotdrop" ${nosebin} -s --processes=-1 --with-coverage --cover-package=dotdrop +else + PYTHONPATH="dotdrop" ${nosebin} --processes=-1 --with-coverage --cover-package=dotdrop +fi #PYTHONPATH="dotdrop" python3 -m pytest tests # enable debug logs From dcc832b674488eca63cdc286f208c979a158134d Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 13 Nov 2020 16:42:22 +0100 Subject: [PATCH 16/82] refactor install --- dotdrop/installer.py | 382 ++++++++++++++++++++++++------------------- 1 file changed, 215 insertions(+), 167 deletions(-) diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 10bed19..514ca01 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -22,7 +22,7 @@ class Installer: dry=False, safe=False, workdir='~/.config/dotdrop', debug=False, diff=True, totemp=None, showdiff=False, backup_suffix='.dotdropbak', diff_cmd=''): - """constructor + """ @base: directory path where to search for templates @create: create directory hierarchy if missing when installing @backup: backup existing dotfile when installing @@ -40,7 +40,11 @@ class Installer: self.backup = backup self.dry = dry self.safe = safe - self.workdir = os.path.expanduser(workdir) + workdir = os.path.expanduser(workdir) + workdir = os.path.normpath(workdir) + self.workdir = workdir + base = os.path.expanduser(base) + base = os.path.normpath(base) self.base = base self.debug = debug self.diff = diff @@ -48,17 +52,25 @@ class Installer: self.showdiff = showdiff self.backup_suffix = backup_suffix self.diff_cmd = diff_cmd - self.comparing = False self.action_executed = False self.umask = utils.get_umask() + # avoids printing file copied logs + # when using install_to_tmp for comparing + self.comparing = False + self.log = Logger() + ######################################################## + # public methods + ######################################################## + def install(self, templater, src, dst, actionexec=None, noempty=False, ignore=[], template=True, chmod=None): """ - install src to dst using a template + install src to dst using a templater + @templater: the templater object @src: dotfile source path in dotpath @dst: dotfile destination path in the FS @@ -73,9 +85,11 @@ class Installer: - False, error_msg : error - False, None : ignored """ - if not self._check_chmod(src, dst, chmod): - err = 'ignoring "{}", not installed'.format(dst) - return False, err + if self.debug: + self.log.dbg('installing \"{}\" to \"{}\"'.format(src, dst)) + src, dst, cont, err = self._check_paths(src, dst, chmod) + if not cont: + return self._log_install(cont, err) r, err = self._install(templater, src, dst, actionexec=actionexec, @@ -86,58 +100,11 @@ class Installer: return self._log_install(r, err) - def _install(self, templater, src, dst, - actionexec=None, noempty=False, - ignore=[], template=True, - chmod=None): - """install link:nolink""" - if self.debug: - self.log.dbg('installing \"{}\" to \"{}\"'.format(src, dst)) - # check dst exists in config - if not dst or not src: - if self.debug: - self.log.dbg('empty dst for {}'.format(src)) - return True, None - - self.action_executed = False - - # check source file exists - src = os.path.join(self.base, os.path.expanduser(src)) - if not os.path.exists(src): - err = 'source dotfile does not exist: {}'.format(src) - return False, err - - # check dst is valid - dst = os.path.expanduser(dst) - if self.totemp: - dst = self._pivot_path(dst, self.totemp) - if utils.samefile(src, dst): - # symlink loop - err = 'dotfile points to itself: {}'.format(dst) - return False, err - - isdir = os.path.isdir(src) - if self.debug: - self.log.dbg('install {} to {}'.format(src, dst)) - self.log.dbg('\"{}\" is a directory: {}'.format(src, isdir)) - if isdir: - b, e = self._deploy_dir(templater, src, dst, - actionexec=actionexec, - noempty=noempty, ignore=ignore, - template=template, - chmod=chmod) - return b, e - b, e = self._deploy_file(templater, src, dst, - actionexec=actionexec, - noempty=noempty, ignore=ignore, - template=template, - chmod=chmod) - return b, e - def link(self, templater, src, dst, actionexec=None, template=True, chmod=None): """ set src as the link target of dst + @templater: the templater @src: dotfile source path in dotpath @dst: dotfile destination path in the FS @@ -150,9 +117,11 @@ class Installer: - False, error_msg : error - False, None : ignored """ - if not self._check_chmod(src, dst, chmod): - err = 'ignoring "{}", nothing installed'.format(dst) - return False, err + if self.debug: + self.log.dbg('link \"{}\" to \"{}\"'.format(src, dst)) + src, dst, cont, err = self._check_paths(src, dst, chmod) + if not cont: + return self._log_install(cont, err) r, err = self._link(templater, src, dst, actionexec=actionexec, @@ -162,22 +131,133 @@ class Installer: utils.chmod(dst, chmod, self.debug) return self._log_install(r, err) - def _link(self, templater, src, dst, actionexec=None, - template=True): - """install link:link""" + def link_children(self, templater, src, dst, actionexec=None, + template=True, chmod=None): + """ + link all files under a given directory + + @templater: the templater + @src: dotfile source path in dotpath + @dst: dotfile destination path in the FS + @actionexec: action executor callback + @template: template this dotfile + @chmod: file mode to apply + + return + - True, None: success + - False, error_msg: error + - False, None, ignored + """ if self.debug: - self.log.dbg('link \"{}\" to \"{}\"'.format(src, dst)) - if not dst or not src: - if self.debug: - self.log.dbg('empty dst for {}'.format(src)) - return True, None + self.log.dbg('link_children \"{}\" to \"{}\"'.format(src, dst)) + src, dst, cont, err = self._check_paths(src, dst, chmod) + if not cont: + return self._log_install(cont, err) + + r, err = self._link_children(templater, src, dst, + actionexec=actionexec, + template=template, chmod=chmod) + return self._log_install(r, err) + + def install_to_temp(self, templater, tmpdir, src, dst, + template=True, chmod=None): + """ + install a dotfile to a tempdir + + @templater: the templater object + @tmpdir: where to install + @src: dotfile source path in dotpath + @dst: dotfile destination path in the FS + @template: template this dotfile + @chmod: rights to apply if any + + return + - success, error-if-any, dir-where-installed + """ + if self.debug: + self.log.dbg('tmp install {} (defined dst: {})'.format(src, dst)) + src, dst, cont, err = self._check_paths(src, dst, chmod) + if not cont: + return self._log_install(cont, err) + + ret = False + tmpdst = '' + + # save flags + self.comparing = True + drysaved = self.dry + self.dry = False + diffsaved = self.diff + self.diff = False + createsaved = self.create + self.create = True + + # install the dotfile to a temp directory + tmpdst = self._pivot_path(dst, tmpdir) + ret, err = self.install(templater, src, tmpdst, + template=template, chmod=chmod) + if self.debug: + if ret: + self.log.dbg('tmp installed in {}'.format(tmpdst)) + + # restore flags + self.dry = drysaved + self.diff = diffsaved + self.create = createsaved + self.comparing = False + return ret, err, tmpdst + + ######################################################## + # low level accessors for public methods + ######################################################## + + def _install(self, templater, src, dst, + actionexec=None, noempty=False, + ignore=[], template=True, + chmod=None): + """install link:nolink""" self.action_executed = False - src = os.path.normpath(os.path.join(self.base, - os.path.expanduser(src))) + + # check source file exists + src = os.path.join(self.base, src) + if not os.path.exists(src): + err = 'source dotfile does not exist: {}'.format(src) + return False, err + + # check dst is valid + if self.totemp: + dst = self._pivot_path(dst, self.totemp) + if utils.samefile(src, dst): + # symlink loop + err = 'dotfile points to itself: {}'.format(dst) + return False, err + + isdir = os.path.isdir(src) + if self.debug: + self.log.dbg('install {} to {}'.format(src, dst)) + self.log.dbg('\"{}\" is a directory: {}'.format(src, isdir)) + if isdir: + b, e = self._copy_dir(templater, src, dst, + actionexec=actionexec, + noempty=noempty, ignore=ignore, + template=template, + chmod=chmod) + return b, e + b, e = self._copy_file(templater, src, dst, + actionexec=actionexec, + noempty=noempty, ignore=ignore, + template=template, + chmod=chmod) + return b, e + + def _link(self, templater, src, dst, actionexec=None, + template=True): + """install link:link""" + self.action_executed = False + src = os.path.join(self.base, src) if not os.path.exists(src): err = 'source dotfile does not exist: {}'.format(src) return False, err - dst = os.path.normpath(os.path.expanduser(dst)) if self.totemp: # ignore actions b, e = self.install(templater, src, dst, actionexec=None, @@ -197,41 +277,15 @@ class Installer: b, e = self._symlink(src, dst, actionexec=actionexec) return b, e - def link_children(self, templater, src, dst, actionexec=None, - template=True, chmod=None): - """ - link all files under a given directory - @templater: the templater - @src: dotfile source path in dotpath - @dst: dotfile destination path in the FS - @actionexec: action executor callback - @template: template this dotfile - @chmod: file mode to apply - - return - - True, None: success - - False, error_msg: error - - False, None, ignored - """ - if not self._check_chmod(src, dst, chmod): - err = 'ignoring "{}", nothing installed'.format(dst) - return False, err - r, err = self._link_children(templater, src, dst, - actionexec=actionexec, - template=template, chmod=chmod) - return self._log_install(r, err) - def _link_children(self, templater, src, dst, actionexec=None, template=True, chmod=None): """install link:link_children""" - if self.debug: - self.log.dbg('link_children \"{}\" to \"{}\"'.format(src, dst)) if not dst or not src: if self.debug: self.log.dbg('empty dst for {}'.format(src)) return True, None self.action_executed = False - parent = os.path.join(self.base, os.path.expanduser(src)) + parent = os.path.join(self.base, src) # Fail if source doesn't exist if not os.path.exists(parent): @@ -246,7 +300,6 @@ class Installer: err = 'source dotfile is not a directory: {}'.format(parent) return False, err - dst = os.path.normpath(os.path.expanduser(dst)) if not os.path.lexists(dst): self.log.sub('creating directory "{}"'.format(dst)) os.makedirs(dst) @@ -306,6 +359,10 @@ class Installer: return installed > 0, None + ######################################################## + # file operations + ######################################################## + def _symlink(self, src, dst, actionexec=None): """ set src as a link target of dst @@ -362,16 +419,10 @@ class Installer: self.log.sub('linked {} to {}'.format(dst, src)) return True, None - def _get_tmp_file_vars(self, src, dst): - tmp = {} - tmp['_dotfile_sub_abs_src'] = src - tmp['_dotfile_sub_abs_dst'] = dst - return tmp - - def _deploy_file(self, templater, src, dst, - actionexec=None, noempty=False, - ignore=[], template=True, - chmod=None): + def _copy_file(self, templater, src, dst, + actionexec=None, noempty=False, + ignore=[], template=True, + chmod=None): """install src to dst when is a file""" if self.debug: self.log.dbg('deploy file: {}'.format(src)) @@ -439,9 +490,9 @@ class Installer: err = 'installing {} to {}'.format(src, dst) return False, err - def _deploy_dir(self, templater, src, dst, - actionexec=None, noempty=False, - ignore=[], template=True, chmod=None): + def _copy_dir(self, templater, src, dst, + actionexec=None, noempty=False, + ignore=[], template=True, chmod=None): """install src to dst when is a directory""" if self.debug: self.log.dbg('deploy dir {}'.format(src)) @@ -462,13 +513,13 @@ class Installer: self.log.dbg('deploy sub from {}: {}'.format(dst, entry)) if not os.path.isdir(f): # is file - res, err = self._deploy_file(templater, f, - os.path.join(dst, entry), - actionexec=actionexec, - noempty=noempty, - ignore=ignore, - template=template, - chmod=None) + res, err = self._copy_file(templater, f, + os.path.join(dst, entry), + actionexec=actionexec, + noempty=noempty, + ignore=ignore, + template=template, + chmod=None) if not res and err: # error occured ret = res, err @@ -478,13 +529,13 @@ class Installer: ret = True, None else: # is directory - res, err = self._deploy_dir(templater, f, - os.path.join(dst, entry), - actionexec=actionexec, - noempty=noempty, - ignore=ignore, - template=template, - chmod=None) + res, err = self._copy_dir(templater, f, + os.path.join(dst, entry), + actionexec=actionexec, + noempty=noempty, + ignore=ignore, + template=template, + chmod=None) if not res and err: # error occured ret = res, err @@ -592,6 +643,16 @@ class Installer: utils.chmod(dst, chmod, debug=self.debug) return 0, None + ######################################################## + # helpers + ######################################################## + + def _get_tmp_file_vars(self, src, dst): + tmp = {} + tmp['_dotfile_sub_abs_src'] = src + tmp['_dotfile_sub_abs_dst'] = dst + return tmp + def _diff_before_write(self, src, dst, content=None, quiet=False): """ diff before writing @@ -664,45 +725,6 @@ class Installer: self.action_executed = True return ret, err - def _install_to_temp(self, templater, src, dst, tmpdir, - template=True, chmod=None): - """install a dotfile to a tempdir""" - tmpdst = self._pivot_path(dst, tmpdir) - r = self.install(templater, src, tmpdst, - template=template, chmod=chmod) - return r, tmpdst - - def install_to_temp(self, templater, tmpdir, src, dst, - template=True, chmod=None): - """install a dotfile to a tempdir""" - ret = False - tmpdst = '' - # save some flags while comparing - self.comparing = True - drysaved = self.dry - self.dry = False - diffsaved = self.diff - self.diff = False - createsaved = self.create - self.create = True - # normalize src and dst - src = os.path.expanduser(src) - dst = os.path.expanduser(dst) - if self.debug: - self.log.dbg('tmp install {} (defined dst: {})'.format(src, dst)) - # install the dotfile to a temp directory for comparing - r, tmpdst = self._install_to_temp(templater, src, dst, tmpdir, - template=template, chmod=chmod) - ret, err = r - if self.debug: - self.log.dbg('tmp installed in {}'.format(tmpdst)) - # reset flags - self.dry = drysaved - self.diff = diffsaved - self.comparing = False - self.create = createsaved - return ret, err, tmpdst - def _log_install(self, boolean, err): """log installation process""" if not self.debug: @@ -717,7 +739,7 @@ class Installer: return boolean, err def _check_chmod(self, src, dst, chmod): - """chmod file if needed and return True to continue""" + """check chmod needs change""" if chmod: return True if not os.path.exists(dst): @@ -728,12 +750,38 @@ class Installer: perms = utils.get_file_perm(dst) if perms == cperms: return True - if self.safe: - if perms & self.umask: - # perms and umask differ + if perms & self.umask: + # perms and umask differ + if self.safe: msg = 'File mode ({}) differs from umask ({})' msg.format(perms, self.umask) msg += ', continue' if not self.log.ask(msg): return False return True + + def _check_paths(self, src, dst, chmod): + """ + check and normalize param + returns , , , + """ + # check both path are valid + if not dst or not src: + err = 'empty dst or src for {}'.format(src) + if self.debug: + self.log.dbg(err) + return None, None, False, err + + # normalize src and dst + src = os.path.expanduser(src) + src = os.path.normpath(src) + + dst = os.path.expanduser(dst) + dst = os.path.normpath(dst) + + # check chmod + if not self._check_chmod(src, dst, chmod): + err = 'ignoring "{}", not installed'.format(dst) + return None, None, False, err + + return src, dst, True, None From dc2513a995c4d75398cce70a480cdfc0ceb84ef3 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 13 Nov 2020 16:42:31 +0100 Subject: [PATCH 17/82] handle fake dotfile --- dotdrop/dotdrop.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 6bb509f..9ae4022 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -196,10 +196,17 @@ def cmd_install(o): # in parallel ex = futures.ThreadPoolExecutor(max_workers=o.install_parallel) - wait_for = [ - ex.submit(_dotfile_install, o, dotfile, tmpdir=tmpdir) - for dotfile in dotfiles - ] + wait_for = [] + for dotfile in dotfiles: + if not dotfile.src or not dotfile.dst: + # fake dotfile are always considered installed + if o.debug: + LOG.dbg('fake dotfile installed') + installed += 1 + else: + j = ex.submit(_dotfile_install, o, dotfile, tmpdir=tmpdir) + wait_for.append(j) + # check result for f in futures.as_completed(wait_for): r, key, err = f.result() if r: @@ -210,7 +217,16 @@ def cmd_install(o): else: # sequentially for dotfile in dotfiles: - r, key, err = _dotfile_install(o, dotfile, tmpdir=tmpdir) + if not dotfile.src or not dotfile.dst: + # fake dotfile are always considered installed + if o.debug: + LOG.dbg('fake dotfile installed') + key = dotfile.key + r = True + err = None + else: + r, key, err = _dotfile_install(o, dotfile, tmpdir=tmpdir) + # check result if r: installed += 1 elif err: From 1a885998394355f965f302a6c4539e0fcfbb7da8 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 13 Nov 2020 17:34:06 +0100 Subject: [PATCH 18/82] fix coverage for gh actions --- tests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests.sh b/tests.sh index a6c56e2..29cc9c2 100755 --- a/tests.sh +++ b/tests.sh @@ -37,7 +37,8 @@ if [ -z ${GITHUB_WORKFLOW} ]; then #PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop PYTHONPATH="dotdrop" ${nosebin} -s --processes=-1 --with-coverage --cover-package=dotdrop else - PYTHONPATH="dotdrop" ${nosebin} --processes=-1 --with-coverage --cover-package=dotdrop + #PYTHONPATH="dotdrop" ${nosebin} --processes=-1 --with-coverage --cover-package=dotdrop + PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop fi #PYTHONPATH="dotdrop" python3 -m pytest tests From a6ba5d99bf639aa407e0c35311e7fb899b26ba71 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 13 Nov 2020 17:41:59 +0100 Subject: [PATCH 19/82] chmod for link children too --- dotdrop/cfg_yaml.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 663d06e..2d83512 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -340,9 +340,7 @@ class CfgYaml: # chmod if chmod: - lnkval = df_dict.get(self.key_dotfile_link, None) - if lnkval != self.lnk_children: - df_dict[self.key_dotfile_chmod] = str(format(chmod, 'o')) + df_dict[self.key_dotfile_chmod] = str(format(chmod, 'o')) # add to global dict self._yaml_dict[self.key_dotfiles][key] = df_dict From 15617ceaea6e7a19ecccd118d9ef5ad8aa81db7f Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 14 Nov 2020 12:12:47 +0100 Subject: [PATCH 20/82] enable parallel testing --- .github/workflows/testing.yml | 1 + tests.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 29de141..b039042 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -20,6 +20,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r tests-requirements.txt + pip install --user --upgrade coverage pip install -r requirements.txt npm install -g remark-cli remark-validate-links npm install -g markdown-link-check diff --git a/tests.sh b/tests.sh index 29cc9c2..f72e9a0 100755 --- a/tests.sh +++ b/tests.sh @@ -37,8 +37,8 @@ if [ -z ${GITHUB_WORKFLOW} ]; then #PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop PYTHONPATH="dotdrop" ${nosebin} -s --processes=-1 --with-coverage --cover-package=dotdrop else - #PYTHONPATH="dotdrop" ${nosebin} --processes=-1 --with-coverage --cover-package=dotdrop - PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop + PYTHONPATH="dotdrop" ${nosebin} --processes=-1 --with-coverage --cover-package=dotdrop + #PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop fi #PYTHONPATH="dotdrop" python3 -m pytest tests From 32e667a59bc9edf573bffc6fccef4d0ad997ab6e Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 14 Nov 2020 12:13:08 +0100 Subject: [PATCH 21/82] properly handle chmod on import --- dotdrop/dotdrop.py | 6 ++++-- dotdrop/utils.py | 8 ++++++++ tests-ng/chmod-import.sh | 19 ++++++++++++++++--- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 9ae4022..32f957c 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -19,7 +19,8 @@ from dotdrop.installer import Installer from dotdrop.updater import Updater from dotdrop.comparator import Comparator from dotdrop.utils import get_tmpdir, removepath, strip_home, \ - uniq_list, patch_ignores, dependencies_met, get_file_perm + uniq_list, patch_ignores, dependencies_met, get_file_perm, \ + get_default_file_perms from dotdrop.linktypes import LinkTypes from dotdrop.exceptions import YamlException, UndefinedException @@ -521,7 +522,8 @@ def cmd_importer(o): shutil.copy2(dst, srcf) chmod = None - if o.import_mode or perm & o.umask: + dflperm = get_default_file_perms(dst, o.umask) + if o.import_mode or perm != dflperm: # insert chmod chmod = perm retconf = o.conf.new(src, dst, linktype, chmod=chmod) diff --git a/dotdrop/utils.py b/dotdrop/utils.py index a2fdba0..d8f77b8 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -312,6 +312,14 @@ def get_umask(): return cur +def get_default_file_perms(path, umask): + """get default rights for a file""" + base = 0o666 + if os.path.isdir(path): + base = 0o777 + return base - umask + + def get_file_perm(path): """return file permission""" return os.stat(path).st_mode & 0o777 diff --git a/tests-ng/chmod-import.sh b/tests-ng/chmod-import.sh index 5845f90..054a0c0 100755 --- a/tests-ng/chmod-import.sh +++ b/tests-ng/chmod-import.sh @@ -47,6 +47,19 @@ echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" # this is the test ################################################################ +# $1 file +chmod_to_umask() +{ + u=`umask` + u=`echo ${u} | sed 's/^0*//'` + if [ -d ${1} ]; then + v=$((777 - u)) + else + v=$((666 - u)) + fi + chmod ${v} ${1} +} + # the dotfile source tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` mkdir -p ${tmps}/dotfiles @@ -155,7 +168,7 @@ mkdir -p ${tmps}/dotfiles # import with --preserve-mode for i in ${toimport}; do - chmod `umask -S` ${i} + chmod_to_umask ${i} cd ${ddpath} | ${bin} import -c ${cfg} -m -f -p p1 -V ${i} done @@ -182,9 +195,9 @@ _EOF rm -rf ${tmps}/dotfiles mkdir -p ${tmps}/dotfiles -# import with --preserve-mode +# import without --preserve-mode for i in ${toimport}; do - chmod `umask -S` ${i} + chmod_to_umask ${i} cd ${ddpath} | ${bin} import -c ${cfg} -f -p p1 -V ${i} done From 5ec753e053acf23d738a0a931afef1b84846d1bc Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 14 Nov 2020 12:22:49 +0100 Subject: [PATCH 22/82] disable parallel unittests due to coverage issue --- tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests.sh b/tests.sh index f72e9a0..04431ed 100755 --- a/tests.sh +++ b/tests.sh @@ -34,8 +34,8 @@ export COVERAGE_FILE="${cur}/.coverage" # execute tests with coverage if [ -z ${GITHUB_WORKFLOW} ]; then - #PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop - PYTHONPATH="dotdrop" ${nosebin} -s --processes=-1 --with-coverage --cover-package=dotdrop + PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop + #PYTHONPATH="dotdrop" ${nosebin} -s --processes=-1 --with-coverage --cover-package=dotdrop else PYTHONPATH="dotdrop" ${nosebin} --processes=-1 --with-coverage --cover-package=dotdrop #PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop From f02c204d447336c2154ac0110f1af13e95f5c2cd Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 14 Nov 2020 13:12:15 +0100 Subject: [PATCH 23/82] add test for chmod on update --- tests-ng/chmod-update.sh | 145 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100755 tests-ng/chmod-update.sh diff --git a/tests-ng/chmod-update.sh b/tests-ng/chmod-update.sh new file mode 100755 index 0000000..06cff60 --- /dev/null +++ b/tests-ng/chmod-update.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2020, deadc0de6 +# +# test chmod on update +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +#echo "dotfile destination: ${tmpd}" + +# create the dotfile +dnormal="${tmpd}/dir_normal" +mkdir -p ${dnormal} +echo "dir_normal/f1" > ${dnormal}/file1 +echo "dir_normal/f2" > ${dnormal}/file2 + +dlink="${tmpd}/dir_link" +mkdir -p ${dlink} +echo "dir_link/f1" > ${dlink}/file1 +echo "dir_link/f2" > ${dlink}/file2 + +dlinkchildren="${tmpd}/dir_link_children" +mkdir -p ${dlinkchildren} +echo "dir_linkchildren/f1" > ${dlinkchildren}/file1 +echo "dir_linkchildren/f2" > ${dlinkchildren}/file2 + +fnormal="${tmpd}/filenormal" +echo "filenormal" > ${fnormal} + +flink="${tmpd}/filelink" +echo "filelink" > ${flink} + +toimport="${dnormal} ${dlink} ${dlinkchildren} ${fnormal} ${flink}" + +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: +profiles: +_EOF +#cat ${cfg} + +# import +for i in ${toimport}; do + cd ${ddpath} | ${bin} import -c ${cfg} -f -p p1 -V ${i} +done + +# test no chmod +cnt=`cat ${cfg} | grep chmod | wc -l` +[ "${cnt}" != "0" ] && echo "chmod wrongly inserted" && exit 1 + +###################### +# update dnormal +chmod 777 ${dnormal} +cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V ${dnormal} + +# check rights updated +[ "`stat -c '%a' ${tmps}/dotfiles/${tmpd}/$(basename ${dnormal})`" != "777" ] && echo "rights not updated (1)" && exit 1 + +###################### +# update dlink +chmod 777 ${dlink} +cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V ${dlink} + +# check rights updated +[ "`stat -c '%a' ${tmps}/dotfiles/${tmpd}/$(basename ${dlink})`" != "777" ] && echo "rights not updated (2)" && exit 1 + +###################### +# update dlinkchildren +chmod 777 ${dlinkchildren} +cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V ${dlinkchildren} + +# check rights updated +[ "`stat -c '%a' ${tmps}/dotfiles/${tmpd}/$(basename ${dlinkchildren})`" != "777" ] && echo "rights not updated (3)" && exit 1 + +###################### +# update fnormal +chmod 777 ${fnormal} +cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V ${fnormal} + +# check rights updated +[ "`stat -c '%a' ${tmps}/dotfiles/${tmpd}/$(basename ${fnormal})`" != "777" ] && echo "rights not updated (4)" && exit 1 + +###################### +# update flink +chmod 777 ${flink} +cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V ${flink} + +# check rights updated +[ "`stat -c '%a' ${tmps}/dotfiles/${tmpd}/$(basename ${flink})`" != "777" ] && echo "rights not updated (5)" && exit 1 + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 From e093bb29d13259208b003ec1e9159fdf0c1f9db6 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 14 Nov 2020 13:12:28 +0100 Subject: [PATCH 24/82] fix ci/cd --- tests.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests.sh b/tests.sh index 04431ed..c32ddec 100755 --- a/tests.sh +++ b/tests.sh @@ -34,11 +34,13 @@ export COVERAGE_FILE="${cur}/.coverage" # execute tests with coverage if [ -z ${GITHUB_WORKFLOW} ]; then - PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop - #PYTHONPATH="dotdrop" ${nosebin} -s --processes=-1 --with-coverage --cover-package=dotdrop -else - PYTHONPATH="dotdrop" ${nosebin} --processes=-1 --with-coverage --cover-package=dotdrop + ## local #PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop + PYTHONPATH="dotdrop" ${nosebin} -s --processes=-1 --with-coverage --cover-package=dotdrop +else + ## CI/CD + #PYTHONPATH="dotdrop" ${nosebin} --processes=-1 --with-coverage --cover-package=dotdrop + PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop fi #PYTHONPATH="dotdrop" python3 -m pytest tests From 458b6a0bf18915adb478f7ee2cd2a545bcc849b2 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 14 Nov 2020 13:12:40 +0100 Subject: [PATCH 25/82] update doc --- docs/config.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/config.md b/docs/config.md index a5f0c94..3a742a6 100644 --- a/docs/config.md +++ b/docs/config.md @@ -74,13 +74,14 @@ On `import` the following rules are applied: On `install` the following rules are applied: -* if `chmod` is specified, it will be applied to the installed dotfile +* if `chmod` is specified in the dotfile, it will be applied to the installed dotfile * if file exists and its permissions differ from `umask` and no `chmod` is specified user needs to confirm installation (unless `-f --force` is used or the permission match the dotfile in the `dotpath`) On `update`: -* **TODO** +* no `chmod` entry is added/updated in the config however the rights are mirrored from the filesystem to the + file in `dotpath` ## Symlink dotfiles From 81d585a37fa727e9df92dfa75de11cb1bed03f61 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 14 Nov 2020 13:12:52 +0100 Subject: [PATCH 26/82] fix mirror rights on update --- dotdrop/updater.py | 17 ++++++++++++----- dotdrop/utils.py | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/dotdrop/updater.py b/dotdrop/updater.py index 3f92faf..fcbe568 100644 --- a/dotdrop/updater.py +++ b/dotdrop/updater.py @@ -13,7 +13,7 @@ import filecmp from dotdrop.logger import Logger from dotdrop.templategen import Templategen from dotdrop.utils import patch_ignores, removepath, get_unique_tmp_name, \ - write_to_tmpfile, must_ignore, mirror_file_rights + write_to_tmpfile, must_ignore, mirror_file_rights, get_file_perm from dotdrop.exceptions import UndefinedException @@ -162,15 +162,20 @@ class Updater: def _same_rights(self, left, right): """return True if files have the same modes""" try: - lefts = os.stat(left) - rights = os.stat(right) - return lefts.st_mode == rights.st_mode + lefts = get_file_perm(left) + rights = get_file_perm(right) + return lefts == rights except OSError as e: self.log.err(e) return False def _mirror_rights(self, src, dst): try: + if self.debug: + srcr = get_file_perm(src) + dstr = get_file_perm(dst) + msg = 'copy rights from {} ({:o}) to {} ({:o})' + self.log.dbg(msg.format(src, srcr, dst, dstr)) mirror_file_rights(src, dst) except OSError as e: self.log.err(e) @@ -228,7 +233,9 @@ class Updater: # find the differences diff = filecmp.dircmp(path, dtpath, ignore=None) # handle directories diff - return self._merge_dirs(diff) + ret = self._merge_dirs(diff) + self._mirror_rights(path, dtpath) + return ret def _merge_dirs(self, diff): """Synchronize directories recursively.""" diff --git a/dotdrop/utils.py b/dotdrop/utils.py index d8f77b8..0c989e9 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -300,7 +300,7 @@ def dependencies_met(): def mirror_file_rights(src, dst): """mirror file rights of src to dst (can rise exc)""" - rights = os.stat(src).st_mode + rights = get_file_perm(src) os.chmod(dst, rights) From 87b70053c89e866ea0c94ae527ff9cd94cefe7fb Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 14 Nov 2020 15:01:56 +0100 Subject: [PATCH 27/82] improve tests --- tests-ng/chmod-compare.sh | 3 +-- tests-ng/chmod-import.sh | 19 +++++++++------- tests-ng/chmod-install.sh | 47 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 10 deletions(-) diff --git a/tests-ng/chmod-compare.sh b/tests-ng/chmod-compare.sh index c8e77e2..c92be03 100755 --- a/tests-ng/chmod-compare.sh +++ b/tests-ng/chmod-compare.sh @@ -94,7 +94,7 @@ profiles: _EOF #cat ${cfg} -# import without --preserve-mode +# import for i in ${toimport}; do cd ${ddpath} | ${bin} import -c ${cfg} -f -p p1 ${i} done @@ -111,7 +111,6 @@ chmod 700 ${flink} set +e cnt=`cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 2>&1 | grep 'modes differ' | wc -l` set -e - [ "${cnt}" != "5" ] && echo "compare modes failed" && exit 1 ## CLEANING diff --git a/tests-ng/chmod-import.sh b/tests-ng/chmod-import.sh index 054a0c0..6d1a5da 100755 --- a/tests-ng/chmod-import.sh +++ b/tests-ng/chmod-import.sh @@ -67,7 +67,7 @@ mkdir -p ${tmps}/dotfiles tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` #echo "dotfile destination: ${tmpd}" -# create the dotfile +# create the dotfiles dnormal="${tmpd}/dir_normal" mkdir -p ${dnormal} echo "dir_normal/f1" > ${dnormal}/file1 @@ -120,8 +120,8 @@ cat ${cfg} cd ${ddpath} | ${bin} detail -c ${cfg} -p p1 -V tot=`echo ${toimport} | wc -w` -cnt=`cat ${cfg} | grep chmod | wc -l` -[ "${cnt}" != "${tot}" ] && echo "not all chmod inserted" && exit 1 +cnt=`cat ${cfg} | grep "chmod: '777'" | wc -l` +[ "${cnt}" != "${tot}" ] && echo "not all chmod inserted (1)" && exit 1 ## with link cat > ${cfg} << _EOF @@ -138,7 +138,6 @@ rm -rf ${tmps}/dotfiles mkdir -p ${tmps}/dotfiles # import without --preserve-mode and link - for i in ${toimport}; do cd ${ddpath} | ${bin} import -c ${cfg} -l link -f -p p1 -V ${i} done @@ -149,8 +148,12 @@ cat ${cfg} cd ${ddpath} | ${bin} detail -c ${cfg} -p p1 -V tot=`echo ${toimport} | wc -w` -cnt=`cat ${cfg} | grep chmod | wc -l` -[ "${cnt}" != "${tot}" ] && echo "not all chmod inserted" && exit 1 +cnt=`cat ${cfg} | grep "chmod: '777'" | wc -l` +[ "${cnt}" != "${tot}" ] && echo "not all chmod inserted (2)" && exit 1 + +tot=`echo ${toimport} | wc -w` +cnt=`cat ${cfg} | grep 'link: link' | wc -l` +[ "${cnt}" != "${tot}" ] && echo "not all link inserted" && exit 1 ## --preserve-mode cat > ${cfg} << _EOF @@ -178,8 +181,8 @@ cat ${cfg} cd ${ddpath} | ${bin} detail -c ${cfg} -p p1 -V tot=`echo ${toimport} | wc -w` -cnt=`cat ${cfg} | grep chmod | wc -l` -[ "${cnt}" != "${tot}" ] && echo "not all chmod inserted" && exit 1 +cnt=`cat ${cfg} | grep "chmod: " | wc -l` +[ "${cnt}" != "${tot}" ] && echo "not all chmod inserted (3)" && exit 1 ## import normal cat > ${cfg} << _EOF diff --git a/tests-ng/chmod-install.sh b/tests-ng/chmod-install.sh index 05ff2a8..01ed80a 100755 --- a/tests-ng/chmod-install.sh +++ b/tests-ng/chmod-install.sh @@ -58,6 +58,22 @@ echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" # this is the test ################################################################ +get_file_mode() +{ + u=`umask` + u=`echo ${u} | sed 's/^0*//'` + v=$((666 - u)) + echo "${v}" +} + +get_dir_mode() +{ + u=`umask` + u=`echo ${u} | sed 's/^0*//'` + v=$((777 - u)) + echo "${v}" +} + # the dotfile source tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` mkdir -p ${tmps}/dotfiles @@ -98,6 +114,8 @@ echo '{{@@ profile @@}}' > ${tmps}/dotfiles/symlinktemplate mkdir -p ${tmps}/dotfiles/symlinktemplatedir echo "{{@@ profile @@}}" > ${tmps}/dotfiles/symlinktemplatedir/t +echo 'nomode' > ${tmps}/dotfiles/nomode + cat > ${cfg} << _EOF config: backup: true @@ -145,6 +163,9 @@ dotfiles: dst: ${tmpd}/symlinktemplatedir chmod: 777 link: link + f_nomode: + src: nomode + dst: ${tmpd}/nomode profiles: p1: dotfiles: @@ -157,12 +178,14 @@ profiles: - d_linkchildren - f_symlinktemplate - d_symlinktemplatedir + - f_nomode p2: dotfiles: - f_exists - f_existslink - d_linkchildren - f_symlinktemplate + - f_nomode _EOF #cat ${cfg} @@ -182,6 +205,8 @@ has_rights "${tmpd}/linkchildren/f1" "644" has_rights "${tmpd}/linkchildren/d1" "755" has_rights "${tmpd}/linkchildren/d1/f2" "644" has_rights "${tmpd}/symlinktemplate" "777" +m=`get_file_mode` +has_rights "${tmpd}/nomode" "${m}" grep 'p1' ${tmpd}/symlinktemplate grep 'p1' ${tmpd}/symlinktemplatedir/t @@ -207,6 +232,28 @@ has_rights "${tmpd}/linkchildren/f1" "644" has_rights "${tmpd}/linkchildren/d1" "755" has_rights "${tmpd}/linkchildren/d1/f2" "644" has_rights "${tmpd}/symlinktemplate" "777" +m=`get_file_mode` +has_rights "${tmpd}/nomode" "${m}" + +## no user confirmation expected +## same mode +echo "same mode" +echo "nomode" > ${tmps}/dotfiles/nomode +chmod 600 ${tmps}/dotfiles/nomode +echo "nomode" > ${tmpd}/nomode +chmod 600 ${tmpd}/nomode +cd ${ddpath} | ${bin} install -c ${cfg} -p p2 -V f_nomode +has_rights "${tmpd}/nomode" "600" + +## user confirmation expected +## different mode +echo "different mode" +echo "nomode" > ${tmps}/dotfiles/nomode +chmod 600 ${tmps}/dotfiles/nomode +echo "nomode" > ${tmpd}/nomode +chmod 700 ${tmpd}/nomode +cd ${ddpath} | printf 'y\n' | ${bin} install -c ${cfg} -p p2 -V f_nomode +has_rights "${tmpd}/nomode" "600" ## CLEANING rm -rf ${tmps} ${tmpd} From 2f17d36ffb3a90de03096525108fed6233626ea2 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 14 Nov 2020 15:02:39 +0100 Subject: [PATCH 28/82] fix chmod --- dotdrop/installer.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 514ca01..0fc32fa 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -53,7 +53,6 @@ class Installer: self.backup_suffix = backup_suffix self.diff_cmd = diff_cmd self.action_executed = False - self.umask = utils.get_umask() # avoids printing file copied logs # when using install_to_tmp for comparing self.comparing = False @@ -588,7 +587,7 @@ class Installer: if self.debug: self.log.dbg('{} is the same'.format(dst)) return 1, None - if self.safe: + if diff and self.safe: if self.debug: self.log.dbg('change detected for {}'.format(dst)) if self.showdiff: @@ -739,7 +738,10 @@ class Installer: return boolean, err def _check_chmod(self, src, dst, chmod): - """check chmod needs change""" + """ + check chmod needs change + returns True to continue installation + """ if chmod: return True if not os.path.exists(dst): @@ -750,11 +752,10 @@ class Installer: perms = utils.get_file_perm(dst) if perms == cperms: return True - if perms & self.umask: - # perms and umask differ + elif self.safe: if self.safe: - msg = 'File mode ({}) differs from umask ({})' - msg.format(perms, self.umask) + msg = 'Mode differs ({:o} and {:o} ({})' + msg = msg.format(cperms, perms, dst) msg += ', continue' if not self.log.ask(msg): return False From ae15569041fbe08148d1af1a4aab135ced5dba73 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 14 Nov 2020 15:04:25 +0100 Subject: [PATCH 29/82] update chmod doc --- docs/config.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/config.md b/docs/config.md index 3a742a6..11110d9 100644 --- a/docs/config.md +++ b/docs/config.md @@ -61,27 +61,29 @@ Here are some rules on the use of variables in configs: ## Permissions -Dotfile allows to control the permission applied to a dotfile using the -config dotfile entry `chmod`. -A `chmod` entry on a directory is applied to the directory only, not recursively. +Dotdrop allows to control the permission applied to a dotfile using the +config dotfile entry [chmod](config-format.md#dotfiles-entry). +A [chmod](config-format.md#dotfiles-entry) entry on a directory is applied to the +directory only, not recursively. On `import` the following rules are applied: * if the `-m --preserve-mode` switch is provided the imported file permissions are + stored in a `chmod` entry +* if imported file permissions differ from umask then its permissions are automatically stored in the `chmod` entry -* if imported file permissions differ from umask its permissions are automatically - stored in the `chmod` entry +* otherwise no `chmod` entry is added On `install` the following rules are applied: * if `chmod` is specified in the dotfile, it will be applied to the installed dotfile -* if file exists and its permissions differ from `umask` and no `chmod` is specified user needs - to confirm installation (unless `-f --force` is used or the permission match the dotfile in the `dotpath`) +* if no `chmod` is specified and the file exists on the filesystem with different permissions than the file in the `dotpath` + then the user needs to confirm the dotfile installation (unless `-f --force` is used) On `update`: -* no `chmod` entry is added/updated in the config however the rights are mirrored from the filesystem to the - file in `dotpath` +* no `chmod` entry is added/updated in the config however the rights are mirrored + from the filesystem to the dotfile in the `dotpath` ## Symlink dotfiles From 36be2027da5ecd41cc17b5abbb39a031118a3dc8 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 14 Nov 2020 15:04:35 +0100 Subject: [PATCH 30/82] more tests --- tests-ng/chmod-install.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests-ng/chmod-install.sh b/tests-ng/chmod-install.sh index 01ed80a..67aa009 100755 --- a/tests-ng/chmod-install.sh +++ b/tests-ng/chmod-install.sh @@ -245,6 +245,16 @@ chmod 600 ${tmpd}/nomode cd ${ddpath} | ${bin} install -c ${cfg} -p p2 -V f_nomode has_rights "${tmpd}/nomode" "600" +## no user confirmation with force +## different mode +echo "different mode" +echo "nomode" > ${tmps}/dotfiles/nomode +chmod 600 ${tmps}/dotfiles/nomode +echo "nomode" > ${tmpd}/nomode +chmod 700 ${tmpd}/nomode +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p2 -V f_nomode +has_rights "${tmpd}/nomode" "600" + ## user confirmation expected ## different mode echo "different mode" From fcbe750d42922abbe178c81b6406df534f9c43db Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 15 Nov 2020 10:58:01 +0100 Subject: [PATCH 31/82] more tests --- tests-ng/chmod-more.sh | 127 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100755 tests-ng/chmod-more.sh diff --git a/tests-ng/chmod-more.sh b/tests-ng/chmod-more.sh new file mode 100755 index 0000000..f78a719 --- /dev/null +++ b/tests-ng/chmod-more.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2020, deadc0de6 +# +# test chmod on import +# with files and directories +# with different link +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# $1 path +# $2 rights +has_rights() +{ + echo "testing ${1} is ${2}" + [ ! -e "$1" ] && echo "`basename $1` does not exist" && exit 1 + local mode=`stat -L -c '%a' "$1"` + [ "${mode}" != "$2" ] && echo "bad mode for `basename $1` (${mode} instead of ${2})" && exit 1 + true +} + +# $1 file +chmod_to_umask() +{ + u=`umask` + u=`echo ${u} | sed 's/^0*//'` + if [ -d ${1} ]; then + v=$((777 - u)) + else + v=$((666 - u)) + fi + chmod ${v} ${1} +} + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +#echo "dotfile destination: ${tmpd}" + +# create the dotfiles +f1="${tmpd}/f1" +touch ${f1} +chmod 777 ${f1} +stat -c '%a' ${f1} + +f2="${tmpd}/f2" +touch ${f2} +chmod 644 ${f2} +stat -c '%a' ${f2} + +toimport="${f1} ${f2}" + +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: +profiles: +_EOF +#cat ${cfg} + +# import without --preserve-mode +for i in ${toimport}; do + stat -c '%a' ${i} + cd ${ddpath} | ${bin} import -c ${cfg} -f -p p1 -V ${i} +done + +cat ${cfg} + +has_rights "${tmpd}/f1" "777" +has_rights "${tmps}/dotfiles/${tmpd}/f1" "777" +has_rights "${tmpd}/f2" "644" +has_rights "${tmps}/dotfiles/${tmpd}/f2" "644" + +# install +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 -V | grep '0 dotfile(s) installed' || (echo "should not install" && exit 1) + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 From c967776f8ce1de7eff8b21b0f16244821a378a05 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 15 Nov 2020 10:58:11 +0100 Subject: [PATCH 32/82] mode is int --- dotdrop/utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 0c989e9..4d6b06d 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -326,7 +326,6 @@ def get_file_perm(path): def chmod(path, mode, debug=False): - cm = int(mode, 8) if debug: - LOG.dbg('chmod {} {}'.format(oct(cm), path)) - os.chmod(path, cm) + LOG.dbg('chmod {} {}'.format(oct(mode), path)) + os.chmod(path, mode) From 908a006adc993111fbccbd6833d56dfe6d7e662d Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 15 Nov 2020 10:58:20 +0100 Subject: [PATCH 33/82] add debug header --- dotdrop/options.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dotdrop/options.py b/dotdrop/options.py index 0cbe1c6..2c5cf93 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -131,6 +131,9 @@ class Options(AttrMonitor): if not self.confpath: raise YamlException('no config file found') if self.debug: + self.log.dbg('#################################################') + self.log.dbg('#################### DOTDROP ####################') + self.log.dbg('#################################################') self.log.dbg('version: {}'.format(VERSION)) self.log.dbg('command: {}'.format(' '.join(sys.argv))) self.log.dbg('config file: {}'.format(self.confpath)) From a3c6abd32be637b28c08ac0bee5c87a270283ee2 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 15 Nov 2020 10:58:46 +0100 Subject: [PATCH 34/82] adapt question to user for mode change --- dotdrop/installer.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 0fc32fa..ea3f4bf 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -578,6 +578,10 @@ class Installer: # broken symlink err = 'broken symlink {}'.format(dst) return -1, err + if self.debug: + d = 'src mode {:o}, dst mode {:o}' + self.log.dbg(d.format(rights, dstrights)) + diff = None if self.diff: diff = self._diff_before_write(src, dst, @@ -748,15 +752,17 @@ class Installer: return True if not os.path.exists(src): return True - cperms = utils.get_file_perm(src) - perms = utils.get_file_perm(dst) - if perms == cperms: + sperms = utils.get_file_perm(src) + dperms = utils.get_file_perm(dst) + if sperms == dperms: return True elif self.safe: + # this only happens if no + # chmod is provided + # and dst/src modes differ if self.safe: - msg = 'Mode differs ({:o} and {:o} ({})' - msg = msg.format(cperms, perms, dst) - msg += ', continue' + msg = 'Set mode {:o} to \"{}\"' + msg = msg.format(sperms, dst) if not self.log.ask(msg): return False return True From fbc669a66c9c89edbb8e80f0bb65f7703ecb7761 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 15 Nov 2020 10:59:06 +0100 Subject: [PATCH 35/82] more debug logs --- dotdrop/dotdrop.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 32f957c..85a0471 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -523,7 +523,12 @@ def cmd_importer(o): chmod = None dflperm = get_default_file_perms(dst, o.umask) + + if o.debug: + LOG.dbg('import mode: {}'.format(o.import_mode)) if o.import_mode or perm != dflperm: + if o.debug: + LOG.dbg('adopt mode {:o} (umask {:o})'.format(perm, dflperm)) # insert chmod chmod = perm retconf = o.conf.new(src, dst, linktype, chmod=chmod) From 687f0cdaab2ad8433c99128fbb2432b08173d14e Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 15 Nov 2020 10:59:11 +0100 Subject: [PATCH 36/82] mode is int --- dotdrop/cfg_yaml.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 2d83512..84127ef 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -326,9 +326,7 @@ class CfgYaml: self._dbg('new dotfile src: {}'.format(src)) self._dbg('new dotfile dst: {}'.format(dst)) self._dbg('new dotfile link: {}'.format(link)) - if chmod: - self._dbg('new dotfile chmod: {}'.format(chmod)) - + self._dbg('new dotfile chmod: {}'.format(chmod)) df_dict = { self.key_dotfile_src: src, self.key_dotfile_dst: dst, @@ -655,7 +653,7 @@ class CfgYaml: err = 'bad format for chmod: {}'.format(val) self._log.err(err) raise YamlException('config content error: {}'.format(err)) - v[self.key_dotfile_chmod] = val + v[self.key_dotfile_chmod] = int(val, 8) return new From 944afec604a97aaf0be51e0f506805355eb5e80e Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 15 Nov 2020 15:16:37 +0100 Subject: [PATCH 37/82] ensure chmod entry added on update --- tests-ng/chmod-update.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests-ng/chmod-update.sh b/tests-ng/chmod-update.sh index 06cff60..6e96a66 100755 --- a/tests-ng/chmod-update.sh +++ b/tests-ng/chmod-update.sh @@ -87,13 +87,14 @@ config: dotfiles: profiles: _EOF -#cat ${cfg} # import for i in ${toimport}; do cd ${ddpath} | ${bin} import -c ${cfg} -f -p p1 -V ${i} done +cat ${cfg} + # test no chmod cnt=`cat ${cfg} | grep chmod | wc -l` [ "${cnt}" != "0" ] && echo "chmod wrongly inserted" && exit 1 @@ -106,6 +107,9 @@ cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V ${dnormal} # check rights updated [ "`stat -c '%a' ${tmps}/dotfiles/${tmpd}/$(basename ${dnormal})`" != "777" ] && echo "rights not updated (1)" && exit 1 +cnt=`cat ${cfg} | grep "chmod: '777'" | wc -l` +[ "${cnt}" != "1" ] && echo "chmod not updated (1)" && exit 1 + ###################### # update dlink chmod 777 ${dlink} @@ -113,6 +117,8 @@ cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V ${dlink} # check rights updated [ "`stat -c '%a' ${tmps}/dotfiles/${tmpd}/$(basename ${dlink})`" != "777" ] && echo "rights not updated (2)" && exit 1 +cnt=`cat ${cfg} | grep "chmod: '777'" | wc -l` +[ "${cnt}" != "2" ] && echo "chmod not updated (2)" && exit 1 ###################### # update dlinkchildren @@ -121,6 +127,8 @@ cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V ${dlinkchildren} # check rights updated [ "`stat -c '%a' ${tmps}/dotfiles/${tmpd}/$(basename ${dlinkchildren})`" != "777" ] && echo "rights not updated (3)" && exit 1 +cnt=`cat ${cfg} | grep "chmod: '777'" | wc -l` +[ "${cnt}" != "3" ] && echo "chmod not updated (3)" && exit 1 ###################### # update fnormal @@ -129,6 +137,8 @@ cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V ${fnormal} # check rights updated [ "`stat -c '%a' ${tmps}/dotfiles/${tmpd}/$(basename ${fnormal})`" != "777" ] && echo "rights not updated (4)" && exit 1 +cnt=`cat ${cfg} | grep "chmod: '777'" | wc -l` +[ "${cnt}" != "4" ] && echo "chmod not updated (4)" && exit 1 ###################### # update flink @@ -137,6 +147,8 @@ cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V ${flink} # check rights updated [ "`stat -c '%a' ${tmps}/dotfiles/${tmpd}/$(basename ${flink})`" != "777" ] && echo "rights not updated (5)" && exit 1 +cnt=`cat ${cfg} | grep "chmod: '777'" | wc -l` +[ "${cnt}" != "5" ] && echo "chmod not updated (5)" && exit 1 ## CLEANING rm -rf ${tmps} ${tmpd} From 442d7e53fdc91d955d4f08a4c339b3636f995b94 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 15 Nov 2020 15:16:46 +0100 Subject: [PATCH 38/82] update doc --- docs/config.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/config.md b/docs/config.md index 11110d9..fa5ee18 100644 --- a/docs/config.md +++ b/docs/config.md @@ -82,8 +82,8 @@ On `install` the following rules are applied: On `update`: -* no `chmod` entry is added/updated in the config however the rights are mirrored - from the filesystem to the dotfile in the `dotpath` +* if the permissions of the file in the filesystem differ from the dotfile in the `dotpath` + then the dotfile entry `chmod` is updated accordingly ## Symlink dotfiles From 240e554604f5c47609b006f379ae853f4c29843b Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 15 Nov 2020 15:18:22 +0100 Subject: [PATCH 39/82] ensure file exists for mirroring rights --- dotdrop/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 4d6b06d..eeaba8a 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -300,6 +300,8 @@ def dependencies_met(): def mirror_file_rights(src, dst): """mirror file rights of src to dst (can rise exc)""" + if not os.path.exists(src) or not os.path.exists(dst): + return rights = get_file_perm(src) os.chmod(dst, rights) From aa5bbb60892439e9ee8a472a67d9c39bee69d5cd Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 15 Nov 2020 15:18:46 +0100 Subject: [PATCH 40/82] refactor and add ability to update dotfile entry (chmod) --- dotdrop/cfg_aggregator.py | 413 ++++++++++++++++++++------------------ dotdrop/cfg_yaml.py | 22 ++ dotdrop/dotdrop.py | 7 +- dotdrop/updater.py | 48 +++-- 4 files changed, 275 insertions(+), 215 deletions(-) diff --git a/dotdrop/cfg_aggregator.py b/dotdrop/cfg_aggregator.py index 779f281..3853386 100644 --- a/dotdrop/cfg_aggregator.py +++ b/dotdrop/cfg_aggregator.py @@ -43,103 +43,9 @@ class CfgAggregator: self.log = Logger() self._load() - def _load(self): - """load lower level config""" - self.cfgyaml = CfgYaml(self.path, - self.profile_key, - debug=self.debug) - - # settings - self.settings = Settings.parse(None, self.cfgyaml.settings) - - # dotfiles - self.dotfiles = Dotfile.parse_dict(self.cfgyaml.dotfiles) - if self.debug: - self._debug_list('dotfiles', self.dotfiles) - - # profiles - self.profiles = Profile.parse_dict(self.cfgyaml.profiles) - if self.debug: - self._debug_list('profiles', self.profiles) - - # actions - self.actions = Action.parse_dict(self.cfgyaml.actions) - if self.debug: - self._debug_list('actions', self.actions) - - # trans_r - self.trans_r = Transform.parse_dict(self.cfgyaml.trans_r) - if self.debug: - self._debug_list('trans_r', self.trans_r) - - # trans_w - self.trans_w = Transform.parse_dict(self.cfgyaml.trans_w) - if self.debug: - self._debug_list('trans_w', self.trans_w) - - # variables - self.variables = self.cfgyaml.variables - if self.debug: - self._debug_dict('variables', self.variables) - - # patch dotfiles in profiles - self._patch_keys_to_objs(self.profiles, - "dotfiles", self.get_dotfile) - - # patch action in dotfiles actions - self._patch_keys_to_objs(self.dotfiles, - "actions", self._get_action_w_args) - # patch action in profiles actions - self._patch_keys_to_objs(self.profiles, - "actions", self._get_action_w_args) - - # patch actions in settings default_actions - self._patch_keys_to_objs([self.settings], - "default_actions", self._get_action_w_args) - if self.debug: - msg = 'default actions: {}'.format(self.settings.default_actions) - self.log.dbg(msg) - - # patch trans_w/trans_r in dotfiles - self._patch_keys_to_objs(self.dotfiles, - "trans_r", - self._get_trans_w_args(self._get_trans_r), - islist=False) - self._patch_keys_to_objs(self.dotfiles, - "trans_w", - self._get_trans_w_args(self._get_trans_w), - islist=False) - - def _patch_keys_to_objs(self, containers, keys, get_by_key, islist=True): - """ - map for each key in the attribute 'keys' in 'containers' - the returned object from the method 'get_by_key' - """ - if not containers: - return - if self.debug: - self.log.dbg('patching {} ...'.format(keys)) - for c in containers: - objects = [] - okeys = getattr(c, keys) - if not okeys: - continue - if not islist: - okeys = [okeys] - for k in okeys: - o = get_by_key(k) - if not o: - err = '{} does not contain'.format(c) - err += ' a {} entry named {}'.format(keys, k) - self.log.err(err) - raise Exception(err) - objects.append(o) - if not islist: - objects = objects[0] - # if self.debug: - # er = 'patching {}.{} with {}' - # self.log.dbg(er.format(c, keys, objects)) - setattr(c, keys, objects) + ######################################################## + # public methods + ######################################################## def del_dotfile(self, dotfile): """remove this dotfile from the config""" @@ -149,18 +55,7 @@ class CfgAggregator: """remove this dotfile from this profile""" return self.cfgyaml.del_dotfile_from_profile(dotfile.key, profile.key) - def _create_new_dotfile(self, src, dst, link, chmod=None): - """create a new dotfile""" - # get a new dotfile with a unique key - key = self._get_new_dotfile_key(dst) - if self.debug: - self.log.dbg('new dotfile key: {}'.format(key)) - # add the dotfile - if not self.cfgyaml.add_dotfile(key, src, dst, link, chmod=chmod): - return None - return Dotfile(key, dst, src) - - def new(self, src, dst, link, chmod=None): + def new_dotfile(self, src, dst, link, chmod=None): """ import a new dotfile @src: path in dotpath @@ -182,82 +77,16 @@ class CfgAggregator: msg = 'new dotfile {} to profile {}' self.log.dbg(msg.format(key, self.profile_key)) - self.save() - if ret and not self.dry: - # reload - if self.debug: - self.log.dbg('reloading config') - olddebug = self.debug - self.debug = False - self._load() - self.debug = olddebug + if ret: + self._save_and_reload() return ret - def _get_new_dotfile_key(self, dst): - """return a new unique dotfile key""" - path = os.path.expanduser(dst) - existing_keys = self.cfgyaml.get_all_dotfile_keys() - if self.settings.longkey: - return self._get_long_key(path, existing_keys) - return self._get_short_key(path, existing_keys) - - def _norm_key_elem(self, elem): - """normalize path element for sanity""" - elem = elem.lstrip('.') - elem = elem.replace(' ', '-') - return elem.lower() - - def _split_path_for_key(self, path): - """return a list of path elements, excluded home path""" - p = strip_home(path) - dirs = [] - while True: - p, f = os.path.split(p) - dirs.append(f) - if not p or not f: - break - dirs.reverse() - # remove empty entries - dirs = filter(None, dirs) - # normalize entries - return list(map(self._norm_key_elem, dirs)) - - def _get_long_key(self, path, keys): - """ - return a unique long key representing the - absolute path of path - """ - dirs = self._split_path_for_key(path) - prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix - key = self.key_sep.join([prefix] + dirs) - return self._uniq_key(key, keys) - - def _get_short_key(self, path, keys): - """ - return a unique key where path - is known not to be an already existing dotfile - """ - dirs = self._split_path_for_key(path) - dirs.reverse() - prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix - entries = [] - for d in dirs: - entries.insert(0, d) - key = self.key_sep.join([prefix] + entries) - if key not in keys: - return key - return self._uniq_key(key, keys) - - def _uniq_key(self, key, keys): - """unique dotfile key""" - newkey = key - cnt = 1 - while newkey in keys: - # if unable to get a unique path - # get a random one - newkey = self.key_sep.join([key, str(cnt)]) - cnt += 1 - return newkey + def update_dotfile(self, key, chmod): + """update an existing dotfile""" + ret = self.cfgyaml.update_dotfile(key, chmod) + if ret: + self._save_and_reload() + return ret def path_to_dotfile_dst(self, path): """normalize the path to match dotfile dst""" @@ -358,6 +187,216 @@ class CfgAggregator: except StopIteration: return None + ######################################################## + # accessors for public methods + ######################################################## + + def _create_new_dotfile(self, src, dst, link, chmod=None): + """create a new dotfile""" + # get a new dotfile with a unique key + key = self._get_new_dotfile_key(dst) + if self.debug: + self.log.dbg('new dotfile key: {}'.format(key)) + # add the dotfile + if not self.cfgyaml.add_dotfile(key, src, dst, link, chmod=chmod): + return None + return Dotfile(key, dst, src) + + ######################################################## + # parsing + ######################################################## + + def _load(self): + """load lower level config""" + self.cfgyaml = CfgYaml(self.path, + self.profile_key, + debug=self.debug) + + # settings + self.settings = Settings.parse(None, self.cfgyaml.settings) + + # dotfiles + self.dotfiles = Dotfile.parse_dict(self.cfgyaml.dotfiles) + if self.debug: + self._debug_list('dotfiles', self.dotfiles) + + # profiles + self.profiles = Profile.parse_dict(self.cfgyaml.profiles) + if self.debug: + self._debug_list('profiles', self.profiles) + + # actions + self.actions = Action.parse_dict(self.cfgyaml.actions) + if self.debug: + self._debug_list('actions', self.actions) + + # trans_r + self.trans_r = Transform.parse_dict(self.cfgyaml.trans_r) + if self.debug: + self._debug_list('trans_r', self.trans_r) + + # trans_w + self.trans_w = Transform.parse_dict(self.cfgyaml.trans_w) + if self.debug: + self._debug_list('trans_w', self.trans_w) + + # variables + self.variables = self.cfgyaml.variables + if self.debug: + self._debug_dict('variables', self.variables) + + # patch dotfiles in profiles + self._patch_keys_to_objs(self.profiles, + "dotfiles", self.get_dotfile) + + # patch action in dotfiles actions + self._patch_keys_to_objs(self.dotfiles, + "actions", self._get_action_w_args) + # patch action in profiles actions + self._patch_keys_to_objs(self.profiles, + "actions", self._get_action_w_args) + + # patch actions in settings default_actions + self._patch_keys_to_objs([self.settings], + "default_actions", self._get_action_w_args) + if self.debug: + msg = 'default actions: {}'.format(self.settings.default_actions) + self.log.dbg(msg) + + # patch trans_w/trans_r in dotfiles + self._patch_keys_to_objs(self.dotfiles, + "trans_r", + self._get_trans_w_args(self._get_trans_r), + islist=False) + self._patch_keys_to_objs(self.dotfiles, + "trans_w", + self._get_trans_w_args(self._get_trans_w), + islist=False) + + def _patch_keys_to_objs(self, containers, keys, get_by_key, islist=True): + """ + map for each key in the attribute 'keys' in 'containers' + the returned object from the method 'get_by_key' + """ + if not containers: + return + if self.debug: + self.log.dbg('patching {} ...'.format(keys)) + for c in containers: + objects = [] + okeys = getattr(c, keys) + if not okeys: + continue + if not islist: + okeys = [okeys] + for k in okeys: + o = get_by_key(k) + if not o: + err = '{} does not contain'.format(c) + err += ' a {} entry named {}'.format(keys, k) + self.log.err(err) + raise Exception(err) + objects.append(o) + if not islist: + objects = objects[0] + # if self.debug: + # er = 'patching {}.{} with {}' + # self.log.dbg(er.format(c, keys, objects)) + setattr(c, keys, objects) + + ######################################################## + # dotfile key + ######################################################## + + def _get_new_dotfile_key(self, dst): + """return a new unique dotfile key""" + path = os.path.expanduser(dst) + existing_keys = self.cfgyaml.get_all_dotfile_keys() + if self.settings.longkey: + return self._get_long_key(path, existing_keys) + return self._get_short_key(path, existing_keys) + + def _norm_key_elem(self, elem): + """normalize path element for sanity""" + elem = elem.lstrip('.') + elem = elem.replace(' ', '-') + return elem.lower() + + def _get_long_key(self, path, keys): + """ + return a unique long key representing the + absolute path of path + """ + dirs = self._split_path_for_key(path) + prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix + key = self.key_sep.join([prefix] + dirs) + return self._uniq_key(key, keys) + + def _get_short_key(self, path, keys): + """ + return a unique key where path + is known not to be an already existing dotfile + """ + dirs = self._split_path_for_key(path) + dirs.reverse() + prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix + entries = [] + for d in dirs: + entries.insert(0, d) + key = self.key_sep.join([prefix] + entries) + if key not in keys: + return key + return self._uniq_key(key, keys) + + def _uniq_key(self, key, keys): + """unique dotfile key""" + newkey = key + cnt = 1 + while newkey in keys: + # if unable to get a unique path + # get a random one + newkey = self.key_sep.join([key, str(cnt)]) + cnt += 1 + return newkey + + ######################################################## + # helpers + ######################################################## + + def _save_and_reload(self): + if self.dry: + return + self.save() + if self.debug: + self.log.dbg('reloading config') + olddebug = self.debug + self.debug = False + self._load() + self.debug = olddebug + + def _norm_path(self, path): + if not path: + return path + path = os.path.expanduser(path) + path = os.path.expandvars(path) + path = os.path.abspath(path) + return path + + def _split_path_for_key(self, path): + """return a list of path elements, excluded home path""" + p = strip_home(path) + dirs = [] + while True: + p, f = os.path.split(p) + dirs.append(f) + if not p or not f: + break + dirs.reverse() + # remove empty entries + dirs = filter(None, dirs) + # normalize entries + return list(map(self._norm_key_elem, dirs)) + def _get_action(self, key): """return action by key""" try: @@ -409,14 +448,6 @@ class CfgAggregator: except StopIteration: return None - def _norm_path(self, path): - if not path: - return path - path = os.path.expanduser(path) - path = os.path.expandvars(path) - path = os.path.abspath(path) - return path - def _debug_list(self, title, elems): """pretty print list""" if not self.debug: diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 84127ef..16461c5 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -317,6 +317,28 @@ class CfgYaml: """return all existing dotfile keys""" return self.dotfiles.keys() + def update_dotfile(self, key, chmod): + """update an existing dotfile""" + if key not in self.dotfiles.keys(): + return False + df = self._yaml_dict[self.key_dotfiles][key] + old = None + if self.key_dotfile_chmod in df: + old = df[self.key_dotfile_chmod] + if old == chmod: + return False + if self._debug: + self._dbg('update dotfile: {}'.format(key)) + self._dbg('old chmod value: {}'.format(old)) + self._dbg('new chmod value: {}'.format(chmod)) + df = self._yaml_dict[self.key_dotfiles][key] + if not chmod: + del df[self.key_dotfile_chmod] + else: + df[self.key_dotfile_chmod] = str(format(chmod, 'o')) + self._dirty = True + return True + def add_dotfile(self, key, src, dst, link, chmod=None): """add a new dotfile""" if key in self.dotfiles.keys(): diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 85a0471..bb9f410 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -382,10 +382,7 @@ def cmd_update(o): if o.debug: LOG.dbg('dotfile to update: {}'.format(paths)) - updater = Updater(o.dotpath, o.variables, - o.conf.get_dotfile, - o.conf.get_dotfile_by_dst, - o.conf.path_to_dotfile_dst, + updater = Updater(o.dotpath, o.variables, o.conf, dry=o.dry, safe=o.safe, debug=o.debug, ignore=ignore, showpatch=showpatch) if not iskey: @@ -531,7 +528,7 @@ def cmd_importer(o): LOG.dbg('adopt mode {:o} (umask {:o})'.format(perm, dflperm)) # insert chmod chmod = perm - retconf = o.conf.new(src, dst, linktype, chmod=chmod) + retconf = o.conf.new_dotfile(src, dst, linktype, chmod=chmod) if retconf: LOG.sub('\"{}\" imported'.format(path)) cnt += 1 diff --git a/dotdrop/updater.py b/dotdrop/updater.py index fcbe568..e6cbf3c 100644 --- a/dotdrop/updater.py +++ b/dotdrop/updater.py @@ -22,17 +22,13 @@ TILD = '~' class Updater: - def __init__(self, dotpath, variables, - dotfile_key_getter, dotfile_dst_getter, - dotfile_path_normalizer, - dry=False, safe=True, - debug=False, ignore=[], showpatch=False): + def __init__(self, dotpath, variables, conf, + dry=False, safe=True, debug=False, + ignore=[], showpatch=False): """constructor @dotpath: path where dotfiles are stored @variables: dictionary of variables for the templates - @dotfile_key_getter: func to get a dotfile by key - @dotfile_dst_getter: func to get a dotfile by dst - @dotfile_path_normalizer: func to normalize dotfile dst + @conf: configuration manager @dry: simulate @safe: ask for overwrite if True @debug: enable debug @@ -41,9 +37,7 @@ class Updater: """ self.dotpath = dotpath self.variables = variables - self.dotfile_key_getter = dotfile_key_getter - self.dotfile_dst_getter = dotfile_dst_getter - self.dotfile_path_normalizer = dotfile_path_normalizer + self.conf = conf self.dry = dry self.safe = safe self.debug = debug @@ -62,7 +56,7 @@ class Updater: if not os.path.lexists(path): self.log.err('\"{}\" does not exist!'.format(path)) return False - dotfiles = self.dotfile_dst_getter(path) + dotfiles = self.conf.get_dotfile_by_dst(path) if not dotfiles: return False for dotfile in dotfiles: @@ -80,12 +74,12 @@ class Updater: def update_key(self, key): """update the dotfile referenced by key""" - dotfile = self.dotfile_key_getter(key) + dotfile = self.conf.get_dotfile(key) if not dotfile: return False if self.debug: self.log.dbg('updating {} from key \"{}\"'.format(dotfile, key)) - path = self.dotfile_path_normalizer(dotfile.dst) + path = self.conf.path_to_dotfile_dst(dotfile.dst) return self._update(path, dotfile) def _update(self, path, dotfile): @@ -108,10 +102,26 @@ class Updater: new_path = self._apply_trans_w(path, dotfile) if not new_path: return False + + # save current rights + fsmode = get_file_perm(path) + dfmode = get_file_perm(dtpath) + + # handle the pointed file if os.path.isdir(new_path): ret = self._handle_dir(new_path, dtpath) else: ret = self._handle_file(new_path, dtpath) + + if fsmode != dfmode: + # mirror rights + if self.debug: + m = 'adopt mode {:o} for {}' + self.log.dbg(m.format(fsmode, dotfile.key)) + r = self.conf.update_dotfile(dotfile.key, fsmode) + if r: + ret = True + # clean temporary files if new_path != path and os.path.exists(new_path): removepath(new_path, logger=self.log) @@ -170,12 +180,12 @@ class Updater: return False def _mirror_rights(self, src, dst): + if self.debug: + srcr = get_file_perm(src) + dstr = get_file_perm(dst) + msg = 'copy rights from {} ({:o}) to {} ({:o})' + self.log.dbg(msg.format(src, srcr, dst, dstr)) try: - if self.debug: - srcr = get_file_perm(src) - dstr = get_file_perm(dst) - msg = 'copy rights from {} ({:o}) to {} ({:o})' - self.log.dbg(msg.format(src, srcr, dst, dstr)) mirror_file_rights(src, dst) except OSError as e: self.log.err(e) From 87f32478d90297efbc8e7e4e580fcc5143e5b1cf Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 15 Nov 2020 17:49:57 +0100 Subject: [PATCH 41/82] add more logs --- dotdrop/dotdrop.py | 15 ++++++++++++++- dotdrop/updater.py | 6 ++++-- tests-ng/diff-cmd.sh | 6 +++--- tests-ng/global-update-ignore.sh | 5 +++-- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index bb9f410..bce26ae 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -254,6 +254,7 @@ def cmd_install(o): def cmd_compare(o, tmp): """compare dotfiles and return True if all identical""" + cnt = 0 dotfiles = o.dotfiles if not dotfiles: msg = 'no dotfile defined for this profile (\"{}\")' @@ -266,6 +267,7 @@ def cmd_compare(o, tmp): selected = _select(o.compare_focus, dotfiles) if len(selected) < 1: + LOG.log('\nno dotfile to compare') return False t = _get_templater(o) @@ -281,6 +283,8 @@ def cmd_compare(o, tmp): if not dotfile.src and not dotfile.dst: # ignore fake dotfile continue + cnt += 1 + # add dotfile variables t.restore_vars(tvars) newvars = dotfile.get_dotfile_variables() @@ -355,12 +359,14 @@ def cmd_compare(o, tmp): LOG.emph(diff) same = False + LOG.log('\n{} dotfile(s) compared.'.format(cnt)) return same def cmd_update(o): """update the dotfile(s) from path(s) or key(s)""" ret = True + cnt = 0 paths = o.update_path iskey = o.update_iskey ignore = o.update_ignore @@ -374,10 +380,11 @@ def cmd_update(o): paths = [d.dst for d in o.dotfiles] msg = 'Update all dotfiles for profile \"{}\"'.format(o.profile) if o.safe and not LOG.ask(msg): + LOG.log('\n{} file(s) updated.'.format(cnt)) return False if not paths: - LOG.log('no dotfile to update') + LOG.log('\nno dotfile to update') return True if o.debug: LOG.dbg('dotfile to update: {}'.format(paths)) @@ -385,6 +392,7 @@ def cmd_update(o): updater = Updater(o.dotpath, o.variables, o.conf, dry=o.dry, safe=o.safe, debug=o.debug, ignore=ignore, showpatch=showpatch) + cnt = 0 if not iskey: # update paths if o.debug: @@ -392,6 +400,8 @@ def cmd_update(o): for path in paths: if not updater.update_path(path): ret = False + else: + cnt += 1 else: # update keys keys = paths @@ -403,6 +413,9 @@ def cmd_update(o): for key in keys: if not updater.update_key(key): ret = False + else: + cnt += 1 + LOG.log('\n{} file(s) updated.'.format(cnt)) return ret diff --git a/dotdrop/updater.py b/dotdrop/updater.py index e6cbf3c..15c8830 100644 --- a/dotdrop/updater.py +++ b/dotdrop/updater.py @@ -180,9 +180,11 @@ class Updater: return False def _mirror_rights(self, src, dst): + srcr = get_file_perm(src) + dstr = get_file_perm(dst) + if srcr == dstr: + return if self.debug: - srcr = get_file_perm(src) - dstr = get_file_perm(dst) msg = 'copy rights from {} ({:o}) to {} ({:o})' self.log.dbg(msg.format(src, srcr, dst, dstr)) try: diff --git a/tests-ng/diff-cmd.sh b/tests-ng/diff-cmd.sh index 166135d..f44e11f 100755 --- a/tests-ng/diff-cmd.sh +++ b/tests-ng/diff-cmd.sh @@ -81,7 +81,7 @@ echo "modified" > ${tmpd}/singlefile # default diff (unified) echo "[+] comparing with default diff (unified)" set +e -cd ${ddpath} | ${bin} compare -c ${cfg} 2>&1 | grep -v '=>' | grep -v '^+++\|^---' > ${tmpd}/normal +cd ${ddpath} | ${bin} compare -c ${cfg} 2>&1 | grep -v '=>' | grep -v 'dotfile(s) compared' | sed '$d' | grep -v '^+++\|^---' > ${tmpd}/normal diff -u -r ${tmpd}/singlefile ${basedir}/dotfiles/${tmpd}/singlefile | grep -v '^+++\|^---' > ${tmpd}/real set -e @@ -96,7 +96,7 @@ sed '/dotpath: dotfiles/a \ \ diff_command: "diff -r {0} {1}"' ${cfg} > ${cfg2} # normal diff echo "[+] comparing with normal diff" set +e -cd ${ddpath} | ${bin} compare -c ${cfg2} 2>&1 | grep -v '=>' > ${tmpd}/unified +cd ${ddpath} | ${bin} compare -c ${cfg2} 2>&1 | grep -v '=>' | grep -v 'dotfile(s) compared' | sed '$d' > ${tmpd}/unified diff -r ${tmpd}/singlefile ${basedir}/dotfiles/${tmpd}/singlefile > ${tmpd}/real set -e @@ -113,7 +113,7 @@ sed '/dotpath: dotfiles/a \ \ diff_command: "echo fakediff"' ${cfg} > ${cfg3} # fake diff echo "[+] comparing with fake diff" set +e -cd ${ddpath} | ${bin} compare -c ${cfg3} 2>&1 | grep -v '=>' > ${tmpd}/fake +cd ${ddpath} | ${bin} compare -c ${cfg3} 2>&1 | grep -v '=>' | grep -v 'dotfile(s) compared' | sed '$d' > ${tmpd}/fake set -e # verify diff --git a/tests-ng/global-update-ignore.sh b/tests-ng/global-update-ignore.sh index 5b4c302..93c6971 100755 --- a/tests-ng/global-update-ignore.sh +++ b/tests-ng/global-update-ignore.sh @@ -99,8 +99,9 @@ cd ${ddpath} | ${bin} update -f -c ${cfg} --verbose --profile=p1 --key f_abc # check files haven't been updated [ ! -e ${dt}/a/c/acfile ] && echo "acfile not found" && exit 1 -cat ${dt}/a/c/acfile -grep 'b' ${dt}/a/c/acfile >/dev/null +set +e +grep 'b' ${dt}/a/c/acfile || (echo "acfile not updated" && exit 1) +set -e [ -e ${dt}/a/newfile ] && echo "newfile found" && exit 1 ## CLEANING From 8ce96ffeebe802ce0949c67087723fa8edaff13f Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 15 Nov 2020 20:27:49 +0100 Subject: [PATCH 42/82] improve diffing --- dotdrop/installer.py | 73 ++++++++++++++++++++++++-------------- dotdrop/utils.py | 6 ++++ tests-ng/actions-pre.sh | 33 ++++++++++------- tests-ng/chmod-install.sh | 2 +- tests-ng/import-configs.sh | 2 +- tests-ng/update-ignore.sh | 11 +++++- tests.sh | 8 +++-- 7 files changed, 91 insertions(+), 44 deletions(-) diff --git a/dotdrop/installer.py b/dotdrop/installer.py index ea3f4bf..27c8004 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -382,7 +382,7 @@ class Installer: self.log.dry('would remove {} and link to {}'.format(dst, src)) return True, None if self.showdiff: - self._diff_before_write(src, dst, quiet=False) + self._show_diff_before_write(src, dst) msg = 'Remove "{}" for link creation?'.format(dst) if self.safe and not self.log.ask(msg): err = 'ignoring "{}", link was not created'.format(dst) @@ -565,43 +565,31 @@ class Installer: return 0, None if os.path.lexists(dst): - if chmod: - rights = chmod - else: - rights = utils.get_file_perm(src) - samerights = False try: - dstrights = utils.get_file_perm(dst) - samerights = dstrights == rights + os.stat(dst) except OSError as e: if e.errno == errno.ENOENT: # broken symlink err = 'broken symlink {}'.format(dst) return -1, err - if self.debug: - d = 'src mode {:o}, dst mode {:o}' - self.log.dbg(d.format(rights, dstrights)) - diff = None + src_mode = chmod + if not src_mode: + src_mode = utils.get_file_perm(src) if self.diff: - diff = self._diff_before_write(src, dst, - content=content, - quiet=True) - if not diff and samerights: + if not self._is_different(src, dst, + content=content, + src_mode=src_mode): if self.debug: self.log.dbg('{} is the same'.format(dst)) return 1, None - if diff and self.safe: + if self.safe: if self.debug: self.log.dbg('change detected for {}'.format(dst)) if self.showdiff: - if diff is None: - # get diff - diff = self._diff_before_write(src, dst, - content=content, - quiet=True) - if diff: - self._print_diff(src, dst, diff) + # get diff + self._show_diff_before_write(src, dst, + content=content) if not self.log.ask('Overwrite \"{}\"'.format(dst)): self.log.warn('ignoring {}'.format(dst)) return 1, None @@ -656,7 +644,40 @@ class Installer: tmp['_dotfile_sub_abs_dst'] = dst return tmp - def _diff_before_write(self, src, dst, content=None, quiet=False): + def _is_different(self, src, dst, src_mode=None, content=None): + """ + returns True if file is different and + needs to be installed + """ + # check file size + src_size = os.stat(src).st_size + dst_size = os.stat(dst).st_size + if src_size != dst_size: + if self.debug: + self.log.dbg('size differ') + return True + + # check file mode + if not src_mode: + src_mode = utils.get_file_perm(src) + dst_mode = utils.get_file_perm(dst) + if src_mode != dst_mode: + if self.debug: + m = 'mode differ ({:o} vs {:o})' + self.log.dbg(m.format(src_mode, dst_mode)) + return True + + # check file content + if content: + tmp = utils.write_to_tmpfile(content) + src = tmp + r = utils.fastdiff(src, dst) + if r: + if self.debug: + self.log.dbg('content differ') + return r + + def _show_diff_before_write(self, src, dst, content=None): """ diff before writing using a temp file if content is not None @@ -671,7 +692,7 @@ class Installer: if tmp: utils.removepath(tmp, logger=self.log) - if not quiet and diff: + if diff: self._print_diff(src, dst, diff) return diff diff --git a/dotdrop/utils.py b/dotdrop/utils.py index eeaba8a..33c4a8b 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -12,6 +12,7 @@ import uuid import fnmatch import inspect import importlib +import filecmp from shutil import rmtree, which # local import @@ -73,6 +74,11 @@ def shell(cmd, debug=False): return ret == 0, out +def fastdiff(left, right): + """fast compare files and returns True if different""" + return not filecmp.cmp(left, right, shallow=False) + + def diff(original, modified, raw=True, diff_cmd='', debug=False): """compare two files, returns '' if same""" diff --git a/tests-ng/actions-pre.sh b/tests-ng/actions-pre.sh index 24ae575..5f8949b 100755 --- a/tests-ng/actions-pre.sh +++ b/tests-ng/actions-pre.sh @@ -46,6 +46,15 @@ echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" # this is the test ################################################################ +# $1 pattern +# $2 path +grep_or_fail() +{ + set +e + grep "${1}" "${2}" >/dev/null 2>&1 || (echo "pattern not found in ${2}" && exit 1) + set -e +} + # the action temp tmpa=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` # the dotfile source @@ -136,38 +145,36 @@ cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V # checks [ ! -e ${tmpa}/pre ] && echo 'pre action not executed' && exit 1 -grep pre ${tmpa}/pre >/dev/null +grep_or_fail pre ${tmpa}/pre [ ! -e ${tmpa}/naked ] && echo 'naked action not executed' && exit 1 -grep naked ${tmpa}/naked >/dev/null +grep_or_fail naked ${tmpa}/naked [ ! -e ${tmpa}/multiple ] && echo 'pre action multiple not executed' && exit 1 -grep multiple ${tmpa}/multiple >/dev/null +grep_or_fail multiple ${tmpa}/multiple [ "`wc -l ${tmpa}/multiple | awk '{print $1}'`" -gt "1" ] && echo 'pre action multiple executed twice' && exit 1 [ ! -e ${tmpa}/pre2 ] && echo 'pre action 2 not executed' && exit 1 -grep pre2 ${tmpa}/pre2 >/dev/null +grep_or_fail pre2 ${tmpa}/pre2 [ ! -e ${tmpa}/naked2 ] && echo 'naked action 2 not executed' && exit 1 -grep naked2 ${tmpa}/naked2 >/dev/null +grep_or_fail naked2 ${tmpa}/naked2 [ ! -e ${tmpa}/multiple2 ] && echo 'pre action multiple 2 not executed' && exit 1 -grep multiple2 ${tmpa}/multiple2 >/dev/null +grep_or_fail multiple2 ${tmpa}/multiple2 [ "`wc -l ${tmpa}/multiple2 | awk '{print $1}'`" -gt "1" ] && echo 'pre action multiple 2 executed twice' && exit 1 [ ! -e ${tmpa}/naked3 ] && echo 'naked action 3 not executed' && exit 1 -grep naked3 ${tmpa}/naked3 >/dev/null +grep_or_fail naked3 ${tmpa}/naked3 - -# remove the pre action result and re-run +# remove the pre action result and re-install rm ${tmpa}/pre - -cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -[ -e ${tmpa}/pre ] && exit 1 +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V +[ -e ${tmpa}/pre ] && echo "pre exists" && exit 1 # ensure failing actions make the installation fail # install set +e cd ${ddpath} | ${bin} install -f -c ${cfg} -p p2 -V set -e -[ -e ${tmpd}/fail ] && exit 1 +[ -e ${tmpd}/fail ] && echo "fail exists" && exit 1 ## CLEANING rm -rf ${tmps} ${tmpd} ${tmpa} diff --git a/tests-ng/chmod-install.sh b/tests-ng/chmod-install.sh index 67aa009..bd0259e 100755 --- a/tests-ng/chmod-install.sh +++ b/tests-ng/chmod-install.sh @@ -262,7 +262,7 @@ echo "nomode" > ${tmps}/dotfiles/nomode chmod 600 ${tmps}/dotfiles/nomode echo "nomode" > ${tmpd}/nomode chmod 700 ${tmpd}/nomode -cd ${ddpath} | printf 'y\n' | ${bin} install -c ${cfg} -p p2 -V f_nomode +cd ${ddpath} | printf 'y\ny\n' | ${bin} install -c ${cfg} -p p2 -V f_nomode has_rights "${tmpd}/nomode" "600" ## CLEANING diff --git a/tests-ng/import-configs.sh b/tests-ng/import-configs.sh index 4c75d4c..2328fad 100755 --- a/tests-ng/import-configs.sh +++ b/tests-ng/import-configs.sh @@ -172,7 +172,7 @@ profiles: dotfiles: - f_asub _EOF -cd ${ddpath} | ${bin} install -c ${cfg1} -p p2 -V +cd ${ddpath} | ${bin} install -c ${cfg1} -p p2 -V -f cd ${ddpath} | ${bin} compare -c ${cfg1} -p p2 -V ## CLEANING diff --git a/tests-ng/update-ignore.sh b/tests-ng/update-ignore.sh index 9794e36..cf928ce 100755 --- a/tests-ng/update-ignore.sh +++ b/tests-ng/update-ignore.sh @@ -46,6 +46,15 @@ echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" # this is the test ################################################################ +# $1 pattern +# $2 path +grep_or_fail() +{ + set +e + grep "${1}" "${2}" >/dev/null 2>&1 || (echo "pattern not found in ${2}" && exit 1) + set -e +} + # dotdrop directory tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` dt="${tmps}/dotfiles" @@ -98,7 +107,7 @@ cd ${ddpath} | ${bin} update -f -c ${cfg} --verbose --profile=p1 --key f_abc #tree ${dt} # check files haven't been updated -grep 'b' ${dt}/a/c/acfile >/dev/null +grep_or_fail 'b' "${dt}/a/c/acfile" [ -e ${dt}/a/newfile ] && echo "should not have been updated" && exit 1 ## CLEANING diff --git a/tests.sh b/tests.sh index c32ddec..05a4509 100755 --- a/tests.sh +++ b/tests.sh @@ -3,7 +3,8 @@ # Copyright (c) 2017, deadc0de6 # stop on first error -set -ev +#set -ev +set -e # PEP8 tests which pycodestyle >/dev/null 2>&1 @@ -54,9 +55,12 @@ unset DOTDROP_FORCE_NODEBUG [ "$1" = '--python-only' ] || { echo "doing extended tests" logdir=`mktemp -d` + tot=`ls -1 tests-ng/*.sh | wc -l` + cnt=0 for scr in tests-ng/*.sh; do + cnt=$((cnt + 1)) logfile="${logdir}/`basename ${scr}`.log" - echo "-> running test ${scr} (logfile:${logfile})" + echo "-> (${cnt}/${tot}) running test ${scr} (logfile:${logfile})" set +e ${scr} > "${logfile}" 2>&1 if [ "$?" -ne 0 ]; then From 80a9ed4814a71238e70ac36e867915429e4c931c Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 15 Nov 2020 21:18:14 +0100 Subject: [PATCH 43/82] workdir env variable --- docs/usage.md | 4 ++++ dotdrop/settings.py | 7 +++++++ tests-ng/workdir.sh | 1 + tests.sh | 1 + 4 files changed, 13 insertions(+) diff --git a/docs/usage.md b/docs/usage.md index 2cbf1ec..9d3e375 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -255,3 +255,7 @@ export DOTDROP_FORCE_NODEBUG= ```bash export DOTDROP_TMPDIR="/tmp/dotdrop-tmp" ``` +* `DOTDROP_WORKDIR`: overwrite the `workdir` defined in the config +```bash +export DOTDROP_WORKDIR="/tmp/dotdrop-workdir" +``` diff --git a/dotdrop/settings.py b/dotdrop/settings.py index c29e97d..f1062dc 100644 --- a/dotdrop/settings.py +++ b/dotdrop/settings.py @@ -5,11 +5,16 @@ Copyright (c) 2019, deadc0de6 settings block """ +import os + # local imports from dotdrop.linktypes import LinkTypes from dotdrop.dictparser import DictParser +ENV_WORKDIR = 'DOTDROP_WORKDIR' + + class Settings(DictParser): # key in yaml file key_yaml = 'config' @@ -68,6 +73,8 @@ class Settings(DictParser): self.cmpignore = cmpignore self.instignore = instignore self.workdir = workdir + if ENV_WORKDIR in os.environ: + self.workdir = os.environ[ENV_WORKDIR] self.link_dotfile_default = LinkTypes.get(link_dotfile_default) self.link_on_import = LinkTypes.get(link_on_import) self.minversion = minversion diff --git a/tests-ng/workdir.sh b/tests-ng/workdir.sh index dae5751..bdc2d31 100755 --- a/tests-ng/workdir.sh +++ b/tests-ng/workdir.sh @@ -45,6 +45,7 @@ echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" ################################################################ # this is the test ################################################################ +unset DOTDROP_WORKDIR string="blabla" # the dotfile source diff --git a/tests.sh b/tests.sh index 05a4509..741e93f 100755 --- a/tests.sh +++ b/tests.sh @@ -50,6 +50,7 @@ export DOTDROP_DEBUG=yes unset DOTDROP_FORCE_NODEBUG # do not print debugs when running tests (faster) #export DOTDROP_FORCE_NODEBUG=yes +export DOTDROP_WORKDIR=/tmp/dotdrop-tests-workdir ## execute bash script tests [ "$1" = '--python-only' ] || { From 05ffe43e5c5dfad89b890fc88b18588822dc07df Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 15 Nov 2020 23:09:29 +0100 Subject: [PATCH 44/82] improve tests --- tests-ng/tests-launcher.py | 81 ++++++++++++++++++++++++++++++++++++++ tests.sh | 43 ++++++-------------- 2 files changed, 93 insertions(+), 31 deletions(-) create mode 100755 tests-ng/tests-launcher.py diff --git a/tests-ng/tests-launcher.py b/tests-ng/tests-launcher.py new file mode 100755 index 0000000..aa36e8e --- /dev/null +++ b/tests-ng/tests-launcher.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2020, deadc0de6 +# +# tests launcher +# + + +import os +import sys +import subprocess +from concurrent import futures + + +MAX_JOBS = 10 + + +def run_test(path): + cur = os.path.dirname(sys.argv[0]) + name = os.path.basename(path) + path = os.path.join(cur, name) + + p = subprocess.Popen(path, shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + out, _ = p.communicate() + out = out.decode() + r = p.returncode == 0 + reason = 'returncode' + if 'Traceback' in out: + r = False + reason = 'traceback' + return r, reason, path, out + + +def get_tests(): + tests = [] + cur = os.path.dirname(sys.argv[0]) + for (_, _, filenames) in os.walk(cur): + for path in filenames: + if not path.endswith('.sh'): + continue + tests.append(path) + break + return tests + + +def main(): + global MAX_JOBS + if len(sys.argv) > 1: + MAX_JOBS = int(sys.argv[1]) + + tests = get_tests() + + with futures.ThreadPoolExecutor(max_workers=MAX_JOBS) as ex: + wait_for = [] + for test in tests: + j = ex.submit(run_test, test) + wait_for.append(j) + + for f in futures.as_completed(wait_for): + r, reason, p, log = f.result() + if not r: + ex.shutdown(wait=False) + for x in wait_for: + x.cancel() + print() + print(log) + print('test {} failed ({})'.format(p, reason)) + return False + else: + sys.stdout.write('.') + sys.stdout.flush() + sys.stdout.write('\n') + return True + + +if __name__ == '__main__': + if not main(): + sys.exit(1) + sys.exit(0) diff --git a/tests.sh b/tests.sh index 741e93f..54c2fba 100755 --- a/tests.sh +++ b/tests.sh @@ -31,16 +31,15 @@ export DOTDROP_FORCE_NODEBUG=yes # coverage file location cur=`dirname $(readlink -f "${0}")` -export COVERAGE_FILE="${cur}/.coverage" # execute tests with coverage if [ -z ${GITHUB_WORKFLOW} ]; then ## local - #PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop + export COVERAGE_FILE= PYTHONPATH="dotdrop" ${nosebin} -s --processes=-1 --with-coverage --cover-package=dotdrop else ## CI/CD - #PYTHONPATH="dotdrop" ${nosebin} --processes=-1 --with-coverage --cover-package=dotdrop + export COVERAGE_FILE="${cur}/.coverage" PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop fi #PYTHONPATH="dotdrop" python3 -m pytest tests @@ -52,34 +51,16 @@ unset DOTDROP_FORCE_NODEBUG #export DOTDROP_FORCE_NODEBUG=yes export DOTDROP_WORKDIR=/tmp/dotdrop-tests-workdir -## execute bash script tests -[ "$1" = '--python-only' ] || { - echo "doing extended tests" - logdir=`mktemp -d` - tot=`ls -1 tests-ng/*.sh | wc -l` - cnt=0 - for scr in tests-ng/*.sh; do - cnt=$((cnt + 1)) - logfile="${logdir}/`basename ${scr}`.log" - echo "-> (${cnt}/${tot}) running test ${scr} (logfile:${logfile})" - set +e - ${scr} > "${logfile}" 2>&1 - if [ "$?" -ne 0 ]; then - cat ${logfile} - echo "test ${scr} finished with error" - rm -rf ${logdir} - exit 1 - elif grep Traceback ${logfile}; then - cat ${logfile} - echo "test ${scr} crashed" - rm -rf ${logdir} - exit 1 - fi - set -e - echo "test ${scr} ok" - done - rm -rf ${logdir} -} +# run bash tests +if [ -z ${GITHUB_WORKFLOW} ]; then + ## local + export COVERAGE_FILE= + tests-ng/tests-launcher.py +else + ## CI/CD + export COVERAGE_FILE="${cur}/.coverage" + tests-ng/tests-launcher.py 1 +fi ## test the doc with remark ## https://github.com/remarkjs/remark-validate-links From 5edcfe189716cdf1a826a710c0cb29a7da7a236f Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 16 Nov 2020 13:27:03 +0100 Subject: [PATCH 45/82] chmod in files and detail commands --- dotdrop/dotdrop.py | 9 +++++---- tests-ng/deprecated-link.sh | 12 ++++++------ tests-ng/import-link-children.sh | 2 +- tests-ng/link-value-tests.sh | 22 +++++++++++----------- 4 files changed, 23 insertions(+), 22 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index bce26ae..7750fdb 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -584,15 +584,17 @@ def cmd_list_files(o): if not Templategen.is_template(src): continue if o.files_grepable: - fmt = '{},dst:{},src:{},link:{}' + fmt = '{},dst:{},src:{},link:{},chmod:{}' fmt = fmt.format(dotfile.key, dotfile.dst, - dotfile.src, dotfile.link.name.lower()) + dotfile.src, dotfile.link.name.lower(), + dotfile.chmod) LOG.raw(fmt) else: LOG.log('{}'.format(dotfile.key), bold=True) LOG.sub('dst: {}'.format(dotfile.dst)) LOG.sub('src: {}'.format(dotfile.src)) LOG.sub('link: {}'.format(dotfile.link.name.lower())) + LOG.sub('chmod: {}'.format(dotfile.chmod)) LOG.log('') @@ -730,8 +732,7 @@ def _detail(dotpath, dotfile): attribs = [] attribs.append('dst: \"{}\"'.format(dotfile.dst)) attribs.append('link: \"{}\"'.format(dotfile.link.name.lower())) - if dotfile.chmod: - attribs.append('chmod: \"{}\"'.format(dotfile.chmod)) + attribs.append('chmod: \"{}\"'.format(dotfile.chmod)) LOG.log('{} ({})'.format(entry, ', '.join(attribs))) path = os.path.join(dotpath, os.path.expanduser(dotfile.src)) if not os.path.isdir(path): diff --git a/tests-ng/deprecated-link.sh b/tests-ng/deprecated-link.sh index b63238b..be96783 100755 --- a/tests-ng/deprecated-link.sh +++ b/tests-ng/deprecated-link.sh @@ -126,12 +126,12 @@ set -e # test values have been correctly updated echo "========> test for updated entries" -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_link' | head -1 | grep ',link:link$' -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_nolink' | head -1 | grep ',link:nolink$' -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_nolink1' | head -1 | grep ',link:nolink$' -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_children' | head -1 | grep ',link:link_children$' -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_children2' | head -1 | grep ',link:link_children$' -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_children3' | head -1 | grep ',link:nolink$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_link' | head -1 | grep ',link:link,' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_nolink' | head -1 | grep ',link:nolink,' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_nolink1' | head -1 | grep ',link:nolink,' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_children' | head -1 | grep ',link:link_children,' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_children2' | head -1 | grep ',link:link_children,' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_children3' | head -1 | grep ',link:nolink,' ## CLEANING rm -rf ${tmps} ${tmpd} diff --git a/tests-ng/import-link-children.sh b/tests-ng/import-link-children.sh index 036e667..309fd54 100755 --- a/tests-ng/import-link-children.sh +++ b/tests-ng/import-link-children.sh @@ -89,7 +89,7 @@ _EOF cd ${ddpath} | ${bin} import -c ${cfg} -p p1 -V --link=link_children ${dt} # check is set to link_children -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "d_`basename ${dt}`" | grep ',link:link_children$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "d_`basename ${dt}`" | grep ',link:link_children,' # checks file exists in dotpath [ ! -e ${dotpath}/${dt} ] && echo "dotfile not imported" && exit 1 diff --git a/tests-ng/link-value-tests.sh b/tests-ng/link-value-tests.sh index 4988e6f..853fade 100755 --- a/tests-ng/link-value-tests.sh +++ b/tests-ng/link-value-tests.sh @@ -80,7 +80,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink,' # try to install rm -rf ${tmpd}/qwert @@ -114,7 +114,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink,' # try to install rm -rf ${tmpd}/qwert @@ -148,7 +148,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V --link=nolink # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink,' # try to install rm -rf ${tmpd}/qwert @@ -182,7 +182,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V --link=link # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:link$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:link,' # try to install rm -rf ${tmpd}/qwert @@ -216,7 +216,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:link$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:link,' # try to install rm -rf ${tmpd}/qwert @@ -250,7 +250,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V --link=nolink # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink,' # try to install rm -rf ${tmpd}/qwert @@ -284,7 +284,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V --link=nolink # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink,' # try to install rm -rf ${tmpd}/qwert @@ -318,7 +318,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V --link=nolink # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink,' # try to install rm -rf ${tmpd}/qwert @@ -350,7 +350,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} --link=link -p p1 ${df} -V # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:link$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:link,' # try to install rm -rf ${tmpd}/qwert @@ -411,7 +411,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} --link=link_children -p p1 ${df} -V # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "d_`basename ${df}`" | head -1 | grep ',link:link_children$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "d_`basename ${df}`" | head -1 | grep ',link:link_children,' # try to install rm -rf ${tmpd}/qwert @@ -451,7 +451,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "d_`basename ${df}`" | head -1 | grep ',link:link_children$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "d_`basename ${df}`" | head -1 | grep ',link:link_children,' # try to install rm -rf ${tmpd}/qwert From 3fc635495d669a49aa3bb76cf3699a0e9fb676e6 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 16 Nov 2020 13:50:38 +0100 Subject: [PATCH 46/82] clean run --- dotdrop/comparator.py | 2 +- dotdrop/installer.py | 2 +- dotdrop/templategen.py | 2 +- dotdrop/utils.py | 13 +++---------- tests-ng/profile-actions.sh | 3 +++ 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/dotdrop/comparator.py b/dotdrop/comparator.py index 8c28266..78c1dd4 100644 --- a/dotdrop/comparator.py +++ b/dotdrop/comparator.py @@ -143,7 +143,7 @@ class Comparator: def _diff(self, left, right, header=False): """diff two files""" - out = diff(modified=left, original=right, raw=False, + out = diff(modified=left, original=right, diff_cmd=self.diff_cmd, debug=self.debug) if header: lshort = os.path.basename(left) diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 27c8004..55588fe 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -687,7 +687,7 @@ class Installer: if content: tmp = utils.write_to_tmpfile(content) src = tmp - diff = utils.diff(modified=src, original=dst, raw=False, + diff = utils.diff(modified=src, original=dst, diff_cmd=self.diff_cmd) if tmp: utils.removepath(tmp, logger=self.log) diff --git a/dotdrop/templategen.py b/dotdrop/templategen.py index d5a718c..3e34622 100644 --- a/dotdrop/templategen.py +++ b/dotdrop/templategen.py @@ -153,7 +153,7 @@ class Templategen: self.log.dbg('using \"magic\" for filetype identification') except ImportError: _, filetype = utils.run(['file', '-b', '--mime-type', src], - raw=False, debug=self.debug) + debug=self.debug) if self.debug: self.log.dbg('using \"file\" for filetype identification') filetype = filetype.strip() diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 33c4a8b..b04be4b 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -33,7 +33,7 @@ DONOTDELETE = [ NOREMOVE = [os.path.normpath(p) for p in DONOTDELETE] -def run(cmd, raw=True, debug=False, checkerr=False): +def run(cmd, debug=False): """run a command (expects a list)""" if debug: LOG.dbg('exec: {}'.format(' '.join(cmd))) @@ -43,13 +43,6 @@ def run(cmd, raw=True, debug=False, checkerr=False): ret = p.returncode out = out.splitlines(keepends=True) lines = ''.join([x.decode('utf-8', 'replace') for x in out]) - if checkerr and ret != 0: - c = ' '.join(cmd) - errl = lines.rstrip() - m = '\"{}\" returned non zero ({}): {}'.format(c, ret, errl) - LOG.err(m) - if raw: - return ret == 0, out return ret == 0, lines @@ -79,7 +72,7 @@ def fastdiff(left, right): return not filecmp.cmp(left, right, shallow=False) -def diff(original, modified, raw=True, +def diff(original, modified, diff_cmd='', debug=False): """compare two files, returns '' if same""" if not diff_cmd: @@ -92,7 +85,7 @@ def diff(original, modified, raw=True, "{modified}": modified, } cmd = [replacements.get(x, x) for x in diff_cmd.split()] - _, out = run(cmd, raw=raw, debug=debug) + _, out = run(cmd, debug=debug) return out diff --git a/tests-ng/profile-actions.sh b/tests-ng/profile-actions.sh index 5840021..307f5cb 100755 --- a/tests-ng/profile-actions.sh +++ b/tests-ng/profile-actions.sh @@ -94,6 +94,9 @@ profiles: _EOF #cat ${cfg} +# list profiles +cd ${ddpath} | ${bin} profiles -c ${cfg} -V + # create the dotfile echo "test" > ${tmps}/dotfiles/abc echo "test" > ${tmps}/dotfiles/def From 7dd8019582dd817cc3eae3f30a76e6a48bc9ff28 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 16 Nov 2020 14:02:35 +0100 Subject: [PATCH 47/82] workers as env variable --- docs/usage.md | 4 ++++ dotdrop/options.py | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 9d3e375..f1dea23 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -259,3 +259,7 @@ export DOTDROP_TMPDIR="/tmp/dotdrop-tmp" ```bash export DOTDROP_WORKDIR="/tmp/dotdrop-workdir" ``` +* `DOTDROP_WORKERS`: overwrite the `-w --workers` cli argument +```bash +export DOTDROP_WORKERS="10" +``` diff --git a/dotdrop/options.py b/dotdrop/options.py index 2c5cf93..bdf74cf 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -25,6 +25,7 @@ ENV_NOBANNER = 'DOTDROP_NOBANNER' ENV_DEBUG = 'DOTDROP_DEBUG' ENV_NODEBUG = 'DOTDROP_FORCE_NODEBUG' ENV_XDG = 'XDG_CONFIG_HOME' +ENV_WORKERS = 'DOTDROP_WORKERS' BACKUP_SUFFIX = '.dotdropbak' PROFILE = socket.gethostname() @@ -247,7 +248,11 @@ class Options(AttrMonitor): if a.kind == Action.post] self.install_ignore = self.instignore try: - self.install_parallel = int(self.args['--workers']) + if ENV_WORKERS in os.environ: + workers = int(os.environ[ENV_WORKERS]) + else: + workers = int(self.args['--workers']) + self.install_parallel = workers except ValueError: self.log.err('bad option for --workers') sys.exit(USAGE) From 315a898e77af53b40dfc817606168490a40187b4 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 16 Nov 2020 14:25:16 +0100 Subject: [PATCH 48/82] enable force on install --- tests-ng/chmod-install.sh | 4 ++-- tests-ng/corner-case.sh | 2 ++ tests-ng/globs.sh | 2 +- tests-ng/import-configs.sh | 2 +- tests-ng/import-with-empty.sh | 4 ++-- tests-ng/include-order.sh | 1 + tests-ng/install-empty.sh | 2 +- tests-ng/install-ignore.sh | 8 ++++---- tests-ng/install-to-temp.sh | 2 +- tests-ng/macro-with-globals.sh | 2 +- 10 files changed, 16 insertions(+), 13 deletions(-) diff --git a/tests-ng/chmod-install.sh b/tests-ng/chmod-install.sh index bd0259e..72853de 100755 --- a/tests-ng/chmod-install.sh +++ b/tests-ng/chmod-install.sh @@ -242,7 +242,7 @@ echo "nomode" > ${tmps}/dotfiles/nomode chmod 600 ${tmps}/dotfiles/nomode echo "nomode" > ${tmpd}/nomode chmod 600 ${tmpd}/nomode -cd ${ddpath} | ${bin} install -c ${cfg} -p p2 -V f_nomode +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p2 -V f_nomode has_rights "${tmpd}/nomode" "600" ## no user confirmation with force @@ -262,7 +262,7 @@ echo "nomode" > ${tmps}/dotfiles/nomode chmod 600 ${tmps}/dotfiles/nomode echo "nomode" > ${tmpd}/nomode chmod 700 ${tmpd}/nomode -cd ${ddpath} | printf 'y\ny\n' | ${bin} install -c ${cfg} -p p2 -V f_nomode +cd ${ddpath} | printf 'y\ny\n' | ${bin} install -f -c ${cfg} -p p2 -V f_nomode has_rights "${tmpd}/nomode" "600" ## CLEANING diff --git a/tests-ng/corner-case.sh b/tests-ng/corner-case.sh index 9c4b037..e5fb665 100755 --- a/tests-ng/corner-case.sh +++ b/tests-ng/corner-case.sh @@ -56,6 +56,8 @@ basedir=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` echo "[+] dotdrop dir: ${basedir}" echo "[+] dotpath dir: ${basedir}/dotfiles" +export DOTDROP_WORKERS=1 + # create the config file cfg="${basedir}/config.yaml" cat > ${cfg} << _EOF diff --git a/tests-ng/globs.sh b/tests-ng/globs.sh index a682961..d38915f 100755 --- a/tests-ng/globs.sh +++ b/tests-ng/globs.sh @@ -98,7 +98,7 @@ mkdir -p ${tmps}/dotfiles/ echo "abc" > ${tmps}/dotfiles/abc # install -cd ${ddpath} | ${bin} install -c ${cfg} -p p1 -V +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 -V # checks [ ! -e ${tmpd}/abc ] && echo "dotfile not installed" && exit 1 diff --git a/tests-ng/import-configs.sh b/tests-ng/import-configs.sh index 2328fad..41a45e8 100755 --- a/tests-ng/import-configs.sh +++ b/tests-ng/import-configs.sh @@ -143,7 +143,7 @@ cd ${ddpath} | ${bin} files -c ${cfg1} -p pup -V | grep f_sub cd ${ddpath} | ${bin} files -c ${cfg1} -p psubsub -V | grep f_sub # test compare too -cd ${ddpath} | ${bin} install -c ${cfg1} -p p2 -V +cd ${ddpath} | ${bin} install -c ${cfg1} -p p2 -V -f cd ${ddpath} | ${bin} compare -c ${cfg1} -p p2 -V # test with non-existing dotpath this time diff --git a/tests-ng/import-with-empty.sh b/tests-ng/import-with-empty.sh index b43fb93..8937313 100755 --- a/tests-ng/import-with-empty.sh +++ b/tests-ng/import-with-empty.sh @@ -97,9 +97,9 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 --verbose ${dftoimport} [ "$?" != "0" ] && exit 1 echo "[+] install" -cd ${ddpath} | ${bin} install -c ${cfg} -p p1 --verbose | grep '^5 dotfile(s) installed.$' +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 --verbose | grep '^5 dotfile(s) installed.$' rm -f ${dftoimport} -cd ${ddpath} | ${bin} install -c ${cfg} -p p1 --verbose | grep '^6 dotfile(s) installed.$' +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 --verbose | grep '^6 dotfile(s) installed.$' nb=`cd ${ddpath} | ${bin} files -c ${cfg} -p p1 --verbose | grep '^[a-zA-Z]' | wc -l` [ "${nb}" != "6" ] && echo 'error in dotfile list' && exit 1 diff --git a/tests-ng/include-order.sh b/tests-ng/include-order.sh index 01c295a..471edf9 100755 --- a/tests-ng/include-order.sh +++ b/tests-ng/include-order.sh @@ -55,6 +55,7 @@ tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` # temporary tmpa=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +export DOTDROP_WORKERS=1 # create the config file cfg="${tmps}/config.yaml" diff --git a/tests-ng/install-empty.sh b/tests-ng/install-empty.sh index 6773ca7..8ca2ff3 100755 --- a/tests-ng/install-empty.sh +++ b/tests-ng/install-empty.sh @@ -87,7 +87,7 @@ profiles: _EOF echo "[+] install" -cd ${ddpath} | ${bin} install -c ${cfg} -p p1 --verbose | grep '^5 dotfile(s) installed.$' +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 --verbose | grep '^5 dotfile(s) installed.$' [ "$?" != "0" ] && exit 1 ## CLEANING diff --git a/tests-ng/install-ignore.sh b/tests-ng/install-ignore.sh index fd62890..76258ec 100755 --- a/tests-ng/install-ignore.sh +++ b/tests-ng/install-ignore.sh @@ -82,7 +82,7 @@ echo "new data" > ${basedir}/dotfiles/${tmpd}/readmes/README.md # install rm -rf ${tmpd} echo "[+] install normal" -cd ${ddpath} | ${bin} install --showdiff -c ${cfg} --verbose +cd ${ddpath} | ${bin} install --showdiff -c ${cfg} --verbose -f [ "$?" != "0" ] && exit 1 nb=`find ${tmpd} -iname 'README.md' | wc -l` echo "(1) found ${nb} README.md file(s)" @@ -96,7 +96,7 @@ cat ${cfg2} # install rm -rf ${tmpd} echo "[+] install with ignore in dotfile" -cd ${ddpath} | ${bin} install -c ${cfg2} --verbose +cd ${ddpath} | ${bin} install -c ${cfg2} --verbose -f [ "$?" != "0" ] && exit 1 nb=`find ${tmpd} -iname 'README.md' | wc -l` echo "(2) found ${nb} README.md file(s)" @@ -110,7 +110,7 @@ cat ${cfg2} # install rm -rf ${tmpd} echo "[+] install with ignore in config" -cd ${ddpath} | ${bin} install -c ${cfg2} --verbose +cd ${ddpath} | ${bin} install -c ${cfg2} --verbose -f [ "$?" != "0" ] && exit 1 nb=`find ${tmpd} -iname 'README.md' | wc -l` echo "(3) found ${nb} README.md file(s)" @@ -118,7 +118,7 @@ echo "(3) found ${nb} README.md file(s)" ## reinstall to trigger showdiff echo "showdiff" > ${tmpd}/program/a -cd ${ddpath} | echo "y" | ${bin} install --showdiff -c ${cfg} --verbose +cd ${ddpath} | echo "y" | ${bin} install --showdiff -c ${cfg} --verbose -f [ "$?" != "0" ] && exit 1 ## CLEANING diff --git a/tests-ng/install-to-temp.sh b/tests-ng/install-to-temp.sh index d8e95f2..7ffa49c 100755 --- a/tests-ng/install-to-temp.sh +++ b/tests-ng/install-to-temp.sh @@ -84,7 +84,7 @@ echo 'test_y' > ${basedir}/dotfiles/y echo "00000000 01 02 03 04 05" | xxd -r - ${basedir}/dotfiles/z echo "[+] install" -cd ${ddpath} | ${bin} install -c ${cfg} -p p1 --showdiff --verbose --temp | grep '^3 dotfile(s) installed.$' +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 --showdiff --verbose --temp | grep '^3 dotfile(s) installed.$' [ "$?" != "0" ] && exit 1 ## CLEANING diff --git a/tests-ng/macro-with-globals.sh b/tests-ng/macro-with-globals.sh index 48bb1d3..f29b6f3 100755 --- a/tests-ng/macro-with-globals.sh +++ b/tests-ng/macro-with-globals.sh @@ -88,7 +88,7 @@ cat > ${tmps}/dotfiles/abc << _EOF _EOF # install -cd ${ddpath} | ${bin} install -c ${cfg} -p p0 -V +cd ${ddpath} | ${bin} install -c ${cfg} -p p0 -V -f # test file content cat ${tmpd}/abc From d08918487649d0c3e7c351ef03dfc13082dca7b3 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 16 Nov 2020 14:25:44 +0100 Subject: [PATCH 49/82] check for workers in cmd_install --- dotdrop/dotdrop.py | 6 ++++++ dotdrop/options.py | 3 --- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 7750fdb..9738855 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -165,6 +165,12 @@ def cmd_install(o): """install dotfiles for this profile""" dotfiles = o.dotfiles prof = o.conf.get_profile() + + # ensure parallel install is unattended + if o.install_parallel > 1 and o.safe: + LOG.err('\"-w --workers\" must be used with \"-f --force\"') + return False + pro_pre_actions = prof.get_pre_actions() if prof else [] pro_post_actions = prof.get_post_actions() if prof else [] diff --git a/dotdrop/options.py b/dotdrop/options.py index bdf74cf..ad7e6ef 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -256,9 +256,6 @@ class Options(AttrMonitor): except ValueError: self.log.err('bad option for --workers') sys.exit(USAGE) - if self.safe and self.install_parallel > 1: - self.log.err('\"-w --workers\" must be used with \"-f --force\"') - sys.exit(USAGE) # "compare" specifics self.compare_focus = self.args['--file'] From 616c0bee3a09241a116a193e2febd3825e060b0c Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 16 Nov 2020 14:25:53 +0100 Subject: [PATCH 50/82] enable parallel testing --- .github/workflows/testing.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index b039042..ac3d6d0 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -26,7 +26,8 @@ jobs: npm install -g markdown-link-check - name: Run tests run: | - ./tests.sh + DOTDROP_WORKERS=1 ./tests.sh + DOTDROP_WORKERS=4 ./tests.sh env: DOTDROP_FORCE_NODEBUG: yes DOTDROP_NOBANNER: yes From 22d91a9a6270a76d8a3974185c357dfac96e00ab Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 16 Nov 2020 17:37:47 +0100 Subject: [PATCH 51/82] fix ci/cd --- .github/workflows/testing.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index ac3d6d0..b0bb3d9 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -24,13 +24,20 @@ jobs: pip install -r requirements.txt npm install -g remark-cli remark-validate-links npm install -g markdown-link-check - - name: Run tests + - name: Run sequential tests run: | - DOTDROP_WORKERS=1 ./tests.sh - DOTDROP_WORKERS=4 ./tests.sh + ./tests.sh env: DOTDROP_FORCE_NODEBUG: yes DOTDROP_NOBANNER: yes + DOTDROP_WORKERS: 1 + - name: Run parallel tests + run: | + ./tests.sh + env: + DOTDROP_FORCE_NODEBUG: yes + DOTDROP_NOBANNER: yes + DOTDROP_WORKERS: 4 - name: Coveralls run: | coveralls From 3dc653187060f5a56f8799528dc7d2cfd5882a77 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 16 Nov 2020 17:51:12 +0100 Subject: [PATCH 52/82] disable remote links checks as it times out too often --- tests.sh | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests.sh b/tests.sh index 54c2fba..d31ba56 100755 --- a/tests.sh +++ b/tests.sh @@ -75,18 +75,18 @@ else remark -f -u validate-links *.md fi -## test the doc with markdown-link-check -## https://github.com/tcort/markdown-link-check -set +e -which markdown-link-check >/dev/null 2>&1 -r="$?" -set -e -if [ "$r" != "0" ]; then - echo "[WARNING] install \"markdown-link-check\" to test the doc" -else - for i in `find docs -iname '*.md'`; do markdown-link-check $i; done - markdown-link-check README.md -fi +### test the doc with markdown-link-check +### https://github.com/tcort/markdown-link-check +#set +e +#which markdown-link-check >/dev/null 2>&1 +#r="$?" +#set -e +#if [ "$r" != "0" ]; then +# echo "[WARNING] install \"markdown-link-check\" to test the doc" +#else +# for i in `find docs -iname '*.md'`; do markdown-link-check $i; done +# markdown-link-check README.md +#fi ## done echo "All test finished successfully in ${SECONDS}s" From d40d387b9605c78b34472d29138f6fe31410709d Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 16 Nov 2020 20:52:33 +0100 Subject: [PATCH 53/82] dedicated importer class --- dotdrop/dotdrop.py | 138 +++--------------------------- dotdrop/importer.py | 203 ++++++++++++++++++++++++++++++++++++++++++++ dotdrop/options.py | 3 +- 3 files changed, 218 insertions(+), 126 deletions(-) create mode 100644 dotdrop/importer.py diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 9738855..833304f 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -9,7 +9,6 @@ import os import sys import time from concurrent import futures -import shutil # local imports from dotdrop.options import Options @@ -18,9 +17,9 @@ from dotdrop.templategen import Templategen from dotdrop.installer import Installer from dotdrop.updater import Updater from dotdrop.comparator import Comparator -from dotdrop.utils import get_tmpdir, removepath, strip_home, \ - uniq_list, patch_ignores, dependencies_met, get_file_perm, \ - get_default_file_perms +from dotdrop.importer import Importer +from dotdrop.utils import get_tmpdir, removepath, \ + uniq_list, patch_ignores, dependencies_met from dotdrop.linktypes import LinkTypes from dotdrop.exceptions import YamlException, UndefinedException @@ -430,135 +429,26 @@ def cmd_importer(o): ret = True cnt = 0 paths = o.import_path + importer = Importer(o.profile, o.conf, o.dotpath, o.diff_command, + dry=o.dry, safe=o.safe, debug=o.debug, + keepdot=o.keepdot) + for path in paths: - if o.debug: - LOG.dbg('trying to import {}'.format(path)) - if not os.path.exists(path): - LOG.err('\"{}\" does not exist, ignored!'.format(path)) + r = importer.import_path(path, import_as=o.import_as, + import_link=o.import_link, + import_mode=o.import_mode) + if r < 0: ret = False - continue - dst = path.rstrip(os.sep) - dst = os.path.abspath(dst) - - if o.safe: - # ask for symlinks - realdst = os.path.realpath(dst) - if dst != realdst: - msg = '\"{}\" is a symlink, dereference it and continue?' - if not LOG.ask(msg.format(dst)): - continue - - src = strip_home(dst) - if o.import_as: - # handle import as - src = os.path.expanduser(o.import_as) - src = src.rstrip(os.sep) - src = os.path.abspath(src) - src = strip_home(src) - if o.debug: - LOG.dbg('import src for {} as {}'.format(dst, src)) - - strip = '.' + os.sep - if o.keepdot: - strip = os.sep - src = src.lstrip(strip) - - # get the permission - perm = get_file_perm(dst) - - # set the link attribute - linktype = o.import_link - if linktype == LinkTypes.LINK_CHILDREN and \ - not os.path.isdir(path): - LOG.err('importing \"{}\" failed!'.format(path)) - ret = False - continue - - if o.debug: - LOG.dbg('import dotfile: src:{} dst:{}'.format(src, dst)) - - # test no other dotfile exists with same - # dst for this profile but different src - dfs = o.conf.get_dotfile_by_dst(dst) - if dfs: - invalid = False - for df in dfs: - profiles = o.conf.get_profiles_by_dotfile_key(df.key) - profiles = [x.key for x in profiles] - if o.profile in profiles and \ - not o.conf.get_dotfile_by_src_dst(src, dst): - # same profile - # different src - LOG.err('duplicate dotfile for this profile') - ret = False - invalid = True - break - if invalid: - continue - - # prepare hierarchy for dotfile - srcf = os.path.join(o.dotpath, src) - overwrite = not os.path.exists(srcf) - if os.path.exists(srcf): - overwrite = True - if o.safe: - c = Comparator(debug=o.debug, diff_cmd=o.diff_command) - diff = c.compare(srcf, dst) - if diff != '': - # files are different, dunno what to do - LOG.log('diff \"{}\" VS \"{}\"'.format(dst, srcf)) - LOG.emph(diff) - # ask user - msg = 'Dotfile \"{}\" already exists, overwrite?' - overwrite = LOG.ask(msg.format(srcf)) - - if o.debug: - LOG.dbg('will overwrite: {}'.format(overwrite)) - if overwrite: - cmd = 'mkdir -p {}'.format(os.path.dirname(srcf)) - if o.dry: - LOG.dry('would run: {}'.format(cmd)) - else: - try: - os.makedirs(os.path.dirname(srcf), exist_ok=True) - except Exception: - LOG.err('importing \"{}\" failed!'.format(path)) - ret = False - continue - - if o.dry: - LOG.dry('would copy {} to {}'.format(dst, srcf)) - else: - # copy the file to the dotpath - if os.path.isdir(dst): - if os.path.exists(srcf): - shutil.rmtree(srcf) - shutil.copytree(dst, srcf) - else: - shutil.copy2(dst, srcf) - - chmod = None - dflperm = get_default_file_perms(dst, o.umask) - - if o.debug: - LOG.dbg('import mode: {}'.format(o.import_mode)) - if o.import_mode or perm != dflperm: - if o.debug: - LOG.dbg('adopt mode {:o} (umask {:o})'.format(perm, dflperm)) - # insert chmod - chmod = perm - retconf = o.conf.new_dotfile(src, dst, linktype, chmod=chmod) - if retconf: - LOG.sub('\"{}\" imported'.format(path)) + elif r > 0: cnt += 1 - else: - LOG.warn('\"{}\" ignored'.format(path)) + if o.dry: LOG.dry('new config file would be:') LOG.raw(o.conf.dump()) else: o.conf.save() LOG.log('\n{} file(s) imported.'.format(cnt)) + return ret diff --git a/dotdrop/importer.py b/dotdrop/importer.py new file mode 100644 index 0000000..5abaf3d --- /dev/null +++ b/dotdrop/importer.py @@ -0,0 +1,203 @@ +""" +author: deadc0de6 (https://github.com/deadc0de6) +Copyright (c) 2020, deadc0de6 + +handle import of dotfiles +""" + +import os +import shutil + +# local imports +from dotdrop.logger import Logger +from dotdrop.utils import strip_home, get_default_file_perms, \ + get_file_perm, get_umask +from dotdrop.linktypes import LinkTypes +from dotdrop.comparator import Comparator + + +class Importer: + + def __init__(self, profile, conf, dotpath, diff_cmd, + dry=False, safe=True, debug=False, + keepdot=True): + """constructor + @profile: the selected profile + @conf: configuration manager + @dotpath: dotfiles dotpath + @diff_cmd: diff command to use + @dry: simulate + @safe: ask for overwrite if True + @debug: enable debug + @keepdot: keep dot prefix + """ + self.profile = profile + self.conf = conf + self.dotpath = dotpath + self.diff_cmd = diff_cmd + self.dry = dry + self.safe = safe + self.debug = debug + self.keepdot = keepdot + + self.umask = get_umask() + self.log = Logger() + + def import_path(self, path, import_as=None, + import_link=LinkTypes.NOLINK, import_mode=False): + """ + import a dotfile pointed by path + returns: + 1: 1 dotfile imported + 0: ignored + -1: error + """ + if self.debug: + self.log.dbg('import {}'.format(path)) + if not os.path.exists(path): + self.log.err('\"{}\" does not exist, ignored!'.format(path)) + return -1 + + return self._import(path, import_as=import_as, + import_link=import_link, import_mode=import_mode) + + def _import(self, path, import_as=None, + import_link=LinkTypes.NOLINK, import_mode=False): + """ + import path + returns: + 1: 1 dotfile imported + 0: ignored + -1: error + """ + + # normalize path + dst = path.rstrip(os.sep) + dst = os.path.abspath(dst) + + # ask confirmation for symlinks + if self.safe: + realdst = os.path.realpath(dst) + if dst != realdst: + msg = '\"{}\" is a symlink, dereference it and continue?' + if not self.log.ask(msg.format(dst)): + return 0 + + # create src path + src = strip_home(dst) + if import_as: + # handle import as + src = os.path.expanduser(import_as) + src = src.rstrip(os.sep) + src = os.path.abspath(src) + src = strip_home(src) + if self.debug: + self.log.dbg('import src for {} as {}'.format(dst, src)) + # with or without dot prefix + strip = '.' + os.sep + if self.keepdot: + strip = os.sep + src = src.lstrip(strip) + + # get the permission + perm = get_file_perm(dst) + + # get the link attribute + linktype = import_link + if linktype == LinkTypes.LINK_CHILDREN and \ + not os.path.isdir(path): + self.log.err('importing \"{}\" failed!'.format(path)) + return -1 + + if self._already_exists(src, dst): + return -1 + + if self.debug: + self.log.dbg('import dotfile: src:{} dst:{}'.format(src, dst)) + + if not self._prepare_hierarchy(src, dst): + return -1 + + # handle file mode + chmod = None + dflperm = get_default_file_perms(dst, self.umask) + if self.debug: + self.log.dbg('import mode: {}'.format(import_mode)) + if import_mode or perm != dflperm: + if self.debug: + msg = 'adopt mode {:o} (umask {:o})' + self.log.dbg(msg.format(perm, dflperm)) + chmod = perm + + # add file to config file + retconf = self.conf.new_dotfile(src, dst, linktype, chmod=chmod) + if not retconf: + self.log.warn('\"{}\" ignored'.format(path)) + return 0 + + self.log.sub('\"{}\" imported'.format(path)) + return 1 + + def _prepare_hierarchy(self, src, dst): + """prepare hierarchy for dotfile""" + srcf = os.path.join(self.dotpath, src) + + # a dotfile in dotpath already exists at that spot + if os.path.exists(srcf): + if self.safe: + c = Comparator(debug=self.debug, + diff_cmd=self.diff_cmd) + diff = c.compare(srcf, dst) + if diff != '': + # files are different, dunno what to do + self.log.log('diff \"{}\" VS \"{}\"'.format(dst, srcf)) + self.log.emph(diff) + # ask user + msg = 'Dotfile \"{}\" already exists, overwrite?' + if not self.log.ask(msg.format(srcf)): + return False + if self.debug: + self.log.dbg('will overwrite existing file') + + # create directory hierarchy + cmd = 'mkdir -p {}'.format(os.path.dirname(srcf)) + if self.dry: + self.log.dry('would run: {}'.format(cmd)) + else: + try: + os.makedirs(os.path.dirname(srcf), exist_ok=True) + except Exception: + self.log.err('importing \"{}\" failed!'.format(dst)) + return False + + if self.dry: + self.log.dry('would copy {} to {}'.format(dst, srcf)) + else: + # copy the file to the dotpath + if os.path.isdir(dst): + if os.path.exists(srcf): + shutil.rmtree(srcf) + shutil.copytree(dst, srcf) + else: + shutil.copy2(dst, srcf) + + return True + + def _already_exists(self, src, dst): + """ + test no other dotfile exists with same + dst for this profile but different src + """ + dfs = self.conf.get_dotfile_by_dst(dst) + if not dfs: + return False + for df in dfs: + profiles = self.conf.get_profiles_by_dotfile_key(df.key) + profiles = [x.key for x in profiles] + if self.profile in profiles and \ + not self.conf.get_dotfile_by_src_dst(src, dst): + # same profile + # different src + self.log.err('duplicate dotfile for this profile') + return True + return False diff --git a/dotdrop/options.py b/dotdrop/options.py index ad7e6ef..ffdf708 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -16,7 +16,7 @@ from dotdrop.linktypes import LinkTypes from dotdrop.logger import Logger from dotdrop.cfg_aggregator import CfgAggregator as Cfg from dotdrop.action import Action -from dotdrop.utils import uniq_list, get_umask +from dotdrop.utils import uniq_list from dotdrop.exceptions import YamlException ENV_PROFILE = 'DOTDROP_PROFILE' @@ -123,7 +123,6 @@ class Options(AttrMonitor): self.log = Logger() self.debug = self.args['--verbose'] or ENV_DEBUG in os.environ self.dry = self.args['--dry'] - self.umask = get_umask() if ENV_NODEBUG in os.environ: # force disabling debugs self.debug = False From eee431ec397eeffc5e744f463cb89170b85970e8 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 16 Nov 2020 21:18:57 +0100 Subject: [PATCH 54/82] allow parallel update --- dotdrop/dotdrop.py | 57 +++++++++++++++++++++++++--------------------- dotdrop/options.py | 11 ++++++++- 2 files changed, 41 insertions(+), 27 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 833304f..536dc03 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -71,6 +71,16 @@ def action_executor(o, actions, defactions, templater, post=False): return execute +def _dotfile_update(o, path, key=False): + updater = Updater(o.dotpath, o.variables, o.conf, + dry=o.dry, safe=o.safe, debug=o.debug, + ignore=o.update_ignore, + showpatch=o.update_showpatch) + if key: + return updater.update_key(path) + return updater.update_path(path) + + def _dotfile_install(o, dotfile, tmpdir=None): """ install a dotfile @@ -370,18 +380,19 @@ def cmd_compare(o, tmp): def cmd_update(o): """update the dotfile(s) from path(s) or key(s)""" - ret = True cnt = 0 paths = o.update_path iskey = o.update_iskey - ignore = o.update_ignore - showpatch = o.update_showpatch if not paths: # update the entire profile if iskey: + if o.debug: + LOG.dbg('update by keys: {}'.format(paths)) paths = [d.key for d in o.dotfiles] else: + if o.debug: + LOG.dbg('update by paths: {}'.format(paths)) paths = [d.dst for d in o.dotfiles] msg = 'Update all dotfiles for profile \"{}\"'.format(o.profile) if o.safe and not LOG.ask(msg): @@ -391,37 +402,31 @@ def cmd_update(o): if not paths: LOG.log('\nno dotfile to update') return True + if o.debug: LOG.dbg('dotfile to update: {}'.format(paths)) - updater = Updater(o.dotpath, o.variables, o.conf, - dry=o.dry, safe=o.safe, debug=o.debug, - ignore=ignore, showpatch=showpatch) - cnt = 0 - if not iskey: - # update paths - if o.debug: - LOG.dbg('update by paths: {}'.format(paths)) + # update each dotfile + if o.update_parallel > 1: + # in parallel + ex = futures.ThreadPoolExecutor(max_workers=o.update_parallel) + + wait_for = [] for path in paths: - if not updater.update_path(path): - ret = False - else: + j = ex.submit(_dotfile_update, o, path, key=iskey) + wait_for.append(j) + # check result + for f in futures.as_completed(wait_for): + if f.result(): cnt += 1 else: - # update keys - keys = paths - if not keys: - # if not provided, take all keys - keys = [d.key for d in o.dotfiles] - if o.debug: - LOG.dbg('update by keys: {}'.format(keys)) - for key in keys: - if not updater.update_key(key): - ret = False - else: + # sequentially + for path in paths: + if _dotfile_update(o, path, key=iskey): cnt += 1 + LOG.log('\n{} file(s) updated.'.format(cnt)) - return ret + return cnt == len(paths) def cmd_importer(o): diff --git a/dotdrop/options.py b/dotdrop/options.py index ffdf708..efd67c2 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -60,7 +60,7 @@ Usage: dotdrop compare [-LVb] [-c ] [-p ] [-C ...] [-i ...] dotdrop update [-VbfdkP] [-c ] [-p ] - [-i ...] [...] + [-w ] [-i ...] [...] dotdrop remove [-Vbfdk] [-c ] [-p ] [...] dotdrop files [-VbTG] [-c ] [-p ] dotdrop detail [-Vb] [-c ] [-p ] [...] @@ -277,6 +277,15 @@ class Options(AttrMonitor): self.update_ignore.append('*{}'.format(self.install_backup_suffix)) self.update_ignore = uniq_list(self.update_ignore) self.update_showpatch = self.args['--show-patch'] + try: + if ENV_WORKERS in os.environ: + workers = int(os.environ[ENV_WORKERS]) + else: + workers = int(self.args['--workers']) + self.update_parallel = workers + except ValueError: + self.log.err('bad option for --workers') + sys.exit(USAGE) # "detail" specifics self.detail_keys = self.args[''] From 4d12859714112872a2228bc311b2c5f0ad2a0782 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 16 Nov 2020 21:47:14 +0100 Subject: [PATCH 55/82] parallel compare --- dotdrop/dotdrop.py | 222 ++++++++++++++++++++++---------------- dotdrop/installer.py | 2 +- dotdrop/options.py | 30 ++---- tests-ng/chmod-compare.sh | 2 +- 4 files changed, 140 insertions(+), 116 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 536dc03..da4a879 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -72,6 +72,10 @@ def action_executor(o, actions, defactions, templater, post=False): def _dotfile_update(o, path, key=False): + """ + update a dotfile pointed by path + if key is false or by key (in path) + """ updater = Updater(o.dotpath, o.variables, o.conf, dry=o.dry, safe=o.safe, debug=o.debug, ignore=o.update_ignore, @@ -81,6 +85,91 @@ def _dotfile_update(o, path, key=False): return updater.update_path(path) +def _dotfile_compare(o, dotfile, tmp): + """ + compare a dotfile + returns True if same + """ + t = _get_templater(o) + inst = Installer(create=o.create, backup=o.backup, + dry=o.dry, base=o.dotpath, + workdir=o.workdir, debug=o.debug, + backup_suffix=o.install_backup_suffix, + diff_cmd=o.diff_command) + comp = Comparator(diff_cmd=o.diff_command, debug=o.debug) + + # add dotfile variables + newvars = dotfile.get_dotfile_variables() + t.add_tmp_vars(newvars=newvars) + + # dotfiles does not exist / not installed + if o.debug: + LOG.dbg('comparing {}'.format(dotfile)) + + src = dotfile.src + if not os.path.lexists(os.path.expanduser(dotfile.dst)): + line = '=> compare {}: \"{}\" does not exist on destination' + LOG.log(line.format(dotfile.key, dotfile.dst)) + return False + + # apply transformation + tmpsrc = None + if dotfile.trans_r: + if o.debug: + LOG.dbg('applying transformation before comparing') + tmpsrc = apply_trans(o.dotpath, dotfile, t, debug=o.debug) + if not tmpsrc: + # could not apply trans + return False + src = tmpsrc + + # is a symlink pointing to itself + asrc = os.path.join(o.dotpath, os.path.expanduser(src)) + adst = os.path.expanduser(dotfile.dst) + if os.path.samefile(asrc, adst): + if o.debug: + line = '=> compare {}: diffing with \"{}\"' + LOG.dbg(line.format(dotfile.key, dotfile.dst)) + LOG.dbg('points to itself') + return True + + # install dotfile to temporary dir and compare + ret, err, insttmp = inst.install_to_temp(t, tmp, src, dotfile.dst, + template=dotfile.template, + chmod=dotfile.chmod) + if not ret: + # failed to install to tmp + line = '=> compare {}: error' + LOG.log(line.format(dotfile.key, err)) + LOG.err(err) + return False + ignores = list(set(o.compare_ignore + dotfile.cmpignore)) + ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) + diff = comp.compare(insttmp, dotfile.dst, ignore=ignores) + + # clean tmp transformed dotfile if any + if tmpsrc: + tmpsrc = os.path.join(o.dotpath, tmpsrc) + if os.path.exists(tmpsrc): + removepath(tmpsrc, LOG) + + if diff != '': + # print diff results + line = '=> compare {}: diffing with \"{}\"' + LOG.log(line.format(dotfile.key, dotfile.dst)) + if o.compare_fileonly: + LOG.raw('') + else: + LOG.emph(diff) + return False + # no difference + if o.debug: + line = '=> compare {}: diffing with \"{}\"' + LOG.dbg(line.format(dotfile.key, dotfile.dst)) + LOG.dbg('same file') + return True + + def _dotfile_install(o, dotfile, tmpdir=None): """ install a dotfile @@ -175,8 +264,8 @@ def cmd_install(o): dotfiles = o.dotfiles prof = o.conf.get_profile() - # ensure parallel install is unattended - if o.install_parallel > 1 and o.safe: + # ensure parallel is unattended + if o.workers > 1 and o.safe: LOG.err('\"-w --workers\" must be used with \"-f --force\"') return False @@ -208,9 +297,9 @@ def cmd_install(o): return False # install each dotfile - if o.install_parallel > 1: + if o.workers > 1: # in parallel - ex = futures.ThreadPoolExecutor(max_workers=o.install_parallel) + ex = futures.ThreadPoolExecutor(max_workers=o.workers) wait_for = [] for dotfile in dotfiles: @@ -269,14 +358,14 @@ def cmd_install(o): def cmd_compare(o, tmp): """compare dotfiles and return True if all identical""" - cnt = 0 + # ensure parallel is unattended dotfiles = o.dotfiles if not dotfiles: msg = 'no dotfile defined for this profile (\"{}\")' LOG.warn(msg.format(o.profile)) return True + # compare only specific files - same = True selected = dotfiles if o.compare_focus: selected = _select(o.compare_focus, dotfiles) @@ -285,94 +374,32 @@ def cmd_compare(o, tmp): LOG.log('\nno dotfile to compare') return False - t = _get_templater(o) - tvars = t.add_tmp_vars() - inst = Installer(create=o.create, backup=o.backup, - dry=o.dry, base=o.dotpath, - workdir=o.workdir, debug=o.debug, - backup_suffix=o.install_backup_suffix, - diff_cmd=o.diff_command) - comp = Comparator(diff_cmd=o.diff_command, debug=o.debug) - - for dotfile in selected: - if not dotfile.src and not dotfile.dst: - # ignore fake dotfile - continue - cnt += 1 - - # add dotfile variables - t.restore_vars(tvars) - newvars = dotfile.get_dotfile_variables() - t.add_tmp_vars(newvars=newvars) - - # dotfiles does not exist / not installed - if o.debug: - LOG.dbg('comparing {}'.format(dotfile)) - src = dotfile.src - if not os.path.lexists(os.path.expanduser(dotfile.dst)): - line = '=> compare {}: \"{}\" does not exist on destination' - LOG.log(line.format(dotfile.key, dotfile.dst)) - same = False - continue - - # apply transformation - tmpsrc = None - if dotfile.trans_r: - if o.debug: - LOG.dbg('applying transformation before comparing') - tmpsrc = apply_trans(o.dotpath, dotfile, t, debug=o.debug) - if not tmpsrc: - # could not apply trans - same = False + same = True + cnt = 0 + if o.workers > 1: + # in parallel + ex = futures.ThreadPoolExecutor(max_workers=o.workers) + wait_for = [] + for dotfile in selected: + j = ex.submit(_dotfile_compare, o, dotfile, tmp) + wait_for.append(j) + # check result + for f in futures.as_completed(wait_for): + if not dotfile.src and not dotfile.dst: + # ignore fake dotfile continue - src = tmpsrc - - # is a symlink pointing to itself - asrc = os.path.join(o.dotpath, os.path.expanduser(src)) - adst = os.path.expanduser(dotfile.dst) - if os.path.samefile(asrc, adst): - if o.debug: - line = '=> compare {}: diffing with \"{}\"' - LOG.dbg(line.format(dotfile.key, dotfile.dst)) - LOG.dbg('points to itself') - continue - - # install dotfile to temporary dir and compare - ret, err, insttmp = inst.install_to_temp(t, tmp, src, dotfile.dst, - template=dotfile.template, - chmod=dotfile.chmod) - if not ret: - # failed to install to tmp - line = '=> compare {}: error' - LOG.log(line.format(dotfile.key, err)) - LOG.err(err) - same = False - continue - ignores = list(set(o.compare_ignore + dotfile.cmpignore)) - ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) - diff = comp.compare(insttmp, dotfile.dst, ignore=ignores) - - # clean tmp transformed dotfile if any - if tmpsrc: - tmpsrc = os.path.join(o.dotpath, tmpsrc) - if os.path.exists(tmpsrc): - removepath(tmpsrc, LOG) - - if diff == '': - # no difference - if o.debug: - line = '=> compare {}: diffing with \"{}\"' - LOG.dbg(line.format(dotfile.key, dotfile.dst)) - LOG.dbg('same file') - else: - # print diff results - line = '=> compare {}: diffing with \"{}\"' - LOG.log(line.format(dotfile.key, dotfile.dst)) - if o.compare_fileonly: - LOG.raw('') - else: - LOG.emph(diff) - same = False + if not f.result(): + same = False + cnt += 1 + else: + # sequentially + for dotfile in selected: + if not dotfile.src and not dotfile.dst: + # ignore fake dotfile + continue + if not _dotfile_compare(o, dotfile, tmp): + same = False + cnt += 1 LOG.log('\n{} dotfile(s) compared.'.format(cnt)) return same @@ -380,6 +407,11 @@ def cmd_compare(o, tmp): def cmd_update(o): """update the dotfile(s) from path(s) or key(s)""" + # ensure parallel is unattended + if o.workers > 1 and o.safe: + LOG.err('\"-w --workers\" must be used with \"-f --force\"') + return False + cnt = 0 paths = o.update_path iskey = o.update_iskey @@ -407,9 +439,9 @@ def cmd_update(o): LOG.dbg('dotfile to update: {}'.format(paths)) # update each dotfile - if o.update_parallel > 1: + if o.workers > 1: # in parallel - ex = futures.ThreadPoolExecutor(max_workers=o.update_parallel) + ex = futures.ThreadPoolExecutor(max_workers=o.workers) wait_for = [] for path in paths: diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 55588fe..1dee366 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -715,7 +715,7 @@ class Installer: if self.debug: self.log.dbg('mkdir -p {}'.format(directory)) - os.makedirs(directory) + os.makedirs(directory, exist_ok=True) return os.path.exists(directory) def _backup(self, path): diff --git a/dotdrop/options.py b/dotdrop/options.py index efd67c2..7fc7680 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -58,7 +58,7 @@ Usage: dotdrop import [-Vbdfm] [-c ] [-p ] [-s ] [-l ] ... dotdrop compare [-LVb] [-c ] [-p ] - [-C ...] [-i ...] + [-w ] [-C ...] [-i ...] dotdrop update [-VbfdkP] [-c ] [-p ] [-w ] [-i ...] [...] dotdrop remove [-Vbfdk] [-c ] [-p ] [...] @@ -217,6 +217,16 @@ class Options(AttrMonitor): # adapt attributes based on arguments self.safe = not self.args['--force'] + try: + if ENV_WORKERS in os.environ: + workers = int(os.environ[ENV_WORKERS]) + else: + workers = int(self.args['--workers']) + self.workers = workers + except ValueError: + self.log.err('bad option for --workers') + sys.exit(USAGE) + # import link default value self.import_link = self.link_on_import if self.args['--link']: @@ -246,15 +256,6 @@ class Options(AttrMonitor): self.install_default_actions_post = [a for a in self.default_actions if a.kind == Action.post] self.install_ignore = self.instignore - try: - if ENV_WORKERS in os.environ: - workers = int(os.environ[ENV_WORKERS]) - else: - workers = int(self.args['--workers']) - self.install_parallel = workers - except ValueError: - self.log.err('bad option for --workers') - sys.exit(USAGE) # "compare" specifics self.compare_focus = self.args['--file'] @@ -277,15 +278,6 @@ class Options(AttrMonitor): self.update_ignore.append('*{}'.format(self.install_backup_suffix)) self.update_ignore = uniq_list(self.update_ignore) self.update_showpatch = self.args['--show-patch'] - try: - if ENV_WORKERS in os.environ: - workers = int(os.environ[ENV_WORKERS]) - else: - workers = int(self.args['--workers']) - self.update_parallel = workers - except ValueError: - self.log.err('bad option for --workers') - sys.exit(USAGE) # "detail" specifics self.detail_keys = self.args[''] diff --git a/tests-ng/chmod-compare.sh b/tests-ng/chmod-compare.sh index c92be03..6a2b909 100755 --- a/tests-ng/chmod-compare.sh +++ b/tests-ng/chmod-compare.sh @@ -111,7 +111,7 @@ chmod 700 ${flink} set +e cnt=`cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 2>&1 | grep 'modes differ' | wc -l` set -e -[ "${cnt}" != "5" ] && echo "compare modes failed" && exit 1 +[ "${cnt}" != "5" ] && echo "compare modes failed (${cnt})" && exit 1 ## CLEANING rm -rf ${tmps} ${tmpd} From feff368340dcfbe2cee4803ffdc4585a0d895d2c Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 16 Nov 2020 22:05:28 +0100 Subject: [PATCH 56/82] more logs --- dotdrop/dotdrop.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index da4a879..3302de9 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -299,6 +299,8 @@ def cmd_install(o): # install each dotfile if o.workers > 1: # in parallel + if o.debug: + LOG.dbg('run with {} workers'.format(o.workers)) ex = futures.ThreadPoolExecutor(max_workers=o.workers) wait_for = [] @@ -378,6 +380,9 @@ def cmd_compare(o, tmp): cnt = 0 if o.workers > 1: # in parallel + if o.debug: + LOG.dbg('run with {} workers'.format(o.workers)) + ex = futures.ThreadPoolExecutor(max_workers=o.workers) wait_for = [] for dotfile in selected: @@ -441,8 +446,10 @@ def cmd_update(o): # update each dotfile if o.workers > 1: # in parallel - ex = futures.ThreadPoolExecutor(max_workers=o.workers) + if o.debug: + LOG.dbg('run with {} workers'.format(o.workers)) + ex = futures.ThreadPoolExecutor(max_workers=o.workers) wait_for = [] for path in paths: j = ex.submit(_dotfile_update, o, path, key=iskey) From 5fb1024975111373b6b1440fd8e5a44e1f4d2826 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 16 Nov 2020 22:08:12 +0100 Subject: [PATCH 57/82] disable workers for unittests --- tests.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests.sh b/tests.sh index d31ba56..722f973 100755 --- a/tests.sh +++ b/tests.sh @@ -32,6 +32,12 @@ export DOTDROP_FORCE_NODEBUG=yes # coverage file location cur=`dirname $(readlink -f "${0}")` +workers=${DOTDROP_WORKERS} +if [ ! -z ${workers} ]; then + unset DOTDROP_WORKERS + echo "DISABLE workers" +fi + # execute tests with coverage if [ -z ${GITHUB_WORKFLOW} ]; then ## local @@ -51,6 +57,11 @@ unset DOTDROP_FORCE_NODEBUG #export DOTDROP_FORCE_NODEBUG=yes export DOTDROP_WORKDIR=/tmp/dotdrop-tests-workdir +if [ ! -z ${workers} ]; then + DOTDROP_WORKERS=${workers} + echo "ENABLE workers: ${workers}" +fi + # run bash tests if [ -z ${GITHUB_WORKFLOW} ]; then ## local From 89d6a1b6ed901bd71c14a38eb44b026e4c632d72 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 17 Nov 2020 11:25:03 +0100 Subject: [PATCH 58/82] disable workers without --force or with --dry --- dotdrop/dotdrop.py | 13 ------------- dotdrop/options.py | 7 +++++++ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 3302de9..588be51 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -264,11 +264,6 @@ def cmd_install(o): dotfiles = o.dotfiles prof = o.conf.get_profile() - # ensure parallel is unattended - if o.workers > 1 and o.safe: - LOG.err('\"-w --workers\" must be used with \"-f --force\"') - return False - pro_pre_actions = prof.get_pre_actions() if prof else [] pro_post_actions = prof.get_post_actions() if prof else [] @@ -360,7 +355,6 @@ def cmd_install(o): def cmd_compare(o, tmp): """compare dotfiles and return True if all identical""" - # ensure parallel is unattended dotfiles = o.dotfiles if not dotfiles: msg = 'no dotfile defined for this profile (\"{}\")' @@ -382,7 +376,6 @@ def cmd_compare(o, tmp): # in parallel if o.debug: LOG.dbg('run with {} workers'.format(o.workers)) - ex = futures.ThreadPoolExecutor(max_workers=o.workers) wait_for = [] for dotfile in selected: @@ -412,11 +405,6 @@ def cmd_compare(o, tmp): def cmd_update(o): """update the dotfile(s) from path(s) or key(s)""" - # ensure parallel is unattended - if o.workers > 1 and o.safe: - LOG.err('\"-w --workers\" must be used with \"-f --force\"') - return False - cnt = 0 paths = o.update_path iskey = o.update_iskey @@ -448,7 +436,6 @@ def cmd_update(o): # in parallel if o.debug: LOG.dbg('run with {} workers'.format(o.workers)) - ex = futures.ThreadPoolExecutor(max_workers=o.workers) wait_for = [] for path in paths: diff --git a/dotdrop/options.py b/dotdrop/options.py index 7fc7680..1da27db 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -227,6 +227,13 @@ class Options(AttrMonitor): self.log.err('bad option for --workers') sys.exit(USAGE) + if self.safe and self.workers > 1: + self.log.warn('workers set to 1 when --force is not used') + self.workers = 1 + if self.dry and self.workers > 1: + self.log.warn('workers set to 1 when --dry is used') + self.workers = 1 + # import link default value self.import_link = self.link_on_import if self.args['--link']: From e4d2d272a8c01b2b17990f5a05e4906844f877db Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 17 Nov 2020 13:42:53 +0100 Subject: [PATCH 59/82] refactor installer --- docs/config.md | 2 - dotdrop/dotdrop.py | 19 +-- dotdrop/dotfile.py | 6 +- dotdrop/installer.py | 282 +++++++++++++------------------------- dotdrop/utils.py | 1 + tests-ng/chmod-install.sh | 2 +- tests-ng/diff-cmd.sh | 6 +- tests/test_install.py | 31 +++-- 8 files changed, 132 insertions(+), 217 deletions(-) diff --git a/docs/config.md b/docs/config.md index fa5ee18..86a553e 100644 --- a/docs/config.md +++ b/docs/config.md @@ -77,8 +77,6 @@ On `import` the following rules are applied: On `install` the following rules are applied: * if `chmod` is specified in the dotfile, it will be applied to the installed dotfile -* if no `chmod` is specified and the file exists on the filesystem with different permissions than the file in the `dotpath` - then the user needs to confirm the dotfile installation (unless `-f --force` is used) On `update`: diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 588be51..f9cdcc4 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -198,17 +198,19 @@ def _dotfile_install(o, dotfile, tmpdir=None): if hasattr(dotfile, 'link') and dotfile.link == LinkTypes.LINK: # link - r, err = inst.link(t, dotfile.src, dotfile.dst, - actionexec=pre_actions_exec, - template=dotfile.template, - chmod=dotfile.chmod) + r, err = inst.install(t, dotfile.src, dotfile.dst, + dotfile.link, + actionexec=pre_actions_exec, + template=dotfile.template, + chmod=dotfile.chmod) elif hasattr(dotfile, 'link') and \ dotfile.link == LinkTypes.LINK_CHILDREN: # link_children - r, err = inst.link_children(t, dotfile.src, dotfile.dst, - actionexec=pre_actions_exec, - template=dotfile.template, - chmod=dotfile.chmod) + r, err = inst.install(t, dotfile.src, dotfile.dst, + dotfile.link, + actionexec=pre_actions_exec, + template=dotfile.template, + chmod=dotfile.chmod) else: # nolink src = dotfile.src @@ -221,6 +223,7 @@ def _dotfile_install(o, dotfile, tmpdir=None): ignores = list(set(o.install_ignore + dotfile.instignore)) ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) r, err = inst.install(t, src, dotfile.dst, + LinkTypes.NOLINK, actionexec=pre_actions_exec, noempty=dotfile.noempty, ignore=ignores, diff --git a/dotdrop/dotfile.py b/dotdrop/dotfile.py index fee2727..9c96434 100644 --- a/dotdrop/dotfile.py +++ b/dotdrop/dotfile.py @@ -115,7 +115,8 @@ class Dotfile(DictParser): msg += ', dst:\"{}\"'.format(self.dst) msg += ', link:\"{}\"'.format(str(self.link)) msg += ', template:{}'.format(self.template) - msg += ', chmod:{}'.format(self.chmod) + if self.chmod: + msg += ', chmod:{:o}'.format(self.chmod) return msg def prt(self): @@ -126,7 +127,8 @@ class Dotfile(DictParser): out += '\n{}dst: \"{}\"'.format(indent, self.dst) out += '\n{}link: \"{}\"'.format(indent, str(self.link)) out += '\n{}template: \"{}\"'.format(indent, str(self.template)) - out += '\n{}chmod: \"{}\"'.format(indent, str(self.chmod)) + if self.chmod: + out += '\n{}chmod: \"{:o}\"'.format(indent, self.chmod) out += '\n{}pre-action:'.format(indent) some = self.get_pre_actions() diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 1dee366..4867b77 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -11,6 +11,7 @@ import shutil # local imports from dotdrop.logger import Logger +from dotdrop.linktypes import LinkTypes from dotdrop.templategen import Templategen import dotdrop.utils as utils from dotdrop.exceptions import UndefinedException @@ -63,16 +64,17 @@ class Installer: # public methods ######################################################## - def install(self, templater, src, dst, + def install(self, templater, src, dst, linktype, actionexec=None, noempty=False, ignore=[], template=True, chmod=None): """ - install src to dst using a templater + install src to dst @templater: the templater object @src: dotfile source path in dotpath @dst: dotfile destination path in the FS + @linktype: linktypes.LinkTypes @actionexec: action executor callback @noempty: render empty template flag @ignore: pattern to ignore when installing @@ -85,77 +87,79 @@ class Installer: - False, None : ignored """ if self.debug: - self.log.dbg('installing \"{}\" to \"{}\"'.format(src, dst)) + msg = 'installing \"{}\" to \"{}\" (link: {})' + self.log.dbg(msg.format(src, dst, str(linktype))) src, dst, cont, err = self._check_paths(src, dst, chmod) if not cont: return self._log_install(cont, err) - r, err = self._install(templater, src, dst, - actionexec=actionexec, - noempty=noempty, - ignore=ignore, - template=template, - chmod=chmod) + # check source file exists + src = os.path.join(self.base, src) + if not os.path.exists(src): + err = 'source dotfile does not exist: {}'.format(src) + return self._log_install(False, err) - return self._log_install(r, err) + self.action_executed = False - def link(self, templater, src, dst, actionexec=None, - template=True, chmod=None): - """ - set src as the link target of dst + # install to temporary dir + # and ignore any actions + if self.totemp: + r, err, _ = self.install_to_temp(templater, self.totemp, + src, dst, template=template, + chmod=chmod) + return self._log_install(r, err) - @templater: the templater - @src: dotfile source path in dotpath - @dst: dotfile destination path in the FS - @actionexec: action executor callback - @template: template this dotfile - @chmod: rights to apply if any - - return - - True, None : success - - False, error_msg : error - - False, None : ignored - """ + isdir = os.path.isdir(src) if self.debug: - self.log.dbg('link \"{}\" to \"{}\"'.format(src, dst)) - src, dst, cont, err = self._check_paths(src, dst, chmod) - if not cont: - return self._log_install(cont, err) + self.log.dbg('install {} to {}'.format(src, dst)) + self.log.dbg('\"{}\" is a directory: {}'.format(src, isdir)) - r, err = self._link(templater, src, dst, - actionexec=actionexec, - template=template) + # TODO remove template and set templater + # to None when no template should occur + if linktype == LinkTypes.NOLINK: + # normal file + if isdir: + r, err = self._copy_dir(templater, src, dst, + actionexec=actionexec, + noempty=noempty, ignore=ignore, + template=template, + chmod=chmod) + else: + r, err = self._copy_file(templater, src, dst, + actionexec=actionexec, + noempty=noempty, ignore=ignore, + template=template, + chmod=chmod) + elif linktype == LinkTypes.LINK: + # symlink + r, err = self._link(templater, src, dst, + actionexec=actionexec, + template=template) + elif linktype == LinkTypes.LINK_CHILDREN: + # symlink direct children + if not isdir: + if self.debug: + msg = 'symlink children of {} to {}' + self.log.dbg(msg.format(src, dst)) + err = 'source dotfile is not a directory: {}'.format(src) + r = False + else: + r, err = self._link_children(templater, src, dst, + actionexec=actionexec, + template=template) + + # handle chmod if chmod and not self.dry: - # apply mode - utils.chmod(dst, chmod, self.debug) - return self._log_install(r, err) + dstperms = utils.get_file_perm(dst) + if dstperms != chmod: + # apply mode + self.log.sub('chmod {} to {:o}'.format(dst, chmod)) + if utils.chmod(dst, chmod, debug=self.debug): + r = True + else: + r = False + err = 'chmod failed' - def link_children(self, templater, src, dst, actionexec=None, - template=True, chmod=None): - """ - link all files under a given directory - - @templater: the templater - @src: dotfile source path in dotpath - @dst: dotfile destination path in the FS - @actionexec: action executor callback - @template: template this dotfile - @chmod: file mode to apply - - return - - True, None: success - - False, error_msg: error - - False, None, ignored - """ - if self.debug: - self.log.dbg('link_children \"{}\" to \"{}\"'.format(src, dst)) - src, dst, cont, err = self._check_paths(src, dst, chmod) - if not cont: - return self._log_install(cont, err) - - r, err = self._link_children(templater, src, dst, - actionexec=actionexec, - template=template, chmod=chmod) return self._log_install(r, err) def install_to_temp(self, templater, tmpdir, src, dst, @@ -190,10 +194,13 @@ class Installer: self.diff = False createsaved = self.create self.create = True + totemp = self.totemp + self.totemp = None # install the dotfile to a temp directory tmpdst = self._pivot_path(dst, tmpdir) ret, err = self.install(templater, src, tmpdst, + LinkTypes.NOLINK, template=template, chmod=chmod) if self.debug: if ret: @@ -204,101 +211,35 @@ class Installer: self.diff = diffsaved self.create = createsaved self.comparing = False + self.totemp = totemp + return ret, err, tmpdst ######################################################## # low level accessors for public methods ######################################################## - def _install(self, templater, src, dst, - actionexec=None, noempty=False, - ignore=[], template=True, - chmod=None): - """install link:nolink""" - self.action_executed = False - - # check source file exists - src = os.path.join(self.base, src) - if not os.path.exists(src): - err = 'source dotfile does not exist: {}'.format(src) - return False, err - - # check dst is valid - if self.totemp: - dst = self._pivot_path(dst, self.totemp) - if utils.samefile(src, dst): - # symlink loop - err = 'dotfile points to itself: {}'.format(dst) - return False, err - - isdir = os.path.isdir(src) - if self.debug: - self.log.dbg('install {} to {}'.format(src, dst)) - self.log.dbg('\"{}\" is a directory: {}'.format(src, isdir)) - if isdir: - b, e = self._copy_dir(templater, src, dst, - actionexec=actionexec, - noempty=noempty, ignore=ignore, - template=template, - chmod=chmod) - return b, e - b, e = self._copy_file(templater, src, dst, - actionexec=actionexec, - noempty=noempty, ignore=ignore, - template=template, - chmod=chmod) - return b, e - - def _link(self, templater, src, dst, actionexec=None, - template=True): + def _link(self, templater, src, dst, actionexec=None, template=True): """install link:link""" - self.action_executed = False - src = os.path.join(self.base, src) - if not os.path.exists(src): - err = 'source dotfile does not exist: {}'.format(src) - return False, err - if self.totemp: - # ignore actions - b, e = self.install(templater, src, dst, actionexec=None, - template=template) - return b, e - if template and Templategen.is_template(src): if self.debug: - self.log.dbg('dotfile is a template') - self.log.dbg('install to {} and symlink'.format(self.workdir)) + self.log.dbg('is a template') + self.log.dbg('install to {}'.format(self.workdir)) tmp = self._pivot_path(dst, self.workdir, striphome=True) - i, err = self.install(templater, src, tmp, actionexec=actionexec, + r, err = self.install(templater, src, tmp, + LinkTypes.NOLINK, + actionexec=actionexec, template=template) - if not i and not os.path.exists(tmp): - return i, err + if not r and not os.path.exists(tmp): + return r, err src = tmp - b, e = self._symlink(src, dst, actionexec=actionexec) - return b, e + r, err = self._symlink(src, dst, actionexec=actionexec) + return r, err - def _link_children(self, templater, src, dst, actionexec=None, - template=True, chmod=None): + def _link_children(self, templater, src, dst, + actionexec=None, template=True): """install link:link_children""" - if not dst or not src: - if self.debug: - self.log.dbg('empty dst for {}'.format(src)) - return True, None - self.action_executed = False parent = os.path.join(self.base, src) - - # Fail if source doesn't exist - if not os.path.exists(parent): - err = 'source dotfile does not exist: {}'.format(parent) - return False, err - - # Fail if source not a directory - if not os.path.isdir(parent): - if self.debug: - self.log.dbg('symlink children of {} to {}'.format(src, dst)) - - err = 'source dotfile is not a directory: {}'.format(parent) - return False, err - if not os.path.lexists(dst): self.log.sub('creating directory "{}"'.format(dst)) os.makedirs(dst) @@ -313,7 +254,7 @@ class Installer: err = 'ignoring "{}", not installed'.format(dst) return False, err os.unlink(dst) - os.mkdir(dst) + self._create_dirs(dst) children = os.listdir(parent) srcs = [os.path.normpath(os.path.join(parent, child)) @@ -331,11 +272,12 @@ class Installer: if template and Templategen.is_template(subsrc): if self.debug: - self.log.dbg('dotfile is a template') + self.log.dbg('child is a template') self.log.dbg('install to {} and symlink' .format(self.workdir)) tmp = self._pivot_path(subdst, self.workdir, striphome=True) r, e = self.install(templater, subsrc, tmp, + LinkTypes.NOLINK, actionexec=actionexec, template=template) if not r and e and not os.path.exists(tmp): @@ -352,10 +294,6 @@ class Installer: if err: return ret, err - if chmod and not self.dry: - # apply mode - utils.chmod(dst, chmod, debug=self.debug) - return installed > 0, None ######################################################## @@ -430,6 +368,11 @@ class Installer: self.log.dbg('template: {}'.format(template)) self.log.dbg('no empty: {}'.format(noempty)) + # check no loop + if utils.samefile(src, dst): + err = 'dotfile points to itself: {}'.format(dst) + return False, err + if utils.must_ignore([src, dst], ignore, debug=self.debug): if self.debug: self.log.dbg('ignoring install of {} to {}'.format(src, dst)) @@ -483,7 +426,7 @@ class Installer: if ret == 0: # success if not self.dry and not self.comparing: - self.log.sub('copied {} to {}'.format(src, dst)) + self.log.sub('install {} to {}'.format(src, dst)) return True, None # error err = 'installing {} to {}'.format(src, dst) @@ -499,8 +442,6 @@ class Installer: ret = False, None # create the directory anyway - if self.debug: - self.log.dbg('mkdir -p {}'.format(dst)) if not self._create_dirs(dst): err = 'creating directory for {}'.format(dst) return False, err @@ -542,10 +483,6 @@ class Installer: elif res: # something got installed ret = True, None - - if chmod and not self.dry: - # apply mode - utils.chmod(dst, chmod, debug=self.debug) return ret def _write(self, src, dst, content=None, @@ -629,9 +566,6 @@ class Installer: shutil.copymode(src, dst) except Exception as e: return -1, str(e) - - if chmod: - utils.chmod(dst, chmod, debug=self.debug) return 0, None ######################################################## @@ -714,6 +648,7 @@ class Installer: return True if self.debug: self.log.dbg('mkdir -p {}'.format(directory)) + self.log.sub('create directory {}'.format(directory)) os.makedirs(directory, exist_ok=True) return os.path.exists(directory) @@ -762,32 +697,6 @@ class Installer: self.log.dbg('install: IGNORED') return boolean, err - def _check_chmod(self, src, dst, chmod): - """ - check chmod needs change - returns True to continue installation - """ - if chmod: - return True - if not os.path.exists(dst): - return True - if not os.path.exists(src): - return True - sperms = utils.get_file_perm(src) - dperms = utils.get_file_perm(dst) - if sperms == dperms: - return True - elif self.safe: - # this only happens if no - # chmod is provided - # and dst/src modes differ - if self.safe: - msg = 'Set mode {:o} to \"{}\"' - msg = msg.format(sperms, dst) - if not self.log.ask(msg): - return False - return True - def _check_paths(self, src, dst, chmod): """ check and normalize param @@ -807,9 +716,4 @@ class Installer: dst = os.path.expanduser(dst) dst = os.path.normpath(dst) - # check chmod - if not self._check_chmod(src, dst, chmod): - err = 'ignoring "{}", not installed'.format(dst) - return None, None, False, err - return src, dst, True, None diff --git a/dotdrop/utils.py b/dotdrop/utils.py index b04be4b..52fb01c 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -330,3 +330,4 @@ def chmod(path, mode, debug=False): if debug: LOG.dbg('chmod {} {}'.format(oct(mode), path)) os.chmod(path, mode) + return get_file_perm(path) == mode diff --git a/tests-ng/chmod-install.sh b/tests-ng/chmod-install.sh index 72853de..5c4d363 100755 --- a/tests-ng/chmod-install.sh +++ b/tests-ng/chmod-install.sh @@ -17,7 +17,7 @@ has_rights() echo "testing ${1} is ${2}" [ ! -e "$1" ] && echo "`basename $1` does not exist" && exit 1 local mode=`stat -L -c '%a' "$1"` - [ "${mode}" != "$2" ] && echo "bad mode for `basename $1`" && exit 1 + [ "${mode}" != "$2" ] && echo "bad mode for `basename $1` (${mode} VS expected ${2})" && exit 1 true } diff --git a/tests-ng/diff-cmd.sh b/tests-ng/diff-cmd.sh index f44e11f..639c6ae 100755 --- a/tests-ng/diff-cmd.sh +++ b/tests-ng/diff-cmd.sh @@ -81,7 +81,7 @@ echo "modified" > ${tmpd}/singlefile # default diff (unified) echo "[+] comparing with default diff (unified)" set +e -cd ${ddpath} | ${bin} compare -c ${cfg} 2>&1 | grep -v '=>' | grep -v 'dotfile(s) compared' | sed '$d' | grep -v '^+++\|^---' > ${tmpd}/normal +cd ${ddpath} | ${bin} compare -c ${cfg} 2>&1 | grep -v '=>' | grep -v '\->' | grep -v 'dotfile(s) compared' | sed '$d' | grep -v '^+++\|^---' > ${tmpd}/normal diff -u -r ${tmpd}/singlefile ${basedir}/dotfiles/${tmpd}/singlefile | grep -v '^+++\|^---' > ${tmpd}/real set -e @@ -96,7 +96,7 @@ sed '/dotpath: dotfiles/a \ \ diff_command: "diff -r {0} {1}"' ${cfg} > ${cfg2} # normal diff echo "[+] comparing with normal diff" set +e -cd ${ddpath} | ${bin} compare -c ${cfg2} 2>&1 | grep -v '=>' | grep -v 'dotfile(s) compared' | sed '$d' > ${tmpd}/unified +cd ${ddpath} | ${bin} compare -c ${cfg2} 2>&1 | grep -v '=>' | grep -v '\->' | grep -v 'dotfile(s) compared' | sed '$d' > ${tmpd}/unified diff -r ${tmpd}/singlefile ${basedir}/dotfiles/${tmpd}/singlefile > ${tmpd}/real set -e @@ -113,7 +113,7 @@ sed '/dotpath: dotfiles/a \ \ diff_command: "echo fakediff"' ${cfg} > ${cfg3} # fake diff echo "[+] comparing with fake diff" set +e -cd ${ddpath} | ${bin} compare -c ${cfg3} 2>&1 | grep -v '=>' | grep -v 'dotfile(s) compared' | sed '$d' > ${tmpd}/fake +cd ${ddpath} | ${bin} compare -c ${cfg3} 2>&1 | grep -v '=>' | grep -v '\->' | grep -v 'dotfile(s) compared' | sed '$d' > ${tmpd}/fake set -e # verify diff --git a/tests/test_install.py b/tests/test_install.py index 2da99be..cbac55a 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -349,8 +349,9 @@ exec bspwm srcs = [create_random_file(src_dir)[0] for _ in range(3)] installer = Installer() - installer.link_children(templater=MagicMock(), src=src_dir, - dst=dst_dir, actionexec=None) + installer.install(templater=MagicMock(), src=src_dir, + dst=dst_dir, linktype=LinkTypes.LINK_CHILDREN, + actionexec=None) # Ensure all destination files point to source for src in srcs: @@ -365,8 +366,10 @@ exec bspwm # logger = MagicMock() # installer.log.err = logger - res, err = installer.link_children(templater=MagicMock(), src=src, - dst='/dev/null', actionexec=None) + res, err = installer.install(templater=MagicMock(), src=src, + dst='/dev/null', + linktype=LinkTypes.LINK_CHILDREN, + actionexec=None) self.assertFalse(res) e = 'source dotfile does not exist: {}'.format(src) @@ -387,8 +390,10 @@ exec bspwm # installer.log.err = logger # pass src file not src dir - res, err = installer.link_children(templater=templater, src=src, - dst='/dev/null', actionexec=None) + res, err = installer.install(templater=templater, src=src, + dst='/dev/null', + linktype=LinkTypes.LINK_CHILDREN, + actionexec=None) # ensure nothing performed self.assertFalse(res) @@ -410,8 +415,9 @@ exec bspwm self.assertFalse(os.path.exists(dst_dir)) installer = Installer() - installer.link_children(templater=MagicMock(), src=src_dir, - dst=dst_dir, actionexec=None) + installer.install(templater=MagicMock(), src=src_dir, + dst=dst_dir, linktype=LinkTypes.LINK_CHILDREN, + actionexec=None) # ensure dst dir created self.assertTrue(os.path.exists(dst_dir)) @@ -442,8 +448,9 @@ exec bspwm installer.safe = True installer.log.ask = ask - installer.link_children(templater=MagicMock(), src=src_dir, dst=dst, - actionexec=None) + installer.install(templater=MagicMock(), src=src_dir, + dst=dst, linktype=LinkTypes.LINK_CHILDREN, + actionexec=None) # ensure destination now a directory self.assertTrue(os.path.isdir(dst)) @@ -476,8 +483,8 @@ exec bspwm # make templategen treat everything as a template mocked_templategen.is_template.return_value = True - installer.link_children(templater=templater, src=src_dir, dst=dst_dir, - actionexec=None) + installer.install(templater=templater, src=src_dir, dst=dst_dir, + linktype=LinkTypes.LINK_CHILDREN, actionexec=None) for src in srcs: dst = os.path.join(dst_dir, os.path.basename(src)) From cbc829b2d9879be9f18bb45e7c9aac6750a48d5d Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 17 Nov 2020 14:32:54 +0100 Subject: [PATCH 60/82] add tests --- tests-ng/install.sh | 127 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100755 tests-ng/install.sh diff --git a/tests-ng/install.sh b/tests-ng/install.sh new file mode 100755 index 0000000..16afa60 --- /dev/null +++ b/tests-ng/install.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2020, deadc0de6 +# +# test install +# returns 1 in case of error +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +get_file_mode() +{ + u=`umask` + u=`echo ${u} | sed 's/^0*//'` + v=$((666 - u)) + echo "${v}" +} + +# $1 path +# $2 rights +has_rights() +{ + echo "testing ${1} is ${2}" + [ ! -e "$1" ] && echo "`basename $1` does not exist" && exit 1 + local mode=`stat -L -c '%a' "$1"` + [ "${mode}" != "$2" ] && echo "bad mode for `basename $1` (${mode} VS expected ${2})" && exit 1 + true +} + +# dotdrop directory +basedir=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${basedir}/dotfiles +echo "[+] dotdrop dir: ${basedir}" +echo "[+] dotpath dir: ${basedir}/dotfiles" +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` + +echo "content" > ${basedir}/dotfiles/x + +# create the config file +cfg="${basedir}/config.yaml" +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: + f_x: + src: x + dst: ${tmpd}/x +profiles: + p1: + dotfiles: + - f_x +_EOF + +echo "[+] install" +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 --verbose | grep '^1 dotfile(s) installed.$' +[ "$?" != "0" ] && exit 1 + +[ ! -e ${tmpd}/x ] && echo "f_x not installed" && exit 1 + +# update chmod +chmod 666 ${tmpd}/x +cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 --verbose ${tmpd}/x + +# chmod updated +cat ${cfg} | grep "chmod: '666'" + +chmod 644 ${tmpd}/x + +mode=`get_file_mode ${tmpd}/x` +echo "[+] re-install with no" +cd ${ddpath} | printf "N\n" | ${bin} install -c ${cfg} -p p1 --verbose +[ "$?" != "0" ] && exit 1 + +# if user answers N, chmod should not be done +has_rights "${tmpd}/x" "${mode}" + +echo "[+] re-install with yes" +cd ${ddpath} | printf "y\n" | ${bin} install -c ${cfg} -p p1 --verbose +[ "$?" != "0" ] && exit 1 + +has_rights "${tmpd}/x" "666" + +## CLEANING +rm -rf ${basedir} ${tmpd} + +echo "OK" +exit 0 From c39dea9c7c7795f24c65caee696ffe8c6dc0e006 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 17 Nov 2020 14:33:08 +0100 Subject: [PATCH 61/82] update tests --- tests-ng/chmod-install.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests-ng/chmod-install.sh b/tests-ng/chmod-install.sh index 5c4d363..a36f481 100755 --- a/tests-ng/chmod-install.sh +++ b/tests-ng/chmod-install.sh @@ -10,17 +10,6 @@ # exit on first error set -e -# $1 path -# $2 rights -has_rights() -{ - echo "testing ${1} is ${2}" - [ ! -e "$1" ] && echo "`basename $1` does not exist" && exit 1 - local mode=`stat -L -c '%a' "$1"` - [ "${mode}" != "$2" ] && echo "bad mode for `basename $1` (${mode} VS expected ${2})" && exit 1 - true -} - # all this crap to get current path rl="readlink -f" if ! ${rl} "${0}" >/dev/null 2>&1; then @@ -58,6 +47,17 @@ echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" # this is the test ################################################################ +# $1 path +# $2 rights +has_rights() +{ + echo "testing ${1} is ${2}" + [ ! -e "$1" ] && echo "`basename $1` does not exist" && exit 1 + local mode=`stat -L -c '%a' "$1"` + [ "${mode}" != "$2" ] && echo "bad mode for `basename $1` (${mode} VS expected ${2})" && exit 1 + true +} + get_file_mode() { u=`umask` From 15e2a9c26d6bf3314147baf1e7119d16c05f2bce Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 17 Nov 2020 14:33:22 +0100 Subject: [PATCH 62/82] only chmod if successfully installed --- dotdrop/installer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 4867b77..f833c0a 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -114,8 +114,6 @@ class Installer: self.log.dbg('install {} to {}'.format(src, dst)) self.log.dbg('\"{}\" is a directory: {}'.format(src, isdir)) - # TODO remove template and set templater - # to None when no template should occur if linktype == LinkTypes.NOLINK: # normal file if isdir: @@ -149,7 +147,7 @@ class Installer: template=template) # handle chmod - if chmod and not self.dry: + if r and not err and chmod and not self.dry: dstperms = utils.get_file_perm(dst) if dstperms != chmod: # apply mode From 541c6310fb1a9860b54ef8ab37b223f80341e582 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 17 Nov 2020 21:24:26 +0100 Subject: [PATCH 63/82] fix chmod on user aborted --- dotdrop/installer.py | 114 ++++++++++++++++++++++++++----------------- 1 file changed, 69 insertions(+), 45 deletions(-) diff --git a/dotdrop/installer.py b/dotdrop/installer.py index f833c0a..78463b5 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -147,7 +147,13 @@ class Installer: template=template) # handle chmod - if r and not err and chmod and not self.dry: + # - on success (r, not err) + # - no change (not r, not err) + # but not when + # - error (not r, err) + # - aborted (not r, err) + if (r or (not r and not err)) \ + and chmod and not self.dry: dstperms = utils.get_file_perm(dst) if dstperms != chmod: # apply mode @@ -218,7 +224,15 @@ class Installer: ######################################################## def _link(self, templater, src, dst, actionexec=None, template=True): - """install link:link""" + """ + install link:link + + return + - True, None : success + - False, error_msg : error + - False, None : ignored + - False, 'aborted' : user aborted + """ if template and Templategen.is_template(src): if self.debug: self.log.dbg('is a template') @@ -236,7 +250,15 @@ class Installer: def _link_children(self, templater, src, dst, actionexec=None, template=True): - """install link:link_children""" + """ + install link:link_children + + return + - True, None : success + - False, error_msg : error + - False, None : ignored + - False, 'aborted' : user aborted + """ parent = os.path.join(self.base, src) if not os.path.lexists(dst): self.log.sub('creating directory "{}"'.format(dst)) @@ -249,8 +271,7 @@ class Installer: ]).format(dst) if self.safe and not self.log.ask(msg): - err = 'ignoring "{}", not installed'.format(dst) - return False, err + return False, 'aborted' os.unlink(dst) self._create_dirs(dst) @@ -303,9 +324,10 @@ class Installer: set src as a link target of dst return - - True, None: success - - False, error_msg: error - - False, None, ignored + - True, None : success + - False, error_msg : error + - False, None : ignored + - False, 'aborted' : user aborted """ overwrite = not self.safe if os.path.lexists(dst): @@ -321,8 +343,7 @@ class Installer: self._show_diff_before_write(src, dst) msg = 'Remove "{}" for link creation?'.format(dst) if self.safe and not self.log.ask(msg): - err = 'ignoring "{}", link was not created'.format(dst) - return False, err + return False, 'aborted' overwrite = True try: utils.removepath(dst) @@ -343,8 +364,7 @@ class Installer: if os.path.lexists(dst): msg = 'Remove "{}" for link creation?'.format(dst) if self.safe and not overwrite and not self.log.ask(msg): - err = 'ignoring "{}", link was not created'.format(dst) - return False, err + return False, 'aborted' try: utils.removepath(dst) except OSError as e: @@ -358,7 +378,15 @@ class Installer: actionexec=None, noempty=False, ignore=[], template=True, chmod=None): - """install src to dst when is a file""" + """ + install src to dst when is a file + + return + - True, None : success + - False, error_msg : error + - False, None : ignored + - False, 'aborted' : user aborted + """ if self.debug: self.log.dbg('deploy file: {}'.format(src)) self.log.dbg('ignore empty: {}'.format(noempty)) @@ -411,29 +439,23 @@ class Installer: actionexec=actionexec, template=template, chmod=chmod) - - # build return values - if ret < 0: - # error - return False, err - if ret > 0: - # already exists - if self.debug: - self.log.dbg('ignoring {}'.format(dst)) - return False, None - if ret == 0: - # success + if ret and not err: if not self.dry and not self.comparing: self.log.sub('install {} to {}'.format(src, dst)) - return True, None - # error - err = 'installing {} to {}'.format(src, dst) - return False, err + return ret, err def _copy_dir(self, templater, src, dst, actionexec=None, noempty=False, ignore=[], template=True, chmod=None): - """install src to dst when is a directory""" + """ + install src to dst when is a directory + + return + - True, None : success + - False, error_msg : error + - False, None : ignored + - False, 'aborted' : user aborted + """ if self.debug: self.log.dbg('deploy dir {}'.format(src)) # default to nothing installed and no error @@ -488,16 +510,19 @@ class Installer: chmod=None): """ copy dotfile / write content to file - return 0, None: for success, - 1, None: when already exists - -1, err: when error content is always empty if template is False and is to be ignored + + return + - True, None : success + - False, error_msg : error + - False, None : ignored + - False, 'aborted' : user aborted """ overwrite = not self.safe if self.dry: self.log.dry('would install {}'.format(dst)) - return 0, None + return True, None if os.path.lexists(dst): try: @@ -506,7 +531,7 @@ class Installer: if e.errno == errno.ENOENT: # broken symlink err = 'broken symlink {}'.format(dst) - return -1, err + return False, err src_mode = chmod if not src_mode: @@ -517,7 +542,7 @@ class Installer: src_mode=src_mode): if self.debug: self.log.dbg('{} is the same'.format(dst)) - return 1, None + return False, None if self.safe: if self.debug: self.log.dbg('change detected for {}'.format(dst)) @@ -526,25 +551,24 @@ class Installer: self._show_diff_before_write(src, dst, content=content) if not self.log.ask('Overwrite \"{}\"'.format(dst)): - self.log.warn('ignoring {}'.format(dst)) - return 1, None + return False, 'aborted' overwrite = True if self.backup and os.path.lexists(dst): self._backup(dst) base = os.path.dirname(dst) if not self._create_dirs(base): err = 'creating directory for {}'.format(dst) - return -1, err + return False, err r, e = self._exec_pre_actions(actionexec) if not r: - return -1, e + return False, e if self.debug: self.log.dbg('install file to \"{}\"'.format(dst)) # re-check in case action created the file if self.safe and not overwrite and os.path.lexists(dst): if not self.log.ask('Overwrite \"{}\"'.format(dst)): self.log.warn('ignoring {}'.format(dst)) - return 1, None + return False, 'aborted' if template: # write content the file @@ -554,17 +578,17 @@ class Installer: shutil.copymode(src, dst) except NotADirectoryError as e: err = 'opening dest file: {}'.format(e) - return -1, err + return False, err except Exception as e: - return -1, str(e) + return False, str(e) else: # copy file try: shutil.copyfile(src, dst) shutil.copymode(src, dst) except Exception as e: - return -1, str(e) - return 0, None + return False, str(e) + return True, None ######################################################## # helpers From 7ff5ca3f857b41163dd6620bb1731c09ed1c1292 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 18 Nov 2020 13:56:03 +0100 Subject: [PATCH 64/82] update doc --- docs/config.md | 3 ++- docs/howto/global-config-files.md | 26 -------------------------- docs/howto/howto.md | 2 +- docs/howto/system-config-files.md | 29 +++++++++++++++++++++++++++++ 4 files changed, 32 insertions(+), 28 deletions(-) delete mode 100644 docs/howto/global-config-files.md create mode 100644 docs/howto/system-config-files.md diff --git a/docs/config.md b/docs/config.md index 86a553e..b6863b1 100644 --- a/docs/config.md +++ b/docs/config.md @@ -77,11 +77,12 @@ On `import` the following rules are applied: On `install` the following rules are applied: * if `chmod` is specified in the dotfile, it will be applied to the installed dotfile +* otherwise the permissions of the dotfile in the `dotpath` are applied. On `update`: * if the permissions of the file in the filesystem differ from the dotfile in the `dotpath` - then the dotfile entry `chmod` is updated accordingly + then the dotfile entry `chmod` is added/updated accordingly ## Symlink dotfiles diff --git a/docs/howto/global-config-files.md b/docs/howto/global-config-files.md deleted file mode 100644 index 4b44f1f..0000000 --- a/docs/howto/global-config-files.md +++ /dev/null @@ -1,26 +0,0 @@ -# Manage system dotfiles - -Dotdrop doesn't allow to handle file rights and permissions (at least not directly). Every operations (`mkdir`, `cp`, `mv`, `ln`, file creation) are executed with the rights of the user calling dotdrop. The rights of the stored dotfile are mirrored on the deployed dotfile (`chmod` like). It works well for local/user dotfiles but doesn't allow to manage global/system config files (`/etc` or `/var` for example) directly. - -Using dotdrop with `sudo` to handle local **and** global dotfiles in the same *session* is a bad idea as the resulting files will all have messed up owners. - -It is therefore recommended to have two different config files (and thus two different *dotpath*) for handling these two uses cases: - -* one `config.yaml` for the local/user dotfiles (with its dedicated *dotpath*) -* another config file for the global/system dotfiles (with its dedicated *dotpath*) - -The default config file (`config.yaml`) is used when installing the user dotfiles as usual -```bash -# default config file is config.yaml -$ ./dotdrop.sh import -$ ./dotdrop.sh install -... -``` - -A different config file (for example `global-config.yaml` and its associated *dotpath*) is used when installing/managing global dotfiles and is to be used with `sudo` or directly by the root user -```bash -# specifying explicitly the config file with the --cfg switch -$ sudo ./dotdrop.sh import --cfg=global-config.yaml -$ sudo ./dotdrop.sh install --cfg=global-config.yaml -... -``` \ No newline at end of file diff --git a/docs/howto/howto.md b/docs/howto/howto.md index fbe35cb..41e6d43 100644 --- a/docs/howto/howto.md +++ b/docs/howto/howto.md @@ -28,7 +28,7 @@ ## Manage system dotfiles -[Manage system dotfiles](global-config-files.md) +[Manage system dotfiles](system-config-files.md) ## Merge files on install diff --git a/docs/howto/system-config-files.md b/docs/howto/system-config-files.md new file mode 100644 index 0000000..27ed6a9 --- /dev/null +++ b/docs/howto/system-config-files.md @@ -0,0 +1,29 @@ +# Manage system dotfiles + +Dotdrop doesn't allow to handle file owernership (at least not directly). Every file operations (create/copy file/directory, create symlinks, etc) are executed with the rights of the user calling dotdrop. + +Using dotdrop with `sudo` to unprivileged and privileged files in the same *session* is a bad idea as the resulting files will all have messed up owners. + +It is therefore recommended to have two different config files (and thus two different *dotpath*) +for handling these two uses cases: + +For example: + +* one `config-user.yaml` for the local/user dotfiles (with its dedicated *dotpath*, for example `dotfiles-user`) +* one `config-root.yaml` for the system/root dotfiles (with its dedicated *dotpath*, for example `dotfiles-root`) + +`config-user.yaml` is used when managing the user's dotfiles +```bash +## user config file is config-user.yaml +$ ./dotdrop.sh import --cfg config-user.yaml +$ ./dotdrop.sh install --cfg config-user.yaml +... +``` + +`config-root.yaml` is used when managing system's dotfiles and is to be used with `sudo` or directly by the root user +```bash +## root config file is config-root.yaml +$ sudo ./dotdrop.sh import --cfg=config-root.yaml +$ sudo ./dotdrop.sh install --cfg=config-root.yaml +... +``` From 2d7ae92915f132a9ee2ed32537e793a0374267e2 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 18 Nov 2020 14:10:34 +0100 Subject: [PATCH 65/82] add concurrency doc --- docs/usage.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/usage.md b/docs/usage.md index f1dea23..7acb171 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -226,6 +226,28 @@ dotdrop. It will: For more options, see the usage with `dotdrop --help` +## Concurrency + +The command line switch `-w --workers` if set to a value greater than one allows to use +multiple concurrent workers to execute an operation. It can be applied to the following +commands: + +* `install` +* `compare` +* `update` + +It should be set to a maximum of the number of cores available (usually returned +on linux by the command `nproc`). + +It may speed up the operation but cannot be used interractively (it needs `-f --force` to be set) +and cannot be used with `-d --dry`. Also information printed to stdout/stderr will +probably be messed up. + +**WARNING** this feature hasn't been extensively tested and is to be used at your own risk. +If you try it out and find any issue, please [open an issue](https://github.com/deadc0de6/dotdrop/issues). +Also if you find it useful and have been able to successfully speed up your operation when using +`-w --workers`, do please also report it [in an issue](https://github.com/deadc0de6/dotdrop/issues). + ## Environment variables Following environment variables can be used to specify different CLI options. From f811b91a46dbf56bd5e55c25a75192b480e02394 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 18 Nov 2020 15:07:02 +0100 Subject: [PATCH 66/82] add tests for dry --- tests-ng/dry.sh | 319 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100755 tests-ng/dry.sh diff --git a/tests-ng/dry.sh b/tests-ng/dry.sh new file mode 100755 index 0000000..5707f08 --- /dev/null +++ b/tests-ng/dry.sh @@ -0,0 +1,319 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2020, deadc0de6 +# +# test dry +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +#echo "dotfile destination: ${tmpd}" +# workdir +tmpw=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +# temp +tmpa=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` + +# ----------------------------- +# test install +# ----------------------------- +# cleaning +rm -rf ${tmps}/* +mkdir -p ${tmps}/dotfiles +rm -rf ${tmpw}/* +rm -rf ${tmpd}/* +rm -rf ${tmpa}/* +# create the config file +cfg="${tmps}/config.yaml" + +echo '{{@@ profile @@}}' > ${tmps}/dotfiles/file +echo '{{@@ profile @@}}' > ${tmps}/dotfiles/link +mkdir -p ${tmps}/dotfiles/dir +echo "{{@@ profile @@}}" > ${tmps}/dotfiles/dir/f1 +mkdir -p ${tmps}/dotfiles/dirchildren +echo "{{@@ profile @@}}" > ${tmps}/dotfiles/dirchildren/f1 +echo "{{@@ profile @@}}" > ${tmps}/dotfiles/dirchildren/f2 + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + workdir: ${tmpw} +actions: + pre: + preaction: echo 'pre' > ${tmpa}/pre + post: + postaction: echo 'post' > ${tmpa}/post +dotfiles: + f_file: + src: file + dst: ${tmpd}/file + actions: + - preaction + - postaction + f_link: + src: link + dst: ${tmpd}/link + link: link + actions: + - preaction + - postaction + d_dir: + src: dir + dst: ${tmpd}/dir + actions: + - preaction + - postaction + d_dirchildren: + src: dirchildren + dst: ${tmpd}/dirchildren + link: link_children + actions: + - preaction + - postaction +profiles: + p1: + dotfiles: + - f_file + - f_link + - d_dir + - d_dirchildren +_EOF + +# install +echo "dry install" +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 -V --dry + +cnt=`ls -1 ${tmpd} | wc -l` +ls -1 ${tmpd} +[ "${cnt}" != "0" ] && echo "dry install failed (1)" && exit 1 + +cnt=`ls -1 ${tmpw} | wc -l` +ls -1 ${tmpw} +[ "${cnt}" != "0" ] && echo "dry install failed (2)" && exit 1 + +cnt=`ls -1 ${tmpa} | wc -l` +ls -1 ${tmpa} +[ "${cnt}" != "0" ] && echo "dry install failed (3)" && exit 1 + +# ----------------------------- +# test import +# ----------------------------- +# cleaning +rm -rf ${tmps}/* +mkdir -p ${tmps}/dotfiles +rm -rf ${tmpw}/* +rm -rf ${tmpd}/* +rm -rf ${tmpa}/* + +# create the config file +cfg="${tmps}/config.yaml" +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + workdir: ${tmpw} +dotfiles: +profiles: +_EOF +cp ${cfg} ${tmpa}/config.yaml + +echo 'content' > ${tmpd}/file +echo 'content' > ${tmpd}/link +mkdir -p ${tmpd}/dir +echo "content" > ${tmpd}/dir/f1 +mkdir -p ${tmpd}/dirchildren +echo "content" > ${tmpd}/dirchildren/f1 +echo "content" > ${tmpd}/dirchildren/f2 + +dotfiles="${tmpd}/file ${tmpd}/link ${tmpd}/dir ${tmpd}/dirchildren" + +echo "dry import" +cd ${ddpath} | ${bin} import -c ${cfg} -f -p p1 -V --dry ${dotfiles} + +cnt=`ls -1 ${tmps}/dotfiles | wc -l` +ls -1 ${tmps}/dotfiles +[ "${cnt}" != "0" ] && echo "dry import failed (1)" && exit 1 + +diff ${cfg} ${tmpa}/config.yaml || (echo "dry import failed (2)" && exit 1) + +# ----------------------------- +# test update +# ----------------------------- +# cleaning +rm -rf ${tmps}/* +mkdir -p ${tmps}/dotfiles +rm -rf ${tmpw}/* +rm -rf ${tmpd}/* +rm -rf ${tmpa}/* + +echo 'original' > ${tmps}/dotfiles/file +echo 'original' > ${tmps}/dotfiles/link +mkdir -p ${tmps}/dotfiles/dir +echo "original" > ${tmps}/dotfiles/dir/f1 +mkdir -p ${tmps}/dotfiles/dirchildren +echo "original" > ${tmps}/dotfiles/dirchildren/f1 +echo "original" > ${tmps}/dotfiles/dirchildren/f2 + +echo 'modified' > ${tmpd}/file +echo 'modified' > ${tmpd}/link +mkdir -p ${tmpd}/dir +echo "modified" > ${tmpd}/dir/f1 +mkdir -p ${tmpd}/dirchildren +echo "modified" > ${tmpd}/dirchildren/f1 +echo "modified" > ${tmpd}/dirchildren/f2 + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + workdir: ${tmpw} +dotfiles: + f_file: + src: file + dst: ${tmpd}/file + f_link: + src: link + dst: ${tmpd}/link + link: link + d_dir: + src: dir + dst: ${tmpd}/dir + d_dirchildren: + src: dirchildren + dst: ${tmpd}/dirchildren + link: link_children +profiles: + p1: + dotfiles: + - f_file + - f_link + - d_dir + - d_dirchildren +_EOF +cp ${cfg} ${tmpa}/config.yaml + +echo "dry update" +dotfiles="${tmpd}/file ${tmpd}/link ${tmpd}/dir ${tmpd}/dirchildren" +cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V --dry ${dotfiles} + +grep 'modified' ${tmps}/dotfiles/file && echo "dry update failed (1)" && exit 1 +grep 'modified' ${tmps}/dotfiles/link && echo "dry update failed (2)" && exit 1 +grep "modified" ${tmps}/dotfiles/dir/f1 && echo "dry update failed (3)" && exit 1 +grep "modified" ${tmps}/dotfiles/dirchildren/f1 && echo "dry update failed (4)" && exit 1 +grep "modified" ${tmps}/dotfiles/dirchildren/f2 && echo "dry update failed (5)" && exit 1 + +diff ${cfg} ${tmpa}/config.yaml || (echo "dry update failed (6)" && exit 1) + +# ----------------------------- +# test remove +# ----------------------------- +# cleaning +rm -rf ${tmps}/* +mkdir -p ${tmps}/dotfiles +rm -rf ${tmpw}/* +rm -rf ${tmpd}/* +rm -rf ${tmpa}/* + +echo '{{@@ profile @@}}' > ${tmps}/dotfiles/file +echo '{{@@ profile @@}}' > ${tmps}/dotfiles/link +mkdir -p ${tmps}/dotfiles/dir +echo "{{@@ profile @@}}" > ${tmps}/dotfiles/dir/f1 +mkdir -p ${tmps}/dotfiles/dirchildren +echo "{{@@ profile @@}}" > ${tmps}/dotfiles/dirchildren/f1 +echo "{{@@ profile @@}}" > ${tmps}/dotfiles/dirchildren/f2 + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + workdir: ${tmpw} +dotfiles: + f_file: + src: file + dst: ${tmpd}/file + f_link: + src: link + dst: ${tmpd}/link + link: link + d_dir: + src: dir + dst: ${tmpd}/dir + d_dirchildren: + src: dirchildren + dst: ${tmpd}/dirchildren + link: link_children +profiles: + p1: + dotfiles: + - f_file + - f_link + - d_dir + - d_dirchildren +_EOF +cp ${cfg} ${tmpa}/config.yaml + +echo "dry remove" +dotfiles="${tmpd}/file ${tmpd}/link ${tmpd}/dir ${tmpd}/dirchildren" +cd ${ddpath} | ${bin} remove -c ${cfg} -f -p p1 -V --dry ${dotfiles} + +[ ! -e ${tmps}/dotfiles/file ] && echo "dry remove failed (1)" && exit 1 +[ ! -e ${tmps}/dotfiles/link ] && echo "dry remove failed (2)" && exit 1 +[ ! -d ${tmps}/dotfiles/dir ] && echo "dry remove failed (3)" && exit 1 +[ ! -e ${tmps}/dotfiles/dir/f1 ] && echo "dry remove failed (4)" && exit 1 +[ ! -d ${tmps}/dotfiles/dirchildren ] && echo "dry remove failed (5)" && exit 1 +[ ! -e ${tmps}/dotfiles/dirchildren/f1 ] && echo "dry remove failed (6)" && exit 1 +[ ! -e ${tmps}/dotfiles/dirchildren/f2 ] && echo "dry remove failed (7)" && exit 1 + +diff ${cfg} ${tmpa}/config.yaml || (echo "dry remove failed (8)" && exit 1) + +## CLEANING +rm -rf ${tmps} ${tmpd} ${tmpw} ${tmpa} + +echo "OK" +exit 0 From ace846dcfbb404a1568707110f3636debda18a3e Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 18 Nov 2020 15:07:13 +0100 Subject: [PATCH 67/82] fix test --- tests-ng/chmod-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests-ng/chmod-install.sh b/tests-ng/chmod-install.sh index a36f481..1a2298f 100755 --- a/tests-ng/chmod-install.sh +++ b/tests-ng/chmod-install.sh @@ -191,7 +191,7 @@ _EOF # install echo "first install round" -cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 -V ${i} +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 -V has_rights "${tmpd}/f777" "777" has_rights "${tmpd}/link" "777" @@ -224,7 +224,7 @@ chmod 700 ${tmpd}/linkchildren chmod 600 ${tmpd}/symlinktemplate echo "second install round" -cd ${ddpath} | ${bin} install -c ${cfg} -p p2 -f -V ${i} +cd ${ddpath} | ${bin} install -c ${cfg} -p p2 -f -V has_rights "${tmpd}/exists" "777" has_rights "${tmpd}/existslink" "777" From 48756a295e86cbca886f43aea693e4cb73ef7a28 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 18 Nov 2020 15:07:24 +0100 Subject: [PATCH 68/82] logs --- dotdrop/dotdrop.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index f9cdcc4..442f862 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -575,7 +575,8 @@ def cmd_remove(o): k = dotfile.key # ignore if uses any type of link if dotfile.link != LinkTypes.NOLINK: - LOG.warn('dotfile uses link, remove manually') + msg = '{} uses link/link_children, remove manually' + LOG.warn(msg.format(k)) continue if o.debug: From ef6d78ed459cad2dedf46d722e8ea766a0ec24f1 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 18 Nov 2020 15:07:37 +0100 Subject: [PATCH 69/82] fix dry on link_children --- dotdrop/installer.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 78463b5..bf0dece 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -261,8 +261,11 @@ class Installer: """ parent = os.path.join(self.base, src) if not os.path.lexists(dst): - self.log.sub('creating directory "{}"'.format(dst)) - os.makedirs(dst) + if self.dry: + self.log.dry('would create directory "{}"'.format(dst)) + else: + self.log.sub('creating directory "{}"'.format(dst)) + os.makedirs(dst) if os.path.isfile(dst): msg = ''.join([ From 578fb86466588266eab1ed0ddf8d6fff086b01a2 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 18 Nov 2020 15:23:28 +0100 Subject: [PATCH 70/82] speed up template detection --- dotdrop/templategen.py | 18 ++++++++++++------ tests-ng/include-order.sh | 4 ++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/dotdrop/templategen.py b/dotdrop/templategen.py index 3e34622..a90b787 100644 --- a/dotdrop/templategen.py +++ b/dotdrop/templategen.py @@ -6,6 +6,9 @@ jinja2 template generator """ import os +import io +import re +import mmap from jinja2 import Environment, FileSystemLoader, \ ChoiceLoader, FunctionLoader, TemplateNotFound, \ StrictUndefined @@ -244,16 +247,19 @@ class Templategen: """test if file pointed by path is a template""" if not os.path.isfile(path): return False + if os.stat(path).st_size == 0: + return False + markers = [BLOCK_START, VAR_START, COMMENT_START] + patterns = [re.compile(marker.encode()) for marker in markers] try: - with open(path, 'r') as f: - data = f.read() + with io.open(path, "r", encoding="utf-8") as f: + m = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + for pattern in patterns: + if pattern.search(m): + return True except UnicodeDecodeError: # is binary so surely no template return False - markers = [BLOCK_START, VAR_START, COMMENT_START] - for marker in markers: - if marker in data: - return True return False def _debug_dict(self, title, elems): diff --git a/tests-ng/include-order.sh b/tests-ng/include-order.sh index 471edf9..b2a06df 100755 --- a/tests-ng/include-order.sh +++ b/tests-ng/include-order.sh @@ -67,8 +67,8 @@ config: actions: pre: first: 'echo first > ${tmpa}/cookie' - second: 'echo second >> ${tmpa}/cookie' - third: 'echo third >> ${tmpa}/cookie' + second: 'sleep 1; echo second >> ${tmpa}/cookie' + third: 'sleep 1; echo third >> ${tmpa}/cookie' dotfiles: f_first: dst: ${tmpd}/first From c7671165c3255cacea5d60b42ecbb531f90545ff Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 18 Nov 2020 15:37:41 +0100 Subject: [PATCH 71/82] fix order test with epoch --- tests-ng/include-order.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests-ng/include-order.sh b/tests-ng/include-order.sh index b2a06df..3c261e4 100755 --- a/tests-ng/include-order.sh +++ b/tests-ng/include-order.sh @@ -116,9 +116,9 @@ for ((i=0;i<${attempts};i++)); do echo "second timestamp: `stat -c %y ${tmpd}/second`" echo "third timestamp: `stat -c %y ${tmpd}/third`" - ts_first=`date "+%S%N" -d "$(stat -c %y ${tmpd}/first)"` - ts_second=`date "+%S%N" -d "$(stat -c %y ${tmpd}/second)"` - ts_third=`date "+%S%N" -d "$(stat -c %y ${tmpd}/third)"` + ts_first=`date "+%s" -d "$(stat -c %y ${tmpd}/first)"` + ts_second=`date "+%s" -d "$(stat -c %y ${tmpd}/second)"` + ts_third=`date "+%s" -d "$(stat -c %y ${tmpd}/third)"` #echo "first ts: ${ts_first}" #echo "second ts: ${ts_second}" From 962cb775d906cd4ef6465bbe29dcfbf597e1c26d Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 18 Nov 2020 15:39:30 +0100 Subject: [PATCH 72/82] improve compare speed --- dotdrop/dotdrop.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 442f862..ef30e0d 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -133,19 +133,23 @@ def _dotfile_compare(o, dotfile, tmp): LOG.dbg('points to itself') return True - # install dotfile to temporary dir and compare - ret, err, insttmp = inst.install_to_temp(t, tmp, src, dotfile.dst, - template=dotfile.template, - chmod=dotfile.chmod) - if not ret: - # failed to install to tmp - line = '=> compare {}: error' - LOG.log(line.format(dotfile.key, err)) - LOG.err(err) - return False + if dotfile.template or Templategen.is_template(src): + # install dotfile to temporary dir for compare + ret, err, insttmp = inst.install_to_temp(t, tmp, src, dotfile.dst, + template=dotfile.template, + chmod=dotfile.chmod) + if not ret: + # failed to install to tmp + line = '=> compare {} error: {}' + LOG.log(line.format(dotfile.key, err)) + LOG.err(err) + return False + src = insttmp + + # compare ignores = list(set(o.compare_ignore + dotfile.cmpignore)) ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) - diff = comp.compare(insttmp, dotfile.dst, ignore=ignores) + diff = comp.compare(src, dotfile.dst, ignore=ignores) # clean tmp transformed dotfile if any if tmpsrc: @@ -153,6 +157,11 @@ def _dotfile_compare(o, dotfile, tmp): if os.path.exists(tmpsrc): removepath(tmpsrc, LOG) + # clean tmp template dotfile if any + if insttmp: + if os.path.exists(insttmp): + removepath(insttmp, LOG) + if diff != '': # print diff results line = '=> compare {}: diffing with \"{}\"' From 0c0753faaf3e8719373bf30ff47bb9c41b336306 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 18 Nov 2020 15:45:15 +0100 Subject: [PATCH 73/82] typo --- dotdrop/installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotdrop/installer.py b/dotdrop/installer.py index bf0dece..51ecf99 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -179,7 +179,7 @@ class Installer: @chmod: rights to apply if any return - - success, error-if-any, dir-where-installed + - success, error-if-any, dotfile-installed-path """ if self.debug: self.log.dbg('tmp install {} (defined dst: {})'.format(src, dst)) From 2ee70ff8ab424c9cfb7001030a4e21b8d53e83f8 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 19 Nov 2020 15:03:22 +0100 Subject: [PATCH 74/82] add more logs --- tests-ng/transformations.sh | 46 ++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/tests-ng/transformations.sh b/tests-ng/transformations.sh index 0bd329f..470e63e 100755 --- a/tests-ng/transformations.sh +++ b/tests-ng/transformations.sh @@ -89,6 +89,7 @@ dotfiles: src: ghi trans: uncompress trans_write: compress + chmod: 700 profiles: p1: dotfiles: @@ -125,40 +126,43 @@ tar -tf ${tmps}/dotfiles/ghi # test install and compare ########################### +echo "[+] run install" # install cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -b -V # check canary dotfile -[ ! -e ${tmpd}/def ] && exit 1 +[ ! -e ${tmpd}/def ] && echo "def does not exist" && exit 1 # check base64 dotfile -[ ! -e ${tmpd}/abc ] && exit 1 +[ ! -e ${tmpd}/abc ] && echo "abc does not exist" && exit 1 content=`cat ${tmpd}/abc` -[ "${content}" != "${token}" ] && exit 1 +[ "${content}" != "${token}" ] && echo "bad content for abc" && exit 1 # check directory dotfile -[ ! -e ${tmpd}/ghi/a/dir1/otherfile ] && exit 1 +[ ! -e ${tmpd}/ghi/a/dir1/otherfile ] && echo "otherfile does not exist" && exit 1 content=`cat ${tmpd}/ghi/a/somefile` -[ "${content}" != "${tokend}" ] && exit 1 +[ "${content}" != "${tokend}" ] && echo "bad content for somefile" && exit 1 content=`cat ${tmpd}/ghi/a/dir1/otherfile` -[ "${content}" != "${tokend}" ] && exit 1 +[ "${content}" != "${tokend}" ] && echo "bad content for otherfile" && exit 1 # compare +set +e cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 -b -V -[ "$?" != "0" ] && exit 1 +[ "$?" != "0" ] && echo "compare failed (0)" && exit 1 +set -e # change base64 deployed file echo ${touched} > ${tmpd}/abc set +e cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 -b -V -[ "$?" != "1" ] && exit 1 +[ "$?" != "1" ] && echo "compare failed (1)" && exit 1 set -e # change uncompressed deployed dotfile echo ${touched} > ${tmpd}/ghi/a/somefile set +e cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 -b -V -[ "$?" != "1" ] && exit 1 +[ "$?" != "1" ] && echo "compare failed (2)" && exit 1 set -e ########################### @@ -167,38 +171,44 @@ set -e # update single file echo 'update' > ${tmpd}/def +set +e cd ${ddpath} | ${bin} update -f -k -c ${cfg} -p p1 -b -V f_def -[ "$?" != "0" ] && exit 1 +[ "$?" != "0" ] && echo "update failed (1)" && exit 1 +set -e [ ! -e ${tmpd}/def ] && echo 'dotfile in FS removed' && exit 1 [ ! -e ${tmps}/dotfiles/def ] && echo 'dotfile in dotpath removed' && exit 1 # update single file +set +e cd ${ddpath} | ${bin} update -f -k -c ${cfg} -p p1 -b -V f_abc -[ "$?" != "0" ] && exit 1 +[ "$?" != "0" ] && echo "update failed (2)" && exit 1 +set -e # test updated file -[ ! -e ${tmps}/dotfiles/abc ] && exit 1 +[ ! -e ${tmps}/dotfiles/abc ] && echo "abc does not exist" && exit 1 content=`cat ${tmps}/dotfiles/abc` bcontent=`echo ${touched} | base64` -[ "${content}" != "${bcontent}" ] && exit 1 +[ "${content}" != "${bcontent}" ] && echo "bad content for abc" && exit 1 # update directory echo ${touched} > ${tmpd}/ghi/b/newfile rm -r ${tmpd}/ghi/c cd ${ddpath} | ${bin} update -f -k -c ${cfg} -p p1 -b -V d_ghi -[ "$?" != "0" ] && exit 1 +[ "$?" != "0" ] && echo "update failed" && exit 1 # test updated directory -tar -tf ${tmps}/dotfiles/ghi | grep './b/newfile' -tar -tf ${tmps}/dotfiles/ghi | grep './a/dir1/otherfile' +set +e +tar -tf ${tmps}/dotfiles/ghi | grep './b/newfile' || (echo "newfile not found in tar" && exit 1) +tar -tf ${tmps}/dotfiles/ghi | grep './a/dir1/otherfile' || (echo "otherfile not found in tar" && exit 1) +set -e tmpy=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` tar -xf ${tmps}/dotfiles/ghi -C ${tmpy} content=`cat ${tmpy}/a/somefile` -[ "${content}" != "${touched}" ] && exit 1 +[ "${content}" != "${touched}" ] && echo "bad content" && exit 1 # check canary dotfile -[ ! -e ${tmps}/dotfiles/def ] && exit 1 +[ ! -e ${tmps}/dotfiles/def ] && echo "def not found" && exit 1 ## CLEANING rm -rf ${tmps} ${tmpd} ${tmpx} ${tmpy} From d253610f2dbe1eed8eed3fd21730ed62abc702d9 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 19 Nov 2020 15:03:38 +0100 Subject: [PATCH 75/82] debug --- dotdrop/comparator.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dotdrop/comparator.py b/dotdrop/comparator.py index 78c1dd4..6801493 100644 --- a/dotdrop/comparator.py +++ b/dotdrop/comparator.py @@ -43,6 +43,8 @@ class Comparator: # test content if not os.path.isdir(left): + if self.debug: + self.log.dbg('{} is a file'.format(left)) if self.debug: self.log.dbg('is file') ret = self._comp_file(left, right, ignore) @@ -51,7 +53,7 @@ class Comparator: return ret if self.debug: - self.log.dbg('is directory') + self.log.dbg('{} is a directory'.format(left)) ret = self._comp_dir(left, right, ignore) if not ret: @@ -64,7 +66,11 @@ class Comparator: right_mode = get_file_perm(right) if left_mode == right_mode: return '' - ret = 'modes differ ({:o} vs {:o})\n'.format(left_mode, right_mode) + if self.debug: + msg = 'mode differ {} ({:o}) and {} ({:o})' + self.log.dbg(msg.format(left, left_mode, right, right_mode)) + ret = 'modes differ for {} ({:o}) vs {:o}\n' + ret.format(right, right_mode, left_mode) return ret def _comp_file(self, left, right, ignore): From 6ab0b009306f1cafd6f4f4763e7ac317d99c6b7f Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 19 Nov 2020 15:04:11 +0100 Subject: [PATCH 76/82] provide installer with is_template info --- dotdrop/dotdrop.py | 16 +++++++------ dotdrop/installer.py | 54 ++++++++++++++++++++----------------------- tests/test_install.py | 7 ++---- 3 files changed, 36 insertions(+), 41 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index ef30e0d..af20daf 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -133,10 +133,11 @@ def _dotfile_compare(o, dotfile, tmp): LOG.dbg('points to itself') return True - if dotfile.template or Templategen.is_template(src): + insttmp = None + if dotfile.template and Templategen.is_template(src): # install dotfile to temporary dir for compare ret, err, insttmp = inst.install_to_temp(t, tmp, src, dotfile.dst, - template=dotfile.template, + is_template=True, chmod=dotfile.chmod) if not ret: # failed to install to tmp @@ -205,12 +206,13 @@ def _dotfile_install(o, dotfile, tmpdir=None): LOG.dbg('installing dotfile: \"{}\"'.format(dotfile.key)) LOG.dbg(dotfile.prt()) + is_template = dotfile.template and Templategen.is_template(dotfile.src) if hasattr(dotfile, 'link') and dotfile.link == LinkTypes.LINK: # link r, err = inst.install(t, dotfile.src, dotfile.dst, dotfile.link, actionexec=pre_actions_exec, - template=dotfile.template, + is_template=is_template, chmod=dotfile.chmod) elif hasattr(dotfile, 'link') and \ dotfile.link == LinkTypes.LINK_CHILDREN: @@ -218,7 +220,7 @@ def _dotfile_install(o, dotfile, tmpdir=None): r, err = inst.install(t, dotfile.src, dotfile.dst, dotfile.link, actionexec=pre_actions_exec, - template=dotfile.template, + is_template=is_template, chmod=dotfile.chmod) else: # nolink @@ -236,7 +238,7 @@ def _dotfile_install(o, dotfile, tmpdir=None): actionexec=pre_actions_exec, noempty=dotfile.noempty, ignore=ignores, - template=dotfile.template, + is_template=is_template, chmod=dotfile.chmod) if tmp: tmp = os.path.join(o.dotpath, tmp) @@ -677,7 +679,7 @@ def _detail(dotpath, dotfile): path = os.path.join(dotpath, os.path.expanduser(dotfile.src)) if not os.path.isdir(path): template = 'no' - if Templategen.is_template(path): + if dotfile.template and Templategen.is_template(path): template = 'yes' LOG.sub('{} (template:{})'.format(path, template)) else: @@ -685,7 +687,7 @@ def _detail(dotpath, dotfile): for f in files: p = os.path.join(root, f) template = 'no' - if Templategen.is_template(p): + if dotfile.template and Templategen.is_template(p): template = 'yes' LOG.sub('{} (template:{})'.format(p, template)) diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 51ecf99..248c7bf 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -12,7 +12,6 @@ import shutil # local imports from dotdrop.logger import Logger from dotdrop.linktypes import LinkTypes -from dotdrop.templategen import Templategen import dotdrop.utils as utils from dotdrop.exceptions import UndefinedException @@ -66,7 +65,7 @@ class Installer: def install(self, templater, src, dst, linktype, actionexec=None, noempty=False, - ignore=[], template=True, + ignore=[], is_template=True, chmod=None): """ install src to dst @@ -78,7 +77,7 @@ class Installer: @actionexec: action executor callback @noempty: render empty template flag @ignore: pattern to ignore when installing - @template: template this dotfile + @is_template: this dotfile is a template @chmod: rights to apply if any return @@ -105,7 +104,7 @@ class Installer: # and ignore any actions if self.totemp: r, err, _ = self.install_to_temp(templater, self.totemp, - src, dst, template=template, + src, dst, is_template=is_template, chmod=chmod) return self._log_install(r, err) @@ -120,19 +119,19 @@ class Installer: r, err = self._copy_dir(templater, src, dst, actionexec=actionexec, noempty=noempty, ignore=ignore, - template=template, + is_template=is_template, chmod=chmod) else: r, err = self._copy_file(templater, src, dst, actionexec=actionexec, noempty=noempty, ignore=ignore, - template=template, + is_template=is_template, chmod=chmod) elif linktype == LinkTypes.LINK: # symlink r, err = self._link(templater, src, dst, actionexec=actionexec, - template=template) + is_template=is_template) elif linktype == LinkTypes.LINK_CHILDREN: # symlink direct children if not isdir: @@ -144,7 +143,7 @@ class Installer: else: r, err = self._link_children(templater, src, dst, actionexec=actionexec, - template=template) + is_template=is_template) # handle chmod # - on success (r, not err) @@ -167,7 +166,7 @@ class Installer: return self._log_install(r, err) def install_to_temp(self, templater, tmpdir, src, dst, - template=True, chmod=None): + is_template=True, chmod=None): """ install a dotfile to a tempdir @@ -175,7 +174,7 @@ class Installer: @tmpdir: where to install @src: dotfile source path in dotpath @dst: dotfile destination path in the FS - @template: template this dotfile + @is_template: this dotfile is a template @chmod: rights to apply if any return @@ -205,7 +204,8 @@ class Installer: tmpdst = self._pivot_path(dst, tmpdir) ret, err = self.install(templater, src, tmpdst, LinkTypes.NOLINK, - template=template, chmod=chmod) + is_template=is_template, + chmod=chmod) if self.debug: if ret: self.log.dbg('tmp installed in {}'.format(tmpdst)) @@ -223,7 +223,7 @@ class Installer: # low level accessors for public methods ######################################################## - def _link(self, templater, src, dst, actionexec=None, template=True): + def _link(self, templater, src, dst, actionexec=None, is_template=True): """ install link:link @@ -233,7 +233,7 @@ class Installer: - False, None : ignored - False, 'aborted' : user aborted """ - if template and Templategen.is_template(src): + if is_template: if self.debug: self.log.dbg('is a template') self.log.dbg('install to {}'.format(self.workdir)) @@ -241,7 +241,7 @@ class Installer: r, err = self.install(templater, src, tmp, LinkTypes.NOLINK, actionexec=actionexec, - template=template) + is_template=is_template) if not r and not os.path.exists(tmp): return r, err src = tmp @@ -249,7 +249,7 @@ class Installer: return r, err def _link_children(self, templater, src, dst, - actionexec=None, template=True): + actionexec=None, is_template=True): """ install link:link_children @@ -292,7 +292,7 @@ class Installer: if self.debug: self.log.dbg('symlink child {} to {}'.format(subsrc, subdst)) - if template and Templategen.is_template(subsrc): + if is_template: if self.debug: self.log.dbg('child is a template') self.log.dbg('install to {} and symlink' @@ -301,7 +301,7 @@ class Installer: r, e = self.install(templater, subsrc, tmp, LinkTypes.NOLINK, actionexec=actionexec, - template=template) + is_template=is_template) if not r and e and not os.path.exists(tmp): continue subsrc = tmp @@ -379,7 +379,7 @@ class Installer: def _copy_file(self, templater, src, dst, actionexec=None, noempty=False, - ignore=[], template=True, + ignore=[], is_template=True, chmod=None): """ install src to dst when is a file @@ -394,7 +394,7 @@ class Installer: self.log.dbg('deploy file: {}'.format(src)) self.log.dbg('ignore empty: {}'.format(noempty)) self.log.dbg('ignore pattern: {}'.format(ignore)) - self.log.dbg('template: {}'.format(template)) + self.log.dbg('is_template: {}'.format(is_template)) self.log.dbg('no empty: {}'.format(noempty)) # check no loop @@ -418,7 +418,7 @@ class Installer: # handle the file content = None - if template: + if is_template: # template the file saved = templater.add_tmp_vars(self._get_tmp_file_vars(src, dst)) try: @@ -440,7 +440,6 @@ class Installer: ret, err = self._write(src, dst, content=content, actionexec=actionexec, - template=template, chmod=chmod) if ret and not err: if not self.dry and not self.comparing: @@ -449,7 +448,7 @@ class Installer: def _copy_dir(self, templater, src, dst, actionexec=None, noempty=False, - ignore=[], template=True, chmod=None): + ignore=[], is_template=True, chmod=None): """ install src to dst when is a directory @@ -481,7 +480,7 @@ class Installer: actionexec=actionexec, noempty=noempty, ignore=ignore, - template=template, + is_template=is_template, chmod=None) if not res and err: # error occured @@ -497,7 +496,7 @@ class Installer: actionexec=actionexec, noempty=noempty, ignore=ignore, - template=template, + is_template=is_template, chmod=None) if not res and err: # error occured @@ -509,12 +508,9 @@ class Installer: return ret def _write(self, src, dst, content=None, - actionexec=None, template=True, - chmod=None): + actionexec=None, chmod=None): """ copy dotfile / write content to file - content is always empty if template is False - and is to be ignored return - True, None : success @@ -573,7 +569,7 @@ class Installer: self.log.warn('ignoring {}'.format(dst)) return False, 'aborted' - if template: + if content: # write content the file try: with open(dst, 'wb') as f: diff --git a/tests/test_install.py b/tests/test_install.py index cbac55a..c20cd4e 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -6,7 +6,7 @@ basic unittest for the install function import os import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import filecmp from dotdrop.cfg_aggregator import CfgAggregator as Cfg @@ -460,8 +460,7 @@ exec bspwm 'Remove regular file {} and replace with empty directory?' .format(dst)) - @patch('dotdrop.installer.Templategen') - def test_runs_templater(self, mocked_templategen): + def test_runs_templater(self): """test runs templater""" # create source dir src_dir = get_tempdir() @@ -480,8 +479,6 @@ exec bspwm installer = Installer() templater = MagicMock() templater.generate.return_value = b'content' - # make templategen treat everything as a template - mocked_templategen.is_template.return_value = True installer.install(templater=templater, src=src_dir, dst=dst_dir, linktype=LinkTypes.LINK_CHILDREN, actionexec=None) From 5e7309a3ecc5fa6629129e88d2431b27be1da9f4 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 19 Nov 2020 19:54:09 +0100 Subject: [PATCH 77/82] fixes for #286 --- dotdrop/dotdrop.py | 16 ++++++++----- dotdrop/installer.py | 50 ++++++++++++++++----------------------- tests-ng/chmod-install.sh | 3 +++ tests/test_import.py | 4 ++-- tests/test_listings.py | 6 ++--- 5 files changed, 39 insertions(+), 40 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index af20daf..af27493 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -510,7 +510,7 @@ def cmd_list_profiles(o): LOG.log('') -def cmd_list_files(o): +def cmd_files(o): """list all dotfiles for a specific profile""" if o.profile not in [p.key for p in o.profiles]: LOG.warn('unknown profile \"{}\"'.format(o.profile)) @@ -525,17 +525,21 @@ def cmd_list_files(o): if not Templategen.is_template(src): continue if o.files_grepable: - fmt = '{},dst:{},src:{},link:{},chmod:{}' + fmt = '{},dst:{},src:{},link:{}' fmt = fmt.format(dotfile.key, dotfile.dst, - dotfile.src, dotfile.link.name.lower(), - dotfile.chmod) + dotfile.src, dotfile.link.name.lower()) + if dotfile.chmod: + fmt += ',chmod:{:o}' + else: + fmt += ',chmod:None' LOG.raw(fmt) else: LOG.log('{}'.format(dotfile.key), bold=True) LOG.sub('dst: {}'.format(dotfile.dst)) LOG.sub('src: {}'.format(dotfile.src)) LOG.sub('link: {}'.format(dotfile.link.name.lower())) - LOG.sub('chmod: {}'.format(dotfile.chmod)) + if dotfile.chmod: + LOG.sub('chmod: {:o}'.format(dotfile.chmod)) LOG.log('') @@ -773,7 +777,7 @@ def main(): command = 'files' if o.debug: LOG.dbg('running cmd: {}'.format(command)) - cmd_list_files(o) + cmd_files(o) elif o.cmd_install: # install the dotfiles stored in dotdrop diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 248c7bf..1e5428f 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -145,23 +145,35 @@ class Installer: actionexec=actionexec, is_template=is_template) + if self.debug: + self.log.dbg('before chmod: {} err:{}'.format(r, err)) + + if self.dry: + return self._log_install(r, err) + # handle chmod # - on success (r, not err) # - no change (not r, not err) # but not when # - error (not r, err) # - aborted (not r, err) - if (r or (not r and not err)) \ - and chmod and not self.dry: + if (r or (not r and not err)): + if not chmod: + chmod = utils.get_file_perm(src) dstperms = utils.get_file_perm(dst) if dstperms != chmod: # apply mode - self.log.sub('chmod {} to {:o}'.format(dst, chmod)) - if utils.chmod(dst, chmod, debug=self.debug): - r = True - else: + msg = 'chmod {} to {:o}'.format(dst, chmod) + if self.safe and not self.log.ask(msg): r = False - err = 'chmod failed' + err = 'aborted' + else: + self.log.sub('chmod {} to {:o}'.format(dst, chmod)) + if utils.chmod(dst, chmod, debug=self.debug): + r = True + else: + r = False + err = 'chmod failed' return self._log_install(r, err) @@ -536,9 +548,7 @@ class Installer: if not src_mode: src_mode = utils.get_file_perm(src) if self.diff: - if not self._is_different(src, dst, - content=content, - src_mode=src_mode): + if not self._is_different(src, dst, content=content): if self.debug: self.log.dbg('{} is the same'.format(dst)) return False, None @@ -599,29 +609,11 @@ class Installer: tmp['_dotfile_sub_abs_dst'] = dst return tmp - def _is_different(self, src, dst, src_mode=None, content=None): + def _is_different(self, src, dst, content=None): """ returns True if file is different and needs to be installed """ - # check file size - src_size = os.stat(src).st_size - dst_size = os.stat(dst).st_size - if src_size != dst_size: - if self.debug: - self.log.dbg('size differ') - return True - - # check file mode - if not src_mode: - src_mode = utils.get_file_perm(src) - dst_mode = utils.get_file_perm(dst) - if src_mode != dst_mode: - if self.debug: - m = 'mode differ ({:o} vs {:o})' - self.log.dbg(m.format(src_mode, dst_mode)) - return True - # check file content if content: tmp = utils.write_to_tmpfile(content) diff --git a/tests-ng/chmod-install.sh b/tests-ng/chmod-install.sh index 1a2298f..e59562b 100755 --- a/tests-ng/chmod-install.sh +++ b/tests-ng/chmod-install.sh @@ -243,6 +243,7 @@ chmod 600 ${tmps}/dotfiles/nomode echo "nomode" > ${tmpd}/nomode chmod 600 ${tmpd}/nomode cd ${ddpath} | ${bin} install -c ${cfg} -f -p p2 -V f_nomode +echo "same mode" has_rights "${tmpd}/nomode" "600" ## no user confirmation with force @@ -253,6 +254,7 @@ chmod 600 ${tmps}/dotfiles/nomode echo "nomode" > ${tmpd}/nomode chmod 700 ${tmpd}/nomode cd ${ddpath} | ${bin} install -c ${cfg} -f -p p2 -V f_nomode +echo "different mode (1)" has_rights "${tmpd}/nomode" "600" ## user confirmation expected @@ -263,6 +265,7 @@ chmod 600 ${tmps}/dotfiles/nomode echo "nomode" > ${tmpd}/nomode chmod 700 ${tmpd}/nomode cd ${ddpath} | printf 'y\ny\n' | ${bin} install -f -c ${cfg} -p p2 -V f_nomode +echo "different mode (2)" has_rights "${tmpd}/nomode" "600" ## CLEANING diff --git a/tests/test_import.py b/tests/test_import.py index 08df58c..186382a 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -10,7 +10,7 @@ import os from dotdrop.dotdrop import cmd_importer from dotdrop.dotdrop import cmd_list_profiles -from dotdrop.dotdrop import cmd_list_files +from dotdrop.dotdrop import cmd_files from dotdrop.dotdrop import cmd_update from dotdrop.linktypes import LinkTypes @@ -184,7 +184,7 @@ class TestImport(unittest.TestCase): self.assertTrue(os.path.exists(s4)) cmd_list_profiles(o) - cmd_list_files(o) + cmd_files(o) # fake test update editcontent = 'edited' diff --git a/tests/test_listings.py b/tests/test_listings.py index 39905ec..72a82e6 100644 --- a/tests/test_listings.py +++ b/tests/test_listings.py @@ -9,7 +9,7 @@ import unittest import os from dotdrop.dotdrop import cmd_list_profiles -from dotdrop.dotdrop import cmd_list_files +from dotdrop.dotdrop import cmd_files from dotdrop.dotdrop import cmd_detail from dotdrop.dotdrop import cmd_importer @@ -87,9 +87,9 @@ class TestListings(unittest.TestCase): # list files o.files_templateonly = False - cmd_list_files(o) + cmd_files(o) o.files_templateonly = True - cmd_list_files(o) + cmd_files(o) # details o.detail_keys = None From 15a1ab16ea8a649ebef963973cb8b96a8b2a497b Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 20 Nov 2020 21:10:54 +0100 Subject: [PATCH 78/82] fix verbosity --- dotdrop/installer.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 1e5428f..91ff45e 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -168,7 +168,8 @@ class Installer: r = False err = 'aborted' else: - self.log.sub('chmod {} to {:o}'.format(dst, chmod)) + if not self.comparing: + self.log.sub('chmod {} to {:o}'.format(dst, chmod)) if utils.chmod(dst, chmod, debug=self.debug): r = True else: @@ -276,7 +277,8 @@ class Installer: if self.dry: self.log.dry('would create directory "{}"'.format(dst)) else: - self.log.sub('creating directory "{}"'.format(dst)) + if not self.comparing: + self.log.sub('creating directory "{}"'.format(dst)) os.makedirs(dst) if os.path.isfile(dst): @@ -386,7 +388,8 @@ class Installer: err = 'something went wrong with {}: {}'.format(src, e) return False, err os.symlink(src, dst) - self.log.sub('linked {} to {}'.format(dst, src)) + if not self.comparing: + self.log.sub('linked {} to {}'.format(dst, src)) return True, None def _copy_file(self, templater, src, dst, @@ -661,7 +664,8 @@ class Installer: return True if self.debug: self.log.dbg('mkdir -p {}'.format(directory)) - self.log.sub('create directory {}'.format(directory)) + if not self.comparing: + self.log.sub('create directory {}'.format(directory)) os.makedirs(directory, exist_ok=True) return os.path.exists(directory) From cbd86fadbaa179ab5a273d06738a4b43f8974095 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 20 Nov 2020 21:22:28 +0100 Subject: [PATCH 79/82] consistency on creating dirs --- dotdrop/installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 91ff45e..5d6c9c3 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -279,7 +279,7 @@ class Installer: else: if not self.comparing: self.log.sub('creating directory "{}"'.format(dst)) - os.makedirs(dst) + self._create_dirs(dst) if os.path.isfile(dst): msg = ''.join([ From 7a270584e5b075f8537cc146ab8cb037f32c406e Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 20 Nov 2020 21:23:14 +0100 Subject: [PATCH 80/82] workers for compare for #277 --- dotdrop/dotdrop.py | 7 ++++++- dotdrop/options.py | 7 ------- dotdrop/utils.py | 9 +++++++++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index af27493..f1ea6e6 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -19,7 +19,8 @@ from dotdrop.updater import Updater from dotdrop.comparator import Comparator from dotdrop.importer import Importer from dotdrop.utils import get_tmpdir, removepath, \ - uniq_list, patch_ignores, dependencies_met + uniq_list, patch_ignores, dependencies_met, \ + adapt_workers from dotdrop.linktypes import LinkTypes from dotdrop.exceptions import YamlException, UndefinedException @@ -278,6 +279,8 @@ def cmd_install(o): dotfiles = o.dotfiles prof = o.conf.get_profile() + adapt_workers(o, LOG) + pro_pre_actions = prof.get_pre_actions() if prof else [] pro_post_actions = prof.get_post_actions() if prof else [] @@ -423,6 +426,8 @@ def cmd_update(o): paths = o.update_path iskey = o.update_iskey + adapt_workers(o, LOG) + if not paths: # update the entire profile if iskey: diff --git a/dotdrop/options.py b/dotdrop/options.py index 1da27db..7fc7680 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -227,13 +227,6 @@ class Options(AttrMonitor): self.log.err('bad option for --workers') sys.exit(USAGE) - if self.safe and self.workers > 1: - self.log.warn('workers set to 1 when --force is not used') - self.workers = 1 - if self.dry and self.workers > 1: - self.log.warn('workers set to 1 when --dry is used') - self.workers = 1 - # import link default value self.import_link = self.link_on_import if self.args['--link']: diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 52fb01c..a7b3945 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -331,3 +331,12 @@ def chmod(path, mode, debug=False): LOG.dbg('chmod {} {}'.format(oct(mode), path)) os.chmod(path, mode) return get_file_perm(path) == mode + + +def adapt_workers(options, logger): + if options.safe and options.workers > 1: + logger.warn('workers set to 1 when --force is not used') + options.workers = 1 + if options.dry and options.workers > 1: + logger.warn('workers set to 1 when --dry is used') + options.workers = 1 From 288581632aac0c009a3e7d0f962d3f4054bf7be7 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 20 Nov 2020 21:25:20 +0100 Subject: [PATCH 81/82] update doc --- docs/usage.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 7acb171..84de6bb 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -239,12 +239,12 @@ commands: It should be set to a maximum of the number of cores available (usually returned on linux by the command `nproc`). -It may speed up the operation but cannot be used interractively (it needs `-f --force` to be set) -and cannot be used with `-d --dry`. Also information printed to stdout/stderr will -probably be messed up. +It may speed up the operation but cannot be used interractively (it needs `-f --force` to be set +except for `compare`) and cannot be used with `-d --dry`. Also information printed to stdout/stderr +will probably be messed up. **WARNING** this feature hasn't been extensively tested and is to be used at your own risk. -If you try it out and find any issue, please [open an issue](https://github.com/deadc0de6/dotdrop/issues). +If you try it out and find any issue, please [report it](https://github.com/deadc0de6/dotdrop/issues). Also if you find it useful and have been able to successfully speed up your operation when using `-w --workers`, do please also report it [in an issue](https://github.com/deadc0de6/dotdrop/issues). From e0ff3efe041baaba5d3b1c3be8edebddeb659cec Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 20 Nov 2020 21:37:42 +0100 Subject: [PATCH 82/82] fix compare for #286 --- dotdrop/comparator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dotdrop/comparator.py b/dotdrop/comparator.py index 6801493..634a489 100644 --- a/dotdrop/comparator.py +++ b/dotdrop/comparator.py @@ -70,8 +70,7 @@ class Comparator: msg = 'mode differ {} ({:o}) and {} ({:o})' self.log.dbg(msg.format(left, left_mode, right, right_mode)) ret = 'modes differ for {} ({:o}) vs {:o}\n' - ret.format(right, right_mode, left_mode) - return ret + return ret.format(right, right_mode, left_mode) def _comp_file(self, left, right, ignore): """compare a file"""