From 6e6c5fb2e32076495db2316bff1108b337116d76 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 31 May 2019 18:30:19 +0200 Subject: [PATCH 01/58] refactor the parsing --- dotdrop/action.py | 43 +- dotdrop/cfg_aggregator.py | 306 ++++++ dotdrop/cfg_yaml.py | 626 +++++++++++ dotdrop/config.py | 1215 --------------------- dotdrop/dictparser.py | 38 + dotdrop/dotdrop.py | 58 +- dotdrop/dotfile.py | 92 +- dotdrop/linktypes.py | 12 + dotdrop/logger.py | 16 +- dotdrop/options.py | 48 +- dotdrop/profile.py | 50 + dotdrop/settings.py | 96 ++ dotdrop/templategen.py | 2 + dotdrop/updater.py | 15 +- dotdrop/utils.py | 2 +- dotdrop/version.py | 2 +- packages/arch-dotdrop/.SRCINFO | 4 +- packages/arch-dotdrop/PKGBUILD | 2 +- scripts/change-link.py | 2 +- tests-ng/compare.sh | 1 + tests-ng/dotdrop-variables.sh | 4 +- tests-ng/dotfile-variables.sh | 2 +- tests-ng/ext-actions.sh | 2 +- tests-ng/import-configs.sh | 130 +++ tests-ng/import-profile-dotfiles.sh | 127 +++ tests-ng/import.sh | 77 +- tests-ng/include.sh | 14 + tests.sh | 18 +- tests/helpers.py | 9 +- tests/test_import.py | 36 +- tests/test_install.py | 8 +- tests/test_update.py | 2 +- tests/{test_config.py => test_yamlcfg.py} | 166 +-- 33 files changed, 1739 insertions(+), 1486 deletions(-) create mode 100644 dotdrop/cfg_aggregator.py create mode 100644 dotdrop/cfg_yaml.py delete mode 100644 dotdrop/config.py create mode 100644 dotdrop/dictparser.py create mode 100644 dotdrop/profile.py create mode 100644 dotdrop/settings.py create mode 100755 tests-ng/import-configs.sh create mode 100755 tests-ng/import-profile-dotfiles.sh rename tests/{test_config.py => test_yamlcfg.py} (80%) diff --git a/dotdrop/action.py b/dotdrop/action.py index 617bf72..daf1545 100644 --- a/dotdrop/action.py +++ b/dotdrop/action.py @@ -10,10 +10,10 @@ import subprocess import os # local imports -from dotdrop.logger import Logger +from dotdrop.dictparser import DictParser -class Cmd: +class Cmd(DictParser): eq_ignore = ('log',) def __init__(self, key, action): @@ -23,7 +23,10 @@ class Cmd: """ self.key = key self.action = action - self.log = Logger() + + @classmethod + def _adjust_yaml_keys(cls, value): + return {'action': value} def __str__(self): return 'key:{} -> \"{}\"'.format(self.key, self.action) @@ -50,20 +53,35 @@ class Cmd: class Action(Cmd): - def __init__(self, key, kind, action, *args): + pre = 'pre' + post = 'post' + + def __init__(self, key, kind, action): """constructor @key: action key @kind: type of action (pre or post) @action: action string - @args: action arguments """ super(Action, self).__init__(key, action) self.kind = kind - self.args = args + self.args = [] + + @classmethod + def parse(cls, key, value): + """parse key value into object""" + v = {} + v['kind'], v['action'] = value + return cls(key=key, **v) + + def copy(self, args): + """return a copy of this object with arguments""" + action = Action(self.key, self.kind, self.action) + action.args = args + return action def __str__(self): - out = '{}: \"{}\" with args: {}' - return out.format(self.key, self.action, self.args) + out = '{}: \"{}\" ({})' + return out.format(self.key, self.action, self.kind) def __repr__(self): return 'action({})'.format(self.__str__()) @@ -74,6 +92,7 @@ class Action(Cmd): action = self.action if templater: action = templater.generate_string(self.action) + cmd = action try: cmd = action.format(*self.args) except IndexError: @@ -94,9 +113,11 @@ class Action(Cmd): class Transform(Cmd): def transform(self, arg0, arg1): - """execute transformation with {0} and {1} - where {0} is the file to transform and - {1} is the result file""" + """ + execute transformation with {0} and {1} + where {0} is the file to transform + and {1} is the result file + """ ret = 1 cmd = self.action.format(arg0, arg1) if os.path.exists(arg1): diff --git a/dotdrop/cfg_aggregator.py b/dotdrop/cfg_aggregator.py new file mode 100644 index 0000000..5e35bf5 --- /dev/null +++ b/dotdrop/cfg_aggregator.py @@ -0,0 +1,306 @@ +""" +author: deadc0de6 (https://github.com/deadc0de6) +Copyright (c) 2019, deadc0de6 + +handle higher level of the config file +""" + +import os +import shlex + + +# local imports +from dotdrop.cfg_yaml import CfgYaml +from dotdrop.dotfile import Dotfile +from dotdrop.settings import Settings +from dotdrop.profile import Profile +from dotdrop.action import Action, Transform +from dotdrop.logger import Logger +from dotdrop.utils import strip_home + + +class CfgAggregator: + + file_prefix = 'f' + dir_prefix = 'd' + key_sep = '_' + + def __init__(self, path, profile=None, debug=False): + """ + high level config parser + @path: path to the config file + @profile: selected profile + @debug: debug flag + """ + self.path = path + self.profile = profile + self.debug = debug + self.log = Logger() + self._load() + + def _load(self): + """load lower level config""" + self.cfgyaml = CfgYaml(self.path, + self.profile, + debug=self.debug) + + # settings + self.settings = Settings.parse(None, self.cfgyaml.settings) + self.settings.resolve_paths(self.cfgyaml.resolve_path) + if self.debug: + self.log.dbg('settings: {}'.format(self.settings)) + + # dotfiles + self.dotfiles = Dotfile.parse_dict(self.cfgyaml.dotfiles) + if self.debug: + self.log.dbg('dotfiles: {}'.format(self.dotfiles)) + + # profiles + self.profiles = Profile.parse_dict(self.cfgyaml.profiles) + if self.debug: + self.log.dbg('profiles: {}'.format(self.profiles)) + + # actions + self.actions = Action.parse_dict(self.cfgyaml.actions) + if self.debug: + self.log.dbg('actions: {}'.format(self.actions)) + + # trans_r + self.trans_r = Transform.parse_dict(self.cfgyaml.trans_r) + if self.debug: + self.log.dbg('trans_r: {}'.format(self.trans_r)) + + # trans_w + self.trans_w = Transform.parse_dict(self.cfgyaml.trans_w) + if self.debug: + self.log.dbg('trans_w: {}'.format(self.trans_w)) + + # variables + self.variables = self.cfgyaml.variables + if self.debug: + self.log.dbg('variables: {}'.format(self.variables)) + + # patch dotfiles in profiles + self._patch_keys_to_objs(self.profiles, + "dotfiles", self.get_dotfile) + + # patch action in actions + self._patch_keys_to_objs(self.dotfiles, + "actions", self._get_action_w_args) + self._patch_keys_to_objs(self.profiles, + "actions", self._get_action_w_args) + + # patch default actions in settings + self._patch_keys_to_objs([self.settings], + "default_actions", self._get_action_w_args) + if self.debug: + msg = 'default actions: {}'.format(self.settings.default_actions) + self.log.dbg(msg) + + # patch trans_w/trans_r in dotfiles + self._patch_keys_to_objs(self.dotfiles, + "trans_r", self.get_trans_r) + self._patch_keys_to_objs(self.dotfiles, + "trans_w", self.get_trans_w) + + def _patch_keys_to_objs(self, containers, keys, get_by_key): + """ + patch each object in "containers" containing + a list of keys in the attribute "keys" with + the returned object of the function "get_by_key" + """ + if not containers: + return + if self.debug: + self.log.dbg('patching {} ...'.format(keys)) + for c in containers: + objects = [] + okeys = getattr(c, keys) + if not okeys: + continue + for k in okeys: + o = get_by_key(k) + if not o: + err = 'bad key for \"{}\": {}'.format(c.key, k) + raise Exception(err) + objects.append(o) + if self.debug: + self.log.dbg('patching {}.{} with {}'.format(c, keys, objects)) + setattr(c, keys, objects) + + def new(self, src, dst, link, profile_key): + """ + import a new dotfile + @src: path in dotpath + @dst: path in FS + @link: LinkType + @profile_key: to which profile + """ + home = os.path.expanduser('~') + dst = dst.replace(home, '~', 1) + + dotfile = self._get_dotfile_by_dst(dst) + if not dotfile: + # get a new dotfile with a unique key + key = self._get_new_dotfile_key(dst) + if self.debug: + self.log.dbg('new dotfile key: {}'.format(key)) + # add the dotfile + self.cfgyaml.add_dotfile(key, src, dst, link) + dotfile = Dotfile(key, dst, src) + + key = dotfile.key + ret = self.cfgyaml.add_dotfile_to_profile(key, profile_key) + if self.debug: + self.log.dbg('new dotfile {} to profile {}'.format(key, + profile_key)) + + # reload + self.cfgyaml.save() + if self.debug: + self.log.dbg('RELOADING') + self._load() + return ret + + def _get_new_dotfile_key(self, dst): + """return a new unique dotfile key""" + path = os.path.expanduser(dst) + existing_keys = [x.key for x in self.dotfiles] + if self.settings.longkey: + return self._get_long_key(path, existing_keys) + return self._get_short_key(path, existing_keys) + + def _norm_key_elem(self, elem): + """normalize path element for sanity""" + elem = elem.lstrip('.') + elem = elem.replace(' ', '-') + return elem.lower() + + def _split_path_for_key(self, path): + """return a list of path elements, excluded home path""" + p = strip_home(path) + dirs = [] + while True: + p, f = os.path.split(p) + dirs.append(f) + if not p or not f: + break + dirs.reverse() + # remove empty entries + dirs = filter(None, dirs) + # normalize entries + return list(map(self._norm_key_elem, dirs)) + + def _get_long_key(self, path, keys): + """ + return a unique long key representing the + absolute path of path + """ + dirs = self._split_path_for_key(path) + prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix + key = self.key_sep.join([prefix, *dirs]) + return self._uniq_key(key, keys) + + def _get_short_key(self, path, keys): + """ + return a unique key where path + is known not to be an already existing dotfile + """ + dirs = self._split_path_for_key(path) + dirs.reverse() + prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix + entries = [] + for d in dirs: + entries.insert(0, d) + key = self.key_sep.join([prefix, *entries]) + if key not in keys: + return key + return self._uniq_key(key, keys) + + def _uniq_key(self, key, keys): + """unique dotfile key""" + newkey = key + cnt = 1 + while newkey in keys: + # if unable to get a unique path + # get a random one + newkey = self.key_sep.join([key, cnt]) + cnt += 1 + return newkey + + def _get_dotfile_by_dst(self, dst): + """get a dotfile by dst""" + try: + return next(d for d in self.dotfiles if d.dst == dst) + except StopIteration: + return None + + def save(self): + """save the config""" + return self.cfgyaml.save() + + def dump(self): + """dump the config dictionary""" + return self.cfgyaml.dump() + + def get_settings(self): + """return settings as a dict""" + return self.settings.serialize()[Settings.key_yaml] + + def get_variables(self): + """return variables""" + return self.variables + + def get_profiles(self): + """return profiles""" + return self.profiles + + def get_dotfiles(self, profile=None): + """return dotfiles dict for this profile key""" + if not profile: + return self.dotfiles + try: + return next(x.dotfiles for x in self.profiles if x.key == profile) + except StopIteration: + return [] + + def get_dotfile(self, key): + """return dotfile by key""" + try: + return next(x for x in self.dotfiles if x.key == key) + except StopIteration: + return None + + def get_action(self, key): + """return action by key""" + try: + return next(x for x in self.actions if x.key == key) + except StopIteration: + return None + + def _get_action_w_args(self, key): + """return action by key with the arguments""" + fields = shlex.split(key) + if len(fields) > 1: + # we have args + key, *args = fields + if self.debug: + self.log.dbg('action with parm: {} and {}'.format(key, args)) + action = self.get_action(key).copy(args) + else: + action = self.get_action(key) + return action + + def get_trans_r(self, key): + """return the trans_r with this key""" + try: + return next(x for x in self.trans_r if x.key == key) + except StopIteration: + return None + + def get_trans_w(self, key): + """return the trans_w with this key""" + try: + return next(x for x in self.trans_w if x.key == key) + except StopIteration: + return None diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py new file mode 100644 index 0000000..29c4ee2 --- /dev/null +++ b/dotdrop/cfg_yaml.py @@ -0,0 +1,626 @@ +""" +author: deadc0de6 (https://github.com/deadc0de6) +Copyright (c) 2019, deadc0de6 + +handle lower level of the config file +""" + +import os +import yaml + +# local imports +from dotdrop.settings import Settings +from dotdrop.logger import Logger +from dotdrop.templategen import Templategen +from dotdrop.linktypes import LinkTypes +from dotdrop.utils import shell + + +class CfgYaml: + + # global entries + key_settings = 'config' + key_dotfiles = 'dotfiles' + key_profiles = 'profiles' + key_actions = 'actions' + key_trans_r = 'trans' + key_trans_w = 'trans_write' + key_variables = 'variables' + key_dvariables = 'dynvariables' + + action_pre = 'pre' + action_post = 'post' + + # profiles/dotfiles entries + key_profiles_dotfiles = 'dotfiles' + key_dotfile_src = 'src' + key_dotfile_dst = 'dst' + key_dotfile_link = 'link' + key_dotfile_actions = 'actions' + key_dotfile_link_children = 'link_children' + + # profile + key_profile_include = 'include' + key_profile_variables = 'variables' + key_profile_dvariables = 'dynvariables' + key_all = 'ALL' + + # import entries + key_import_actions = 'import_actions' + key_import_configs = 'import_configs' + key_import_variables = 'import_variables' + key_import_profile_dfs = 'import' + + # settings + key_settings_dotpath = 'dotpath' + key_settings_workdir = 'workdir' + key_settings_link_dotfile_default = 'link_dotfile_default' + key_imp_link = 'link_on_import' + + # link values + lnk_nolink = LinkTypes.NOLINK.name.lower() + lnk_link = LinkTypes.LINK.name.lower() + lnk_children = LinkTypes.LINK_CHILDREN.name.lower() + + def __init__(self, path, profile=None, debug=False): + """ + config parser + @path: config file path + @profile: the selected profile + @debug: debug flag + """ + self.path = os.path.abspath(path) + self.profile = profile + self.debug = debug + self.log = Logger() + self.dirty = False + + self.yaml_dict = self._load_yaml(self.path) + self._fix_deprecated(self.yaml_dict) + self._parse_main_yaml(self.yaml_dict) + if self.debug: + self.log.dbg('current dict: {}'.format(self.yaml_dict)) + + # resolve variables + allvars = self._merge_and_apply_variables() + self.variables.update(allvars) + # process imported configs + self._resolve_import_configs() + # process other imports + self._resolve_imports() + # process diverse options + self._resolve_rest() + # patch dotfiles paths + self._resolve_dotfile_paths() + + def _parse_main_yaml(self, dic): + """parse the different blocks""" + self.ori_settings = self._get_entry(self.yaml_dict, self.key_settings) + self.settings = Settings(None).serialize().get(self.key_settings) + self.settings.update(self.ori_settings) + if self.debug: + self.log.dbg('settings: {}'.format(self.settings)) + + # dotfiles + self.dotfiles = self._get_entry(self.yaml_dict, self.key_dotfiles) + if self.debug: + self.log.dbg('dotfiles: {}'.format(self.dotfiles)) + + # profiles + self.profiles = self._get_entry(self.yaml_dict, self.key_profiles) + if self.debug: + self.log.dbg('profiles: {}'.format(self.profiles)) + + # actions + self.actions = self._get_entry(self.yaml_dict, self.key_actions, + mandatory=False) + self.actions = self._patch_actions(self.actions) + if self.debug: + self.log.dbg('actions: {}'.format(self.actions)) + + # trans_r + self.trans_r = self._get_entry(self.yaml_dict, self.key_trans_r, + mandatory=False) + if self.debug: + self.log.dbg('trans_r: {}'.format(self.trans_r)) + + # trans_w + self.trans_w = self._get_entry(self.yaml_dict, self.key_trans_w, + mandatory=False) + if self.debug: + self.log.dbg('trans_w: {}'.format(self.trans_w)) + + # variables + self.variables = self._get_entry(self.yaml_dict, self.key_variables, + mandatory=False) + if self.debug: + self.log.dbg('variables: {}'.format(self.variables)) + + # dynvariables + self.dvariables = self._get_entry(self.yaml_dict, self.key_dvariables, + mandatory=False) + if self.debug: + self.log.dbg('dvariables: {}'.format(self.dvariables)) + + def _resolve_dotfile_paths(self): + """resolve dotfile paths""" + for dotfile in self.dotfiles.values(): + src = dotfile[self.key_dotfile_src] + src = os.path.join(self.settings[self.key_settings_dotpath], src) + dotfile[self.key_dotfile_src] = self.resolve_path(src) + dst = dotfile[self.key_dotfile_dst] + dotfile[self.key_dotfile_dst] = self.resolve_path(dst) + + def _merge_and_apply_variables(self): + """ + resolve all variables across the config + apply them to any needed entries + and return the full list of variables + """ + # first construct the list of variables + var = self._get_variables_dict(self.profile, seen=[self.profile]) + dvar = self._get_dvariables_dict(self.profile, seen=[self.profile]) + + # recursive resolve variables + allvars = var.copy() + allvars.update(dvar) + if self.debug: + self.log.dbg('all variables: {}'.format(allvars)) + + t = Templategen(variables=allvars) + for k in allvars.keys(): + val = allvars[k] + while Templategen.var_is_template(val): + val = t.generate_string(val) + allvars[k] = val + t.update_variables(allvars) + + # exec dynvariables + for k in dvar.keys(): + allvars[k] = shell(allvars[k]) + + if self.debug: + self.log.dbg('variables:') + for k, v in allvars.items(): + self.log.dbg('\t\"{}\": {}'.format(k, v)) + + if self.debug: + self.log.dbg('resolve all uses of variables in config') + + # now resolve blocks + t = Templategen(variables=allvars) + + # dotfiles entries + for k, v in self.dotfiles.items(): + # src + src = v.get(self.key_dotfile_src) + v[self.key_dotfile_src] = t.generate_string(src) + # dst + dst = v.get(self.key_dotfile_dst) + v[self.key_dotfile_dst] = t.generate_string(dst) + # actions + new = [] + for a in v.get(self.key_dotfile_actions, []): + new.append(t.generate_string(a)) + if new: + if self.debug: + self.log.dbg('resolved: {}'.format(new)) + v[self.key_dotfile_actions] = new + + # external actions paths + new = [] + for p in self.settings.get(self.key_import_actions, []): + new.append(t.generate_string(p)) + if new: + if self.debug: + self.log.dbg('resolved: {}'.format(new)) + self.settings[self.key_import_actions] = new + + # external config paths + new = [] + for p in self.settings.get(self.key_import_configs, []): + new.append(t.generate_string(p)) + if new: + if self.debug: + self.log.dbg('resolved: {}'.format(new)) + self.settings[self.key_import_configs] = new + + # external variables paths + new = [] + for p in self.settings.get(self.key_import_variables, []): + new.append(t.generate_string(p)) + if new: + if self.debug: + self.log.dbg('resolved: {}'.format(new)) + self.settings[self.key_import_variables] = new + + # external profiles dotfiles + for k, v in self.profiles.items(): + new = [] + for p in v.get(self.key_import_profile_dfs, []): + new.append(t.generate_string(p)) + if new: + if self.debug: + self.log.dbg('resolved: {}'.format(new)) + v[self.key_import_profile_dfs] = new + + return allvars + + def _patch_actions(self, actions): + """ + ensure each action is either pre or post explicitely + action entry of the form {action_key: (pre|post, action)} + """ + if not actions: + return actions + new = {} + for k, v in actions.items(): + if k == self.action_pre or k == self.action_post: + for key, action in v.items(): + new[key] = (k, action) + else: + new[k] = (self.action_pre, v) + return new + + def _get_variables_dict(self, profile, seen, sub=False): + """return enriched variables""" + variables = {} + if not sub: + # add profile variable + if profile: + variables['profile'] = profile + # add some more variables + p = self.settings.get(self.key_settings_dotpath) + p = self.resolve_path(p) + variables['_dotdrop_dotpath'] = p + variables['_dotdrop_cfgpath'] = self.resolve_path(self.path) + p = self.settings.get(self.key_settings_workdir) + p = self.resolve_path(p) + variables['_dotdrop_workdir'] = p + + # variables + variables.update(self.variables) + + if not profile or profile not in self.profiles.keys(): + return variables + + # profile entry + pentry = self.profiles.get(profile) + + # inherite profile variables + for inherited_profile in pentry.get(self.key_profile_include, []): + if inherited_profile == profile or inherited_profile in seen: + raise Exception('\"include\" loop') + seen.append(inherited_profile) + new = self._get_variables_dict(inherited_profile, seen, sub=True) + variables.update(new) + + # overwrite with profile variables + for k, v in pentry.get(self.key_profile_variables, {}).items(): + variables[k] = v + + return variables + + def _get_dvariables_dict(self, profile, seen, sub=False): + """return dynvariables""" + variables = {} + + # dynvariables + variables.update(self.dvariables) + + if not profile or profile not in self.profiles.keys(): + return variables + + # profile entry + pentry = self.profiles.get(profile) + + # inherite profile dynvariables + for inherited_profile in pentry.get(self.key_profile_include, []): + if inherited_profile == profile or inherited_profile in seen: + raise Exception('\"include loop\"') + seen.append(inherited_profile) + new = self._get_dvariables_dict(inherited_profile, seen, sub=True) + variables.update(new) + + # overwrite with profile dynvariables + for k, v in pentry.get(self.key_profile_dvariables, {}).items(): + variables[k] = v + + return variables + + def _resolve_imports(self): + """handle all the imports""" + # settings -> import_variables + imp = self.settings.get(self.key_import_variables, None) + if imp: + for p in imp: + path = self.resolve_path(p) + if self.debug: + self.log.dbg('import variables from {}'.format(path)) + self.variables = self._import_sub(path, self.key_variables, + self.variables, + mandatory=False) + self.dvariables = self._import_sub(path, self.key_dvariables, + self.dvariables, + mandatory=False) + # settings -> import_actions + imp = self.settings.get(self.key_import_actions, None) + if imp: + for p in imp: + path = self.resolve_path(p) + if self.debug: + self.log.dbg('import actions from {}'.format(path)) + self.actions = self._import_sub(path, self.key_actions, + self.actions, mandatory=False, + patch_func=self._patch_actions) + + # profiles -> import + for k, v in self.profiles.items(): + imp = v.get(self.key_import_profile_dfs, None) + if not imp: + continue + if self.debug: + self.log.dbg('import dotfiles for profile {}'.format(k)) + for p in imp: + current = v.get(self.key_dotfiles, []) + path = self.resolve_path(p) + current = self._import_sub(path, self.key_dotfiles, + current, mandatory=False) + v[self.key_dotfiles] = current + + def _resolve_import_configs(self): + """resolve import_configs""" + # settings -> import_configs + imp = self.settings.get(self.key_import_configs, None) + if not imp: + return + for p in imp: + path = self.resolve_path(p) + if self.debug: + self.log.dbg('import config from {}'.format(path)) + sub = CfgYaml(path, debug=self.debug) + # settings is ignored + self.dotfiles = self._merge_dict(self.dotfiles, sub.dotfiles) + self.profiles = self._merge_dict(self.profiles, sub.profiles) + self.actions = self._merge_dict(self.actions, sub.actions) + self.trans_r = self._merge_dict(self.trans_r, sub.trans_r) + self.trans_w = self._merge_dict(self.trans_w, sub.trans_w) + self.variables = self._merge_dict(self.variables, sub.variables) + self.dvariables = self._merge_dict(self.dvariables, sub.dvariables) + + def _resolve_rest(self): + """resolve some other parts of the config""" + # profile -> ALL + for k, v in self.profiles.items(): + dfs = v.get(self.key_profiles_dotfiles, None) + if not dfs: + continue + if self.debug: + self.log.dbg('add ALL to profile {}'.format(k)) + if self.key_all in dfs: + v[self.key_profiles_dotfiles] = self.dotfiles.keys() + + # profiles -> include other profile + for k, v in self.profiles.items(): + self._rec_resolve_profile_include(k) + + def _rec_resolve_profile_include(self, profile): + """recursively resolve include of other profiles's dotfiles""" + values = self.profiles[profile] + current = values.get(self.key_profiles_dotfiles, []) + inc = values.get(self.key_profile_include, None) + if not inc: + return current + seen = [] + for i in inc: + if i in seen: + raise Exception('\"include loop\"') + seen.append(i) + if i not in self.profiles.keys(): + self.log.warn('include unknown profile: {}'.format(i)) + continue + p = self.profiles[i] + others = p.get(self.key_profiles_dotfiles, []) + if self.key_profile_include in p.keys(): + others.extend(self._rec_resolve_profile_include(i)) + current.extend(others) + # unique them + values[self.key_profiles_dotfiles] = list(set(current)) + return values.get(self.key_profiles_dotfiles, []) + + def resolve_path(self, path): + """resolve a path either absolute or relative to config path""" + path = os.path.expanduser(path) + if not os.path.isabs(path): + d = os.path.dirname(self.path) + return os.path.join(d, path) + return os.path.normpath(path) + + def _import_sub(self, path, key, current, + mandatory=False, patch_func=None): + """ + import the block "key" from "path" + and merge it with "current" + patch_func is applied before merge if defined + """ + if self.debug: + self.log.dbg('import \"{}\" from \"{}\"'.format(key, path)) + self.log.dbg('current: {}'.format(current)) + extdict = self._load_yaml(path) + new = self._get_entry(extdict, key, mandatory=mandatory) + if patch_func: + new = patch_func(new) + if not new: + self.log.warn('no \"{}\" imported from \"{}\"'.format(key, path)) + return + if self.debug: + self.log.dbg('found: {}'.format(new)) + if isinstance(current, dict) and isinstance(new, dict): + # imported entries get more priority than current + current = {**current, **new} + elif isinstance(current, list) and isinstance(new, list): + current = [*current, *new] + else: + raise Exception('invalid import {} from {}'.format(key, path)) + if self.debug: + self.log.dbg('new \"{}\": {}'.format(key, current)) + return current + + def _merge_dict(self, high, low): + """merge low into high""" + return {**low, **high} + + def _get_entry(self, yaml_dict, key, mandatory=True): + """return entry from yaml dictionary""" + if key not in yaml_dict: + if mandatory: + raise Exception('invalid config: no {} found'.format(key)) + yaml_dict[key] = {} + return yaml_dict[key] + if mandatory and not yaml_dict[key]: + # ensure is not none + yaml_dict[key] = {} + return yaml_dict[key] + + def _load_yaml(self, path): + """load a yaml file to a dict""" + content = {} + if not os.path.exists(path): + raise Exception('config path not found: {}'.format(path)) + with open(path, 'r') as f: + try: + content = yaml.safe_load(f) + except Exception as e: + self.log.err(e) + raise Exception('invalid config: {}'.format(path)) + return content + + def _new_profile(self, key): + """add a new profile if it doesn't exist""" + if key not in self.profiles.keys(): + # update yaml_dict + self.yaml_dict[self.key_profiles][key] = { + self.key_profiles_dotfiles: [] + } + if self.debug: + self.log.dbg('adding new profile: {}'.format(key)) + self.dirty = True + + def add_dotfile_to_profile(self, dotfile_key, profile_key): + """add an existing dotfile key to a profile_key""" + self._new_profile(profile_key) + profile = self.yaml_dict[self.key_profiles][profile_key] + if dotfile_key not in profile[self.key_profiles_dotfiles]: + profile[self.key_profiles_dotfiles].append(dotfile_key) + if self.debug: + msg = 'add \"{}\" to profile \"{}\"'.format(dotfile_key, + profile_key) + msg.format(dotfile_key, profile_key) + self.log.dbg(msg) + self.dirty = True + return self.dirty + + def add_dotfile(self, key, src, dst, link): + """add a new dotfile""" + if key in self.dotfiles.keys(): + return False + if self.debug: + self.log.dbg('adding new dotfile: {}'.format(key)) + + df_dict = { + self.key_dotfile_src: src, + self.key_dotfile_dst: dst, + } + dfl = self.settings[self.key_settings_link_dotfile_default] + if str(link) != dfl: + df_dict[self.key_dotfile_link] = str(link) + self.yaml_dict[self.key_dotfiles][key] = df_dict + self.dirty = True + + def _fix_deprecated(self, yamldict): + """fix deprecated entries""" + self._fix_deprecated_link_by_default(yamldict) + self._fix_deprecated_dotfile_link(yamldict) + + def _fix_deprecated_link_by_default(self, yamldict): + """fix deprecated link_by_default""" + key = 'link_by_default' + newkey = self.key_imp_link + if self.key_settings not in yamldict: + return + if not yamldict[self.key_settings]: + return + config = yamldict[self.key_settings] + if key not in config: + return + if config[key]: + config[newkey] = self.lnk_link + else: + config[newkey] = self.lnk_nolink + del config[key] + self.log.warn('deprecated \"link_by_default\"') + self.dirty = True + + def _fix_deprecated_dotfile_link(self, yamldict): + """fix deprecated link in dotfiles""" + if self.key_dotfiles not in yamldict: + return + if not yamldict[self.key_dotfiles]: + return + for k, dotfile in yamldict[self.key_dotfiles].items(): + new = self.lnk_nolink + if self.key_dotfile_link in dotfile and \ + type(dotfile[self.key_dotfile_link]) is bool: + # patch link: + cur = dotfile[self.key_dotfile_link] + new = self.lnk_nolink + if cur: + new = self.lnk_link + dotfile[self.key_dotfile_link] = new + self.dirty = True + self.log.warn('deprecated \"link\" value') + + elif self.key_dotfile_link_children in dotfile and \ + type(dotfile[self.key_dotfile_link_children]) is bool: + # patch link_children: + cur = dotfile[self.key_dotfile_link_children] + new = self.lnk_nolink + if cur: + new = self.lnk_children + del dotfile[self.key_dotfile_link_children] + dotfile[self.key_dotfile_link] = new + self.dirty = True + self.log.warn('deprecated \"link_children\" value') + + def _clear_none(self, dic): + """recursively delete all none/empty values in a dictionary.""" + new = {} + for k, v in dic.items(): + newv = v + if isinstance(v, dict): + newv = self._clear_none(v) + if v is None: + continue + if not v: + continue + new[k] = newv + return new + + def save(self): + """save this instance and return True if saved""" + if not self.dirty: + return False + + content = self._clear_none(self.dump()) + if self.debug: + self.log.dbg('saving: {}'.format(content)) + with open(self.path, 'w') as f: + yaml.safe_dump(content, f, + default_flow_style=False, + indent=2) + self.dirty = False + return True + + def dump(self): + """dump the config dictionary""" + return self.yaml_dict diff --git a/dotdrop/config.py b/dotdrop/config.py deleted file mode 100644 index 657de76..0000000 --- a/dotdrop/config.py +++ /dev/null @@ -1,1215 +0,0 @@ -""" -author: deadc0de6 (https://github.com/deadc0de6) -Copyright (c) 2017, deadc0de6 - -yaml config file manager -""" - -import itertools -import os -import shlex -from functools import partial -from glob import iglob - -import yaml - -# local import -from dotdrop.dotfile import Dotfile -from dotdrop.templategen import Templategen -from dotdrop.logger import Logger -from dotdrop.action import Action, Transform -from dotdrop.utils import strip_home, shell -from dotdrop.linktypes import LinkTypes - - -class Cfg: - key_all = 'ALL' - - # settings keys - key_settings = 'config' - key_dotpath = 'dotpath' - key_backup = 'backup' - key_create = 'create' - key_banner = 'banner' - key_long = 'longkey' - key_keepdot = 'keepdot' - key_ignoreempty = 'ignoreempty' - key_showdiff = 'showdiff' - key_imp_link = 'link_on_import' - key_dotfile_link = 'link_dotfile_default' - key_workdir = 'workdir' - key_cmpignore = 'cmpignore' - key_upignore = 'upignore' - key_defactions = 'default_actions' - - # import keys - key_import_vars = 'import_variables' - key_import_actions = 'import_actions' - - key_import_configs = 'import_configs' - - # actions keys - key_actions = 'actions' - key_actions_pre = 'pre' - key_actions_post = 'post' - - # transformations keys - key_trans_r = 'trans' - key_trans_w = 'trans_write' - - # template variables - key_variables = 'variables' - # shell variables - key_dynvariables = 'dynvariables' - - # dotfiles keys - key_dotfiles = 'dotfiles' - key_dotfiles_src = 'src' - key_dotfiles_dst = 'dst' - key_dotfiles_link = 'link' - key_dotfiles_link_children = 'link_children' - key_dotfiles_noempty = 'ignoreempty' - key_dotfiles_cmpignore = 'cmpignore' - key_dotfiles_actions = 'actions' - key_dotfiles_trans_r = 'trans' - key_dotfiles_trans_w = 'trans_write' - key_dotfiles_upignore = 'upignore' - - # profiles keys - key_profiles = 'profiles' - key_profiles_dots = 'dotfiles' - key_profiles_incl = 'include' - key_profiles_imp = 'import' - - # link values - lnk_nolink = LinkTypes.NOLINK.name.lower() - lnk_link = LinkTypes.LINK.name.lower() - lnk_children = LinkTypes.LINK_CHILDREN.name.lower() - - # settings defaults - default_dotpath = 'dotfiles' - default_backup = True - default_create = True - default_banner = True - default_longkey = False - default_keepdot = False - default_showdiff = False - default_ignoreempty = False - default_link_imp = lnk_nolink - default_link = lnk_nolink - default_workdir = '~/.config/dotdrop' - - def __init__(self, cfgpath, profile=None, debug=False): - """constructor - @cfgpath: path to the config file - @profile: chosen profile - @debug: enable debug - """ - if not cfgpath: - raise ValueError('config file path undefined') - if not os.path.exists(cfgpath): - raise ValueError('config file does not exist: {}'.format(cfgpath)) - - # make sure to have an absolute path to config file - self.cfgpath = os.path.abspath(cfgpath) - self.debug = debug - self._modified = False - - # init the logger - self.log = Logger() - - # represents all entries under "config" - # linked inside the yaml dict (self.content) - self.lnk_settings = {} - - # represents all entries under "profiles" - # linked inside the yaml dict (self.content) - self.lnk_profiles = {} - - # represents all dotfiles - # NOT linked inside the yaml dict (self.content) - self.dotfiles = {} - - # dict of all action objects by action key - # NOT linked inside the yaml dict (self.content) - self.actions = {} - - # dict of all read transformation objects by trans key - # NOT linked inside the yaml dict (self.content) - self.trans_r = {} - - # dict of all write transformation objects by trans key - # NOT linked inside the yaml dict (self.content) - self.trans_w = {} - - # represents all dotfiles per profile by profile key - # NOT linked inside the yaml dict (self.content) - self.prodots = {} - - # represents all variables from external files - # NOT linked inside the yaml dict (self.content) - self.ext_variables = {} - self.ext_dynvariables = {} - - # cmpignore patterns - # NOT linked inside the yaml dict (self.content) - self.cmpignores = [] - - # upignore patterns - # NOT linked inside the yaml dict (self.content) - self.upignores = [] - - # default actions - # NOT linked inside the yaml dict (self.content) - self.defactions = {} - - if not self._load_config(profile=profile): - raise ValueError('config is not valid') - - def __eq__(self, other): - return self.cfgpath == other.cfgpath - - def eval_dotfiles(self, profile, variables, debug=False): - """resolve dotfiles src/dst/actions templating for this profile""" - t = Templategen(variables=variables) - dotfiles = self._get_dotfiles(profile) - tvars = t.add_tmp_vars() - for d in dotfiles: - # add dotfile variables - t.restore_vars(tvars) - newvar = d.get_vars() - t.add_tmp_vars(newvars=newvar) - # src and dst path - d.src = t.generate_string(d.src) - d.dst = t.generate_string(d.dst) - # pre actions - if self.key_actions_pre in d.actions: - for action in d.actions[self.key_actions_pre]: - action.action = t.generate_string(action.action) - # post actions - if self.key_actions_post in d.actions: - for action in d.actions[self.key_actions_post]: - action.action = t.generate_string(action.action) - return dotfiles - - def _load_config(self, profile=None): - """load the yaml file""" - self.content = self._load_yaml(self.cfgpath) - if not self._is_valid(): - return False - return self._parse(profile=profile) - - def _load_yaml(self, path): - """load a yaml file to a dict""" - content = {} - if not os.path.exists(path): - return content - with open(path, 'r') as f: - try: - content = yaml.safe_load(f) - except Exception as e: - self.log.err(e) - return {} - return content - - def _is_valid(self): - """test the yaml dict (self.content) is valid""" - if self.key_profiles not in self.content: - self.log.err('missing \"{}\" in config'.format(self.key_profiles)) - return False - if self.key_settings not in self.content: - self.log.err('missing \"{}\" in config'.format(self.key_settings)) - return False - if self.key_dotfiles not in self.content: - self.log.err('missing \"{}\" in config'.format(self.key_dotfiles)) - return False - return True - - def _get_def_link(self): - """get dotfile link entry when not specified""" - string = self.lnk_settings[self.key_dotfile_link].lower() - return self._string_to_linktype(string) - - def _string_to_linktype(self, string): - """translate string to linktype""" - if string == self.lnk_link.lower(): - return LinkTypes.LINK - elif string == self.lnk_children.lower(): - return LinkTypes.LINK_CHILDREN - return LinkTypes.NOLINK - - def _parse(self, profile=None): - """parse config file""" - # parse the settings - self.lnk_settings = self.content[self.key_settings] or {} - if not self._complete_settings(): - return False - - # parse the profiles - # ensures self.lnk_profiles is a dict - if not isinstance(self.content[self.key_profiles], dict): - self.content[self.key_profiles] = {} - - self.lnk_profiles = self.content[self.key_profiles] - for p in filter(bool, self.lnk_profiles.values()): - # Ensures that the dotfiles entry is an empty list when not given - # or none - p.setdefault(self.key_profiles_dots, []) - if p[self.key_profiles_dots] is None: - p[self.key_profiles_dots] = [] - - # make sure we have an absolute dotpath - self.curdotpath = self.lnk_settings[self.key_dotpath] - self.lnk_settings[self.key_dotpath] = self._abs_path(self.curdotpath) - - # make sure we have an absolute workdir - self.curworkdir = self.lnk_settings[self.key_workdir] - self.lnk_settings[self.key_workdir] = self._abs_path(self.curworkdir) - - # load external variables/dynvariables - try: - paths = self.lnk_settings[self.key_import_vars] or [] - self._load_ext_variables(paths, profile=profile) - except KeyError: - pass - - # load global upignore - if self.key_upignore in self.lnk_settings: - key = self.key_upignore - self.upignores = self.lnk_settings[key].copy() or [] - - # load global cmpignore - if self.key_cmpignore in self.lnk_settings: - key = self.key_cmpignore - self.cmpignores = self.lnk_settings[key].copy() or [] - - # parse external actions - try: - ext_actions = self.lnk_settings[self.key_import_actions] or () - for path in ext_actions: - path = self._abs_path(path) - if self.debug: - self.log.dbg('loading actions from {}'.format(path)) - content = self._load_yaml(path) - # If external actions are None, replaces them with empty dict - try: - external_actions = content[self.key_actions] or {} - self._load_actions(external_actions) - except KeyError: - pass - except KeyError: - pass - - # parse external configs - try: - ext_configs = self.lnk_settings[self.key_import_configs] or () - - try: - iglob('./*', recursive=True) - find_glob = partial(iglob, recursive=True) - except TypeError: - from platform import python_version - - msg = ('Recursive globbing is not available on Python {}: ' - .format(python_version())) - if any('**' in config for config in ext_configs): - msg += "import_configs won't work" - self.log.err(msg) - return False - - msg = 'upgrade to version >3.5 if you want to use this feature' - self.log.warn(msg) - find_glob = iglob - - ext_configs = itertools.chain.from_iterable( - find_glob(self._abs_path(config)) - for config in ext_configs - ) - for config in ext_configs: - self._merge_cfg(config) - except KeyError: - pass - - # parse local actions - # If local actions are None, replaces them with empty dict - try: - local_actions = self.content[self.key_actions] or {} - self._load_actions(local_actions) - except KeyError: - pass - - # load default actions - try: - dactions = self.lnk_settings[self.key_defactions].copy() or [] - self.defactions = self._parse_actions_list(dactions, - profile=profile) - except KeyError: - self.defactions = { - self.key_actions_pre: [], - self.key_actions_post: [], - } - - # parse read transformations - # If read transformations are None, replaces them with empty dict - try: - read_trans = self.content[self.key_trans_r] or {} - self.trans_r.update({ - k: Transform(k, v) - for k, v - in read_trans.items() - }) - except KeyError: - pass - - # parse write transformations - # If write transformations are None, replaces them with empty dict - try: - read_trans = self.content[self.key_trans_w] or {} - self.trans_w.update({ - k: Transform(k, v) - for k, v - in read_trans.items() - }) - except KeyError: - pass - - # parse the dotfiles and construct the dict of objects per dotfile key - # ensures the dotfiles entry is a dict - if not isinstance(self.content[self.key_dotfiles], dict): - self.content[self.key_dotfiles] = {} - - dotfiles = self.content[self.key_dotfiles] - noempty_default = self.lnk_settings[self.key_ignoreempty] - dotpath = self.lnk_settings[self.key_dotpath] - for k, v in dotfiles.items(): - src = v[self.key_dotfiles_src] - if dotpath not in src: - src = os.path.join(dotpath, src) - src = os.path.normpath(self._abs_path(src)) - dst = os.path.normpath(v[self.key_dotfiles_dst]) - - # Fail if both `link` and `link_children` present - if self.key_dotfiles_link in v \ - and self.key_dotfiles_link_children in v: - msg = 'only one of `link` or `link_children` allowed per' - msg += ' dotfile, error on dotfile "{}".' - self.log.err(msg.format(k)) - return False - - # fix it - v = self._fix_dotfile_link(k, v) - dotfiles[k] = v - - # get link type - link = self._get_def_link() - if self.key_dotfiles_link in v: - link = self._string_to_linktype(v[self.key_dotfiles_link]) - - # get ignore empty - noempty = v.get(self.key_dotfiles_noempty, noempty_default) - - # parse actions - itsactions = v.get(self.key_dotfiles_actions, []) - actions = self._parse_actions_list(itsactions, profile=profile) - if self.debug: - self.log.dbg('action for {}'.format(k)) - for t in [self.key_actions_pre, self.key_actions_post]: - for action in actions[t]: - self.log.dbg('- {}: {}'.format(t, action)) - - # parse read transformation - itstrans_r = v.get(self.key_dotfiles_trans_r) - trans_r = None - if itstrans_r: - if type(itstrans_r) is list: - msg = 'One transformation allowed per dotfile' - msg += ', error on dotfile \"{}\"' - self.log.err(msg.format(k)) - msg = 'Please modify your config file to: \"trans: {}\"' - self.log.err(msg.format(itstrans_r[0])) - return False - trans_r = self._parse_trans(itstrans_r, read=True) - if not trans_r: - msg = 'unknown trans \"{}\" for \"{}\"' - self.log.err(msg.format(itstrans_r, k)) - return False - - # parse write transformation - itstrans_w = v.get(self.key_dotfiles_trans_w) - trans_w = None - if itstrans_w: - if type(itstrans_w) is list: - msg = 'One write transformation allowed per dotfile' - msg += ', error on dotfile \"{}\"' - self.log.err(msg.format(k)) - msg = 'Please modify your config file: \"trans_write: {}\"' - self.log.err(msg.format(itstrans_w[0])) - return False - trans_w = self._parse_trans(itstrans_w, read=False) - if not trans_w: - msg = 'unknown trans_write \"{}\" for \"{}\"' - self.log.err(msg.format(itstrans_w, k)) - return False - - # disable transformation when link is true - if link != LinkTypes.NOLINK and (trans_r or trans_w): - msg = 'transformations disabled for \"{}\"'.format(dst) - msg += ' because link|link_children is enabled' - self.log.warn(msg) - trans_r = None - trans_w = None - - # parse cmpignore pattern - cmpignores = v.get(self.key_dotfiles_cmpignore, []).copy() - cmpignores.extend(self.cmpignores) - - # parse upignore pattern - upignores = v.get(self.key_dotfiles_upignore, []).copy() - upignores.extend(self.upignores) - - # create new dotfile - self.dotfiles[k] = Dotfile(k, dst, src, - link=link, actions=actions, - trans_r=trans_r, trans_w=trans_w, - cmpignore=cmpignores, noempty=noempty, - upignore=upignores) - - # assign dotfiles to each profile - self.prodots = {k: [] for k in self.lnk_profiles.keys()} - for name, profile in self.lnk_profiles.items(): - if not profile: - continue - dots = profile[self.key_profiles_dots] - if not dots: - continue - - if self.key_all in dots: - # add all if key ALL is used - self.prodots[name] = list(self.dotfiles.values()) - else: - # add the dotfiles - for d in dots: - if d not in self.dotfiles: - msg = 'unknown dotfile \"{}\" for {}'.format(d, k) - self.log.err(msg) - continue - self.prodots[name].append(self.dotfiles[d]) - - profile_names = self.lnk_profiles.keys() - # handle "import" (from file) for each profile - for k in profile_names: - dots = self._get_imported_dotfiles_keys(k) - for d in dots: - if d not in self.dotfiles: - msg = '(i) unknown dotfile \"{}\" for {}'.format(d, k) - self.log.err(msg) - continue - self.prodots[k].append(self.dotfiles[d]) - - # handle "include" (from other profile) for each profile - for k in profile_names: - ret, dots = self._get_included_dotfiles(k) - if not ret: - return False - self.prodots[k].extend(dots) - - # remove duplicates if any - self.prodots = {k: list(set(v)) for k, v in self.prodots.items()} - - # print dotfiles for each profile - if self.debug: - for k in self.lnk_profiles.keys(): - df = ','.join(d.key for d in self.prodots[k]) - self.log.dbg('dotfiles for \"{}\": {}'.format(k, df)) - return True - - def _merge_dict(self, ext_config, warning_prefix, self_member, - ext_member=None, traceback=False): - """Merge into self a dictionary instance members from an external Cfg. - - This method merges instance members of another Cfg instance into self. - It issues a warning for any key shared between self and the other Cfg. - It can adds an own=False porperty to any dictionary in the external - instance member before merging. - - :param ext_config: The other Cfg to merge from. - :type ext_config: Cfg - :param warnign_prefix: The prefix to th warning messages. - :type warning_prefix: str - :param self_member: The member of self which will be augmented by the - external member. Or the self_member name as a string. - :type self_member: dict or str - :param ext_member: The member of ext_config which wil be merged in - self_member. When not given, self_member is assumed to be a string, - and self_member and ext_member are supposed to have the same name. - :type ext_member: dict or None - :param traceback: Whether to add own=False to ext_member dict values - before merging in. - :type traceback: bool - - """ - if ext_member is None: - member_name = self_member - self_member = getattr(self, member_name) - ext_member = getattr(ext_config, member_name) - - common_keys = ( - key - for key in (set(self_member.keys()) - .intersection(set(ext_member.keys()))) - if not key.startswith('_') # filtering out internal variables - ) - warning_msg = ('%s {} defined both in %s and %s: {} in %s used' - % (warning_prefix, self.cfgpath, ext_config.cfgpath, - self.cfgpath)) - for key in common_keys: - self.log.warn(warning_msg.format(key, key)) - - if traceback: - # Assumes v to be a dict. So far it's only used for profiles, - # that are in fact dicts - merged = { - k: dict(v, own=False) - for k, v in ext_member.items() - } - else: - merged = ext_member.copy() - merged.update(self_member) - self_member.update(merged) - - return self_member - - def _merge_cfg(self, config_path): - """Merge an external config.yaml file into self.""" - # Parsing external config file - try: - ext_config = Cfg(config_path) - except ValueError: - raise ValueError( - 'external config file not found: {}'.format(config_path)) - - # Merging in members from the external config file - self._merge_dict(ext_config=ext_config, warning_prefix='Dotfile', - self_member='dotfiles') - self._merge_dict(ext_config=ext_config, warning_prefix='Profile', - self_member='lnk_profiles', traceback=True) - self._merge_dict(ext_config=ext_config, warning_prefix='Action', - self_member='actions') - self._merge_dict(ext_config=ext_config, - warning_prefix='Transformation', - self_member='trans_r') - self._merge_dict(ext_config=ext_config, - warning_prefix='Write transformation', - self_member='trans_w') - self._merge_dict(ext_config=ext_config, warning_prefix='Profile', - self_member='prodots') - - # variables are merged in ext_*variables so as not to be added in - # self.content. This needs an additional step to account for imported - # variables sharing a key with the ones defined in self.content. - variables = { - k: v - for k, v in ext_config._get_variables(None).items() - if k not in self._get_variables(None).keys() - } - dyn_variables = { - k: v - for k, v in ext_config._get_dynvariables(None).items() - if k not in self._get_dynvariables(None).keys() - } - self._merge_dict(ext_config=ext_config, warning_prefix='Variable', - self_member=self.ext_variables, - ext_member=variables) - self._merge_dict(ext_config=ext_config, - warning_prefix='Dynamic variable', - self_member=self.ext_dynvariables, - ext_member=dyn_variables) - - def _load_ext_variables(self, paths, profile=None): - """load external variables""" - variables = {} - dvariables = {} - cur_vars = self.get_variables(profile, debug=self.debug) - t = Templategen(variables=cur_vars) - for path in paths: - path = self._abs_path(path) - path = t.generate_string(path) - if self.debug: - self.log.dbg('loading variables from {}'.format(path)) - content = self._load_yaml(path) - if not content: - self.log.warn('\"{}\" does not exist'.format(path)) - continue - # variables - if self.key_variables in content: - variables.update(content[self.key_variables]) - # dynamic variables - if self.key_dynvariables in content: - dvariables.update(content[self.key_dynvariables]) - self.ext_variables = variables - if self.debug: - self.log.dbg('loaded ext variables: {}'.format(variables)) - self.ext_dynvariables = dvariables - if self.debug: - self.log.dbg('loaded ext dynvariables: {}'.format(dvariables)) - - def _load_actions(self, dic): - for k, v in dic.items(): - # loop through all actions - if k in [self.key_actions_pre, self.key_actions_post]: - # parse pre/post actions - items = dic[k].items() - for k2, v2 in items: - if k not in self.actions: - self.actions[k] = {} - a = Action(k2, k, v2) - self.actions[k][k2] = a - if self.debug: - self.log.dbg('new action: {}'.format(a)) - else: - # parse naked actions as post actions - if self.key_actions_post not in self.actions: - self.actions[self.key_actions_post] = {} - a = Action(k, '', v) - self.actions[self.key_actions_post][k] = a - if self.debug: - self.log.dbg('new action: {}'.format(a)) - - def _abs_path(self, path): - """return absolute path of path relative to the confpath""" - path = os.path.expanduser(path) - if not os.path.isabs(path): - d = os.path.dirname(self.cfgpath) - return os.path.join(d, path) - return path - - def _get_imported_dotfiles_keys(self, profile): - """import dotfiles from external file""" - keys = [] - if self.key_profiles_imp not in self.lnk_profiles[profile]: - return keys - variables = self.get_variables(profile, debug=self.debug) - t = Templategen(variables=variables) - paths = self.lnk_profiles[profile][self.key_profiles_imp] - for path in paths: - path = self._abs_path(path) - path = t.generate_string(path) - if self.debug: - self.log.dbg('loading dotfiles from {}'.format(path)) - content = self._load_yaml(path) - if not content: - self.log.warn('\"{}\" does not exist'.format(path)) - continue - if self.key_profiles_dots not in content: - self.log.warn('not dotfiles in \"{}\"'.format(path)) - continue - df = content[self.key_profiles_dots] - if self.debug: - self.log.dbg('imported dotfiles keys: {}'.format(df)) - keys.extend(df) - return keys - - def _get_included_dotfiles(self, profile, seen=[]): - """find all dotfiles for a specific profile - when using the include keyword""" - if profile in seen: - self.log.err('cyclic include in profile \"{}\"'.format(profile)) - return False, [] - if not self.lnk_profiles[profile]: - return True, [] - dotfiles = self.prodots[profile] - if self.key_profiles_incl not in self.lnk_profiles[profile]: - # no include found - return True, dotfiles - if not self.lnk_profiles[profile][self.key_profiles_incl]: - # empty include found - return True, dotfiles - variables = self.get_variables(profile, debug=self.debug) - t = Templategen(variables=variables) - if self.debug: - self.log.dbg('handle includes for profile \"{}\"'.format(profile)) - for other in self.lnk_profiles[profile][self.key_profiles_incl]: - # resolve include value - other = t.generate_string(other) - if other not in self.prodots: - # no such profile - self.log.warn('unknown included profile \"{}\"'.format(other)) - continue - if self.debug: - msg = 'include dotfiles from \"{}\" into \"{}\"' - self.log.dbg(msg.format(other, profile)) - lseen = seen.copy() - lseen.append(profile) - ret, recincludes = self._get_included_dotfiles(other, seen=lseen) - if not ret: - return False, [] - dotfiles.extend(recincludes) - dotfiles.extend(self.prodots[other]) - return True, dotfiles - - def _parse_actions_list(self, entries, profile=None): - """parse actions specified for an element - where entries are the ones defined for this dotfile""" - res = { - self.key_actions_pre: [], - self.key_actions_post: [], - } - vars = self.get_variables(profile, debug=self.debug) - t = Templategen(variables=vars) - for line in entries: - fields = shlex.split(line) - entry = fields[0] - args = [] - if len(fields) > 1: - tmpargs = fields[1:] - args = [] - # template args - for arg in tmpargs: - args.append(t.generate_string(arg)) - action = None - if self.key_actions_pre in self.actions and \ - entry in self.actions[self.key_actions_pre]: - kind = self.key_actions_pre - if not args: - action = self.actions[self.key_actions_pre][entry] - else: - a = self.actions[self.key_actions_pre][entry].action - action = Action(entry, kind, a, *args) - elif self.key_actions_post in self.actions and \ - entry in self.actions[self.key_actions_post]: - kind = self.key_actions_post - if not args: - action = self.actions[self.key_actions_post][entry] - else: - a = self.actions[self.key_actions_post][entry].action - action = Action(entry, kind, a, *args) - else: - self.log.warn('unknown action \"{}\"'.format(entry)) - continue - res[kind].append(action) - return res - - def _parse_trans(self, trans, read=True): - """parse transformation key specified for a dotfile""" - transformations = self.trans_r - if not read: - transformations = self.trans_w - if trans not in transformations.keys(): - return None - return transformations[trans] - - def _complete_settings(self): - """set settings defaults if not present""" - self._fix_deprecated() - if self.key_dotpath not in self.lnk_settings: - self.lnk_settings[self.key_dotpath] = self.default_dotpath - if self.key_backup not in self.lnk_settings: - self.lnk_settings[self.key_backup] = self.default_backup - if self.key_create not in self.lnk_settings: - self.lnk_settings[self.key_create] = self.default_create - if self.key_banner not in self.lnk_settings: - self.lnk_settings[self.key_banner] = self.default_banner - if self.key_long not in self.lnk_settings: - self.lnk_settings[self.key_long] = self.default_longkey - if self.key_keepdot not in self.lnk_settings: - self.lnk_settings[self.key_keepdot] = self.default_keepdot - if self.key_workdir not in self.lnk_settings: - self.lnk_settings[self.key_workdir] = self.default_workdir - if self.key_showdiff not in self.lnk_settings: - self.lnk_settings[self.key_showdiff] = self.default_showdiff - if self.key_ignoreempty not in self.lnk_settings: - self.lnk_settings[self.key_ignoreempty] = self.default_ignoreempty - - if self.key_dotfile_link not in self.lnk_settings: - self.lnk_settings[self.key_dotfile_link] = self.default_link - else: - key = self.lnk_settings[self.key_dotfile_link] - if key != self.lnk_link and \ - key != self.lnk_children and \ - key != self.lnk_nolink: - self.log.err('bad value for {}'.format(self.key_dotfile_link)) - return False - - if self.key_imp_link not in self.lnk_settings: - self.lnk_settings[self.key_imp_link] = self.default_link_imp - else: - key = self.lnk_settings[self.key_imp_link] - if key != self.lnk_link and \ - key != self.lnk_children and \ - key != self.lnk_nolink: - self.log.err('bad value for {}'.format(self.key_dotfile_link)) - return False - return True - - def _fix_deprecated(self): - """fix deprecated entries""" - # link_by_default - key = 'link_by_default' - newkey = self.key_imp_link - if key in self.lnk_settings: - if self.lnk_settings[key]: - self.lnk_settings[newkey] = self.lnk_link - else: - self.lnk_settings[newkey] = self.lnk_nolink - del self.lnk_settings[key] - self._modified = True - - def _fix_dotfile_link(self, key, entry): - """fix deprecated link usage in dotfile entry""" - v = entry - - if self.key_dotfiles_link not in v \ - and self.key_dotfiles_link_children not in v: - # nothing defined - return v - - new = self.lnk_nolink - if self.key_dotfiles_link in v \ - and type(v[self.key_dotfiles_link]) is bool: - # patch link: - if v[self.key_dotfiles_link]: - new = self.lnk_link - else: - new = self.lnk_nolink - self._modified = True - if self.debug: - self.log.dbg('link updated for {} to {}'.format(key, new)) - elif self.key_dotfiles_link_children in v \ - and type(v[self.key_dotfiles_link_children]) is bool: - # patch link_children: - if v[self.key_dotfiles_link_children]: - new = self.lnk_children - else: - new = self.lnk_nolink - del v[self.key_dotfiles_link_children] - self._modified = True - if self.debug: - self.log.dbg('link updated for {} to {}'.format(key, new)) - else: - # no change - new = v[self.key_dotfiles_link] - - v[self.key_dotfiles_link] = new - return v - - @classmethod - def _filter_not_own(cls, content): - """Filters out from a dict its dict values with own=False.""" - # This way it recursively explores only dicts. Since own=False is used - # only in profiles, which are in fact dicts, this is fine for now. - return { - k: cls._filter_not_own(v) if isinstance(v, dict) else v - for k, v in content.items() - if not isinstance(v, dict) or v.get('own', True) - } - - def _save(self, content, path): - """writes the config to file""" - ret = False - with open(path, 'w') as f: - ret = yaml.safe_dump(self._filter_not_own(content), f, - default_flow_style=False, - indent=2) - if ret: - self._modified = False - return ret - - def _norm_key_elem(self, elem): - """normalize path element for sanity""" - elem = elem.lstrip('.') - elem = elem.replace(' ', '-') - return elem.lower() - - def _get_paths(self, path): - """return a list of path elements, excluded home path""" - p = strip_home(path) - dirs = [] - while True: - p, f = os.path.split(p) - dirs.append(f) - if not p or not f: - break - dirs.reverse() - # remove empty entries - dirs = filter(None, dirs) - # normalize entries - dirs = list(map(self._norm_key_elem, dirs)) - return dirs - - def _get_long_key(self, path, keys): - """return a unique long key representing the - absolute path of path""" - dirs = self._get_paths(path) - # prepend with indicator - if os.path.isdir(path): - key = 'd_{}'.format('_'.join(dirs)) - else: - key = 'f_{}'.format('_'.join(dirs)) - return self._get_unique_key(key, keys) - - def _get_short_key(self, path, keys): - """return a unique key where path - is known not to be an already existing dotfile""" - dirs = self._get_paths(path) - dirs.reverse() - pre = 'f' - if os.path.isdir(path): - pre = 'd' - entries = [] - for d in dirs: - entries.insert(0, d) - key = '_'.join(entries) - key = '{}_{}'.format(pre, key) - if key not in keys: - return key - return self._get_unique_key(key, keys) - - def _get_unique_key(self, key, keys): - """return a unique dotfile key""" - newkey = key - cnt = 1 - while newkey in keys: - # if unable to get a unique path - # get a random one - newkey = '{}_{}'.format(key, cnt) - cnt += 1 - return newkey - - def _dotfile_exists(self, dotfile): - """return True and the existing dotfile key - if it already exists, False and a new unique key otherwise""" - try: - return True, next(key - for key, d in self.dotfiles.items() - if d.dst == dotfile.dst) - except StopIteration: - pass - # return key for this new dotfile - path = os.path.expanduser(dotfile.dst) - keys = self.dotfiles.keys() - if self.lnk_settings[self.key_long]: - return False, self._get_long_key(path, keys) - return False, self._get_short_key(path, keys) - - def new(self, src, dst, profile, link, debug=False): - """import new dotfile""" - # keep it short - home = os.path.expanduser('~') - dst = dst.replace(home, '~', 1) - dotfile = Dotfile('', dst, src) - - # adding new profile if doesn't exist - if profile not in self.lnk_profiles: - if debug: - self.log.dbg('adding profile to config') - # in the yaml - self.lnk_profiles[profile] = {self.key_profiles_dots: []} - # in the global list of dotfiles per profile - self.prodots[profile] = [] - - exists, key = self._dotfile_exists(dotfile) - if exists: - if debug: - self.log.dbg('key already exists: {}'.format(key)) - # retrieve existing dotfile - dotfile = self.dotfiles[key] - if dotfile in self.prodots[profile]: - self.log.err('\"{}\" already present'.format(dotfile.key)) - return False, dotfile - - # add for this profile - self.prodots[profile].append(dotfile) - - # get a pointer in the yaml profiles->this_profile - # and complete it with the new entry - pro = self.content[self.key_profiles][profile] - if self.key_all not in pro[self.key_profiles_dots]: - pro[self.key_profiles_dots].append(dotfile.key) - return True, dotfile - - if debug: - self.log.dbg('dotfile attributed key: {}'.format(key)) - # adding the new dotfile - dotfile.key = key - dotfile.link = link - if debug: - self.log.dbg('adding new dotfile: {}'.format(dotfile)) - # add the entry in the yaml file - dots = self.content[self.key_dotfiles] - dots[dotfile.key] = { - self.key_dotfiles_dst: dotfile.dst, - self.key_dotfiles_src: dotfile.src, - } - - # set the link flag - if link != self._get_def_link(): - val = link.name.lower() - dots[dotfile.key][self.key_dotfiles_link] = val - - # link it to this profile in the yaml file - pro = self.content[self.key_profiles][profile] - if self.key_all not in pro[self.key_profiles_dots]: - pro[self.key_profiles_dots].append(dotfile.key) - - # add it to the global list of dotfiles - self.dotfiles[dotfile.key] = dotfile - # add it to this profile - self.prodots[profile].append(dotfile) - - return True, dotfile - - def _get_dotfiles(self, profile): - """return a list of dotfiles for a specific profile""" - if profile not in self.prodots: - return [] - return sorted(self.prodots[profile], - key=lambda x: str(x.key)) - - def get_profiles(self): - """return all defined profiles""" - return self.lnk_profiles.keys() - - def get_settings(self): - """return all defined settings""" - settings = self.lnk_settings.copy() - # patch link entries - key = self.key_imp_link - settings[key] = self._string_to_linktype(settings[key]) - key = self.key_dotfile_link - settings[key] = self._string_to_linktype(settings[key]) - # patch defactions - key = self.key_defactions - settings[key] = self.defactions - return settings - - def get_variables(self, profile, debug=False): - """return the variables for this profile""" - # get flat variables - variables = self._get_variables(profile=profile) - - # get interpreted variables - dvariables = self._get_dynvariables(profile) - - # recursive resolve variables - allvars = variables.copy() - allvars.update(dvariables) - var = self._rec_resolve_vars(allvars) - - # execute dynvariables - for k in dvariables.keys(): - var[k] = shell(var[k]) - - if debug: - self.log.dbg('variables:') - for k, v in var.items(): - self.log.dbg('\t\"{}\": {}'.format(k, v)) - - return var - - def _rec_resolve_vars(self, variables): - """recursive resolve all variables""" - t = Templategen(variables=variables) - - for k in variables.keys(): - val = variables[k] - while Templategen.var_is_template(val): - val = t.generate_string(val) - variables[k] = val - t.update_variables(variables) - return variables - - def _get_variables(self, profile=None, sub=False): - """return the un-interpreted variables""" - variables = {} - - if not sub: - # profile variable - if profile: - variables['profile'] = profile - - # add paths variables - variables['_dotdrop_dotpath'] = self.lnk_settings[self.key_dotpath] - variables['_dotdrop_cfgpath'] = self.cfgpath - variables['_dotdrop_workdir'] = self.lnk_settings[self.key_workdir] - - # global variables - if self.key_variables in self.content: - variables.update(self.content[self.key_variables]) - - # external variables - variables.update(self.ext_variables) - - if not profile or profile not in self.lnk_profiles: - return variables - - var = self.lnk_profiles[profile] - - # inherited profile variables - if self.key_profiles_incl in var.keys(): - for inherited_profile in var[self.key_profiles_incl]: - inherited_vars = self._get_variables(inherited_profile, True) - variables.update(inherited_vars) - - # finally we override with profile variables - if self.key_variables in var.keys(): - for k, v in var[self.key_variables].items(): - variables[k] = v - - return variables - - def _get_dynvariables(self, profile): - """return the dyn variables""" - variables = {} - - # global dynvariables - if self.key_dynvariables in self.content: - # interpret dynamic variables - variables.update(self.content[self.key_dynvariables]) - - # external variables - variables.update(self.ext_dynvariables) - - if profile not in self.lnk_profiles: - return variables - - # profile dynvariables - var = self.lnk_profiles[profile] - if self.key_dynvariables in var.keys(): - variables.update(var[self.key_dynvariables]) - - return variables - - def dump(self): - """return a dump of the config""" - # temporary reset paths - dotpath = self.lnk_settings[self.key_dotpath] - workdir = self.lnk_settings[self.key_workdir] - self.lnk_settings[self.key_dotpath] = self.curdotpath - self.lnk_settings[self.key_workdir] = self.curworkdir - # dump - ret = yaml.safe_dump(self.content, - default_flow_style=False, - indent=2) - ret = ret.replace('{}', '') - # restore paths - self.lnk_settings[self.key_dotpath] = dotpath - self.lnk_settings[self.key_workdir] = workdir - return ret - - def is_modified(self): - """need the db to be saved""" - return self._modified - - def save(self): - """save the config to file""" - # temporary reset paths - dotpath = self.lnk_settings[self.key_dotpath] - workdir = self.lnk_settings[self.key_workdir] - self.lnk_settings[self.key_dotpath] = self.curdotpath - self.lnk_settings[self.key_workdir] = self.curworkdir - # save - ret = self._save(self.content, self.cfgpath) - # restore path - self.lnk_settings[self.key_dotpath] = dotpath - self.lnk_settings[self.key_workdir] = workdir - return ret diff --git a/dotdrop/dictparser.py b/dotdrop/dictparser.py new file mode 100644 index 0000000..3031164 --- /dev/null +++ b/dotdrop/dictparser.py @@ -0,0 +1,38 @@ +""" +author: deadc0de6 (https://github.com/deadc0de6) +Copyright (c) 2019, deadc0de6 + +dictionary parser abstract class +""" + +from dotdrop.logger import Logger + + +class DictParser: + + log = Logger() + + @classmethod + def _adjust_yaml_keys(cls, value): + """adjust value for object 'cls'""" + return value + + @classmethod + def parse(cls, key, value): + """parse (key,value) and construct object 'cls'""" + tmp = value + try: + tmp = value.copy() + except AttributeError: + pass + newv = cls._adjust_yaml_keys(tmp) + if not key: + return cls(**newv) + return cls(key=key, **newv) + + @classmethod + def parse_dict(cls, items): + """parse a dictionary and construct object 'cls'""" + if not items: + return [] + return [cls.parse(k, v) for k, v in items.items()] diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 409f4be..b734eae 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -15,7 +15,6 @@ from dotdrop.templategen import Templategen from dotdrop.installer import Installer from dotdrop.updater import Updater from dotdrop.comparator import Comparator -from dotdrop.config import Cfg from dotdrop.utils import get_tmpdir, remove, strip_home, run from dotdrop.linktypes import LinkTypes @@ -95,15 +94,13 @@ def cmd_install(o): for dotfile in dotfiles: # add dotfile variables t.restore_vars(tvars) - newvars = dotfile.get_vars() + newvars = dotfile.get_dotfile_variables() t.add_tmp_vars(newvars=newvars) preactions = [] - if not o.install_temporary and dotfile.actions \ - and Cfg.key_actions_pre in dotfile.actions: - for action in dotfile.actions[Cfg.key_actions_pre]: - preactions.append(action) - defactions = o.install_default_actions[Cfg.key_actions_pre] + if not o.install_temporary: + preactions.extend(dotfile.get_pre_actions()) + defactions = o.install_default_actions_pre pre_actions_exec = action_executor(o, dotfile, preactions, defactions, t, post=False) @@ -132,10 +129,9 @@ def cmd_install(o): if os.path.exists(tmp): remove(tmp) if r: - if not o.install_temporary and \ - Cfg.key_actions_post in dotfile.actions: - defactions = o.install_default_actions[Cfg.key_actions_post] - postactions = dotfile.actions[Cfg.key_actions_post] + if not o.install_temporary: + defactions = o.install_default_actions_post + postactions = dotfile.get_post_actions() post_actions_exec = action_executor(o, dotfile, postactions, defactions, t, post=True) post_actions_exec() @@ -329,8 +325,7 @@ def cmd_importer(o): LOG.err('importing \"{}\" failed!'.format(path)) ret = False continue - retconf, dotfile = o.conf.new(src, dst, o.profile, - linktype, debug=o.debug) + retconf = o.conf.new(src, dst, linktype, o.profile) if retconf: LOG.sub('\"{}\" imported'.format(path)) cnt += 1 @@ -355,7 +350,7 @@ def cmd_list_profiles(o): def cmd_list_files(o): """list all dotfiles for a specific profile""" - if o.profile not in o.profiles: + if o.profile not in [p.key for p in o.profiles]: LOG.warn('unknown profile \"{}\"'.format(o.profile)) return what = 'Dotfile(s)' @@ -375,7 +370,7 @@ def cmd_list_files(o): def cmd_detail(o): """list details on all files for all dotfile entries""" - if o.profile not in o.profiles: + if o.profile not in [p.key for p in o.profiles]: LOG.warn('unknown profile \"{}\"'.format(o.profile)) return dotfiles = o.dotfiles @@ -394,7 +389,7 @@ def cmd_detail(o): def _detail(dotpath, dotfile): - """print details on all files under a dotfile entry""" + """display details on all files under a dotfile entry""" LOG.log('{} (dst: \"{}\", link: {})'.format(dotfile.key, dotfile.dst, dotfile.link.name.lower())) path = os.path.join(dotpath, os.path.expanduser(dotfile.src)) @@ -404,7 +399,7 @@ def _detail(dotpath, dotfile): template = 'yes' LOG.sub('{} (template:{})'.format(path, template)) else: - for root, dir, files in os.walk(path): + for root, _, files in os.walk(path): for f in files: p = os.path.join(root, f) template = 'no' @@ -433,17 +428,17 @@ def apply_trans(dotpath, dotfile, debug=False): return None if fails and new source if succeed""" src = dotfile.src new_src = '{}.{}'.format(src, TRANS_SUFFIX) - trans = dotfile.trans_r - if debug: - LOG.dbg('executing transformation {}'.format(trans)) - s = os.path.join(dotpath, src) - temp = os.path.join(dotpath, new_src) - if not trans.transform(s, temp): - msg = 'transformation \"{}\" failed for {}' - LOG.err(msg.format(trans.key, dotfile.key)) - if new_src and os.path.exists(new_src): - remove(new_src) - return None + for trans in dotfile.trans_r: + if debug: + LOG.dbg('executing transformation {}'.format(trans)) + s = os.path.join(dotpath, src) + temp = os.path.join(dotpath, new_src) + if not trans.transform(s, temp): + msg = 'transformation \"{}\" failed for {}' + LOG.err(msg.format(trans.key, dotfile.key)) + if new_src and os.path.exists(new_src): + remove(new_src) + return None return new_src @@ -456,8 +451,8 @@ def main(): """entry point""" try: o = Options() - except ValueError as e: - LOG.err('Config error: {}'.format(str(e))) + except Exception as e: + LOG.err('options error: {}'.format(str(e))) return False ret = True @@ -512,9 +507,8 @@ def main(): LOG.err('interrupted') ret = False - if ret and o.conf.is_modified(): + if ret and o.conf.save(): LOG.log('config file updated') - o.conf.save() return ret diff --git a/dotdrop/dotfile.py b/dotdrop/dotfile.py index b11f8ce..8c4ab24 100644 --- a/dotdrop/dotfile.py +++ b/dotdrop/dotfile.py @@ -6,15 +6,23 @@ represents a dotfile in dotdrop """ from dotdrop.linktypes import LinkTypes +from dotdrop.dictparser import DictParser +from dotdrop.action import Action -class Dotfile: +class Dotfile(DictParser): + """Represent a dotfile.""" + # dotfile keys + key_noempty = 'ignoreempty' + key_trans_r = 'trans' + key_trans_w = 'trans_write' def __init__(self, key, dst, src, - actions={}, trans_r=None, trans_w=None, + actions=[], trans_r=[], trans_w=[], link=LinkTypes.NOLINK, cmpignore=[], noempty=False, upignore=[]): - """constructor + """ + constructor @key: dotfile key @dst: dotfile dst (in user's home usually) @src: dotfile src (in dotpath) @@ -26,39 +34,73 @@ class Dotfile: @noempty: ignore empty template if True @upignore: patterns to ignore when updating """ - self.key = key - self.dst = dst - self.src = src - self.link = link - # ensure link of right type - if type(link) != LinkTypes: - raise Exception('bad value for link: {}'.format(link)) self.actions = actions + self.cmpignore = cmpignore + self.dst = dst + self.key = key + self.link = LinkTypes.get(link) + self.noempty = noempty + self.src = src self.trans_r = trans_r self.trans_w = trans_w - self.cmpignore = cmpignore - self.noempty = noempty self.upignore = upignore - def get_vars(self): - """return this dotfile templating vars""" - _vars = {} - _vars['_dotfile_abs_src'] = self.src - _vars['_dotfile_abs_dst'] = self.dst - _vars['_dotfile_key'] = self.key - _vars['_dotfile_link'] = self.link.name.lower() + def get_dotfile_variables(self): + """return this dotfile specific variables""" + return { + '_dotfile_abs_src': self.src, + '_dotfile_abs_dst': self.dst, + '_dotfile_key': self.key, + '_dotfile_link': str(self.link), + } - return _vars + def get_pre_actions(self): + """return all 'pre' actions""" + return [a for a in self.actions if a.kind == Action.pre] - def __str__(self): - msg = 'key:\"{}\", src:\"{}\", dst:\"{}\", link:\"{}\"' - return msg.format(self.key, self.src, self.dst, self.link.name.lower()) + def get_post_actions(self): + """return all 'post' actions""" + return [a for a in self.actions if a.kind == Action.post] - def __repr__(self): - return 'dotfile({})'.format(self.__str__()) + def get_trans_r(self): + """return trans_r object""" + if self.trans_r: + return self.trans_r[0] + return None + + def get_trans_w(self): + """return trans_w object""" + if self.trans_w: + return self.trans_w[0] + return None + + @classmethod + def _adjust_yaml_keys(cls, value): + """patch dict""" + value['noempty'] = value.get(cls.key_noempty, False) + value['trans_r'] = value.get(cls.key_trans_r) + if value['trans_r']: + # ensure is a list + value['trans_r'] = [value['trans_r']] + value['trans_w'] = value.get(cls.key_trans_w) + if value['trans_w']: + # ensure is a list + value['trans_w'] = [value['trans_w']] + # remove old entries + value.pop(cls.key_noempty, None) + value.pop(cls.key_trans_r, None) + value.pop(cls.key_trans_w, None) + return value def __eq__(self, other): return self.__dict__ == other.__dict__ def __hash__(self): return hash(self.dst) ^ hash(self.src) ^ hash(self.key) + + def __str__(self): + msg = 'key:\"{}\", src:\"{}\", dst:\"{}\", link:\"{}\"' + return msg.format(self.key, self.src, self.dst, str(self.link)) + + def __repr__(self): + return 'dotfile({!s})'.format(self) diff --git a/dotdrop/linktypes.py b/dotdrop/linktypes.py index 59da01f..68e2b3b 100644 --- a/dotdrop/linktypes.py +++ b/dotdrop/linktypes.py @@ -5,3 +5,15 @@ class LinkTypes(IntEnum): NOLINK = 0 LINK = 1 LINK_CHILDREN = 2 + + @classmethod + def get(cls, key, default=None): + try: + return key if isinstance(key, cls) else cls[key.upper()] + except KeyError: + if default: + return default + raise ValueError('bad {} value: "{}"'.format(cls.__name__, key)) + + def __str__(self): + return self.name.lower() diff --git a/dotdrop/logger.py b/dotdrop/logger.py index fec1e07..51c6e8a 100644 --- a/dotdrop/logger.py +++ b/dotdrop/logger.py @@ -16,8 +16,10 @@ class Logger: YELLOW = '\033[93m' BLUE = '\033[94m' MAGENTA = '\033[95m' + LMAGENTA = '\033[35m' RESET = '\033[0m' EMPH = '\033[33m' + BOLD = '\033[1m' def __init__(self): pass @@ -37,10 +39,14 @@ class Logger: ce = self._color(self.RESET) sys.stderr.write('{}{}{}'.format(cs, string, ce)) - def err(self, string, end='\n'): + def err(self, string, end='\n', *, throw=None): cs = self._color(self.RED) ce = self._color(self.RESET) - sys.stderr.write('{}[ERR] {} {}{}'.format(cs, string, end, ce)) + msg = '{} {}'.format(string, end) + sys.stderr.write('{}[ERR] {}{}'.format(cs, msg, ce)) + + if throw is not None: + raise throw(msg) def warn(self, string, end='\n'): cs = self._color(self.YELLOW) @@ -53,8 +59,10 @@ class Logger: func = inspect.stack()[1][3] cs = self._color(self.MAGENTA) ce = self._color(self.RESET) - line = '{}[DEBUG][{}.{}] {}{}\n' - sys.stderr.write(line.format(cs, mod, func, string, ce)) + cl = self._color(self.LMAGENTA) + bl = self._color(self.BOLD) + line = '{}{}[DEBUG][{}.{}]{}{} {}{}\n' + sys.stderr.write(line.format(bl, cl, mod, func, ce, cs, string, ce)) def dry(self, string, end='\n'): cs = self._color(self.GREEN) diff --git a/dotdrop/options.py b/dotdrop/options.py index ae30cc6..5771789 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -14,7 +14,8 @@ from docopt import docopt from dotdrop.version import __version__ as VERSION from dotdrop.linktypes import LinkTypes from dotdrop.logger import Logger -from dotdrop.config import Cfg +from dotdrop.cfg_aggregator import CfgAggregator as Cfg +from dotdrop.action import Action ENV_PROFILE = 'DOTDROP_PROFILE' ENV_CONFIG = 'DOTDROP_CONFIG' @@ -107,24 +108,23 @@ class Options(AttrMonitor): if not args: self.args = docopt(USAGE, version=VERSION) self.log = Logger() - self.debug = self.args['--verbose'] - if not self.debug and ENV_DEBUG in os.environ: - self.debug = True + self.debug = self.args['--verbose'] or ENV_DEBUG in os.environ if ENV_NODEBUG in os.environ: + # force disabling debugs self.debug = False self.profile = self.args['--profile'] self.confpath = self._get_config_path() if self.debug: self.log.dbg('config file: {}'.format(self.confpath)) - self._read_config(self.profile) + self._read_config() self._apply_args() self._fill_attr() if ENV_NOBANNER not in os.environ \ and self.banner \ and not self.args['--no-banner']: self._header() - self._print_attr() + self._debug_attr() # start monitoring for bad attribute self._set_attr_err = True @@ -167,25 +167,18 @@ class Options(AttrMonitor): return None - def _find_cfg(self, paths): - """try to find the config in the paths list""" - for path in paths: - if os.path.exists(path): - return path - return None - def _header(self): - """print the header""" + """display the header""" self.log.log(BANNER) self.log.log('') - def _read_config(self, profile=None): + def _read_config(self): """read the config file""" - self.conf = Cfg(self.confpath, profile=profile, debug=self.debug) + self.conf = Cfg(self.confpath, self.profile, debug=self.debug) # transform the config settings to self attribute for k, v in self.conf.get_settings().items(): if self.debug: - self.log.dbg('setting: {}={}'.format(k, v)) + self.log.dbg('new setting: {}={}'.format(k, v)) setattr(self, k, v) def _apply_args(self): @@ -212,8 +205,6 @@ class Options(AttrMonitor): self.log.err('bad option for --link: {}'.format(link)) sys.exit(USAGE) self.import_link = OPT_LINK[link] - if self.debug: - self.log.dbg('link_import value: {}'.format(self.import_link)) # "listfiles" specifics self.listfiles_templateonly = self.args['--template'] @@ -223,7 +214,10 @@ class Options(AttrMonitor): self.install_diff = not self.args['--nodiff'] self.install_showdiff = self.showdiff or self.args['--showdiff'] self.install_backup_suffix = BACKUP_SUFFIX - self.install_default_actions = self.default_actions + self.install_default_actions_pre = [a for a in self.default_actions + if a.kind == Action.pre] + self.install_default_actions_post = [a for a in self.default_actions + if a.kind == Action.post] # "compare" specifics self.compare_dopts = self.args['--dopts'] self.compare_focus = self.args['--file'] @@ -243,26 +237,24 @@ class Options(AttrMonitor): def _fill_attr(self): """create attributes from conf""" # variables - self.variables = self.conf.get_variables(self.profile, - debug=self.debug).copy() + self.variables = self.conf.get_variables() # the dotfiles - self.dotfiles = self.conf.eval_dotfiles(self.profile, self.variables, - debug=self.debug).copy() + self.dotfiles = self.conf.get_dotfiles(self.profile) # the profiles self.profiles = self.conf.get_profiles() - def _print_attr(self): - """print all of this class attributes""" + def _debug_attr(self): + """debug display all of this class attributes""" if not self.debug: return - self.log.dbg('options:') + self.log.dbg('CLI options:') for att in dir(self): if att.startswith('_'): continue val = getattr(self, att) if callable(val): continue - self.log.dbg('- {}: \"{}\"'.format(att, val)) + self.log.dbg('- {}: {}'.format(att, val)) def _attr_set(self, attr): """error when some inexistent attr is set""" diff --git a/dotdrop/profile.py b/dotdrop/profile.py new file mode 100644 index 0000000..5a1e671 --- /dev/null +++ b/dotdrop/profile.py @@ -0,0 +1,50 @@ +""" +author: deadc0de6 (https://github.com/deadc0de6) +Copyright (c) 2019, deadc0de6 + +represent a profile in dotdrop +""" + +from dotdrop.dictparser import DictParser + + +class Profile(DictParser): + + # profile keys + key_include = 'include' + key_import = 'import' + + def __init__(self, key, actions=[], dotfiles=[], variables=[]): + """ + constructor + @key: profile key + @actions: list of action keys + @dotfiles: list of dotfile keys + @variables: list of variable keys + """ + self.key = key + self.actions = actions + self.dotfiles = dotfiles + self.variables = variables + + @classmethod + def _adjust_yaml_keys(cls, value): + """patch dict""" + value.pop(cls.key_import, None) + value.pop(cls.key_include, None) + return value + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + def __hash__(self): + return (hash(self.key) ^ + hash(tuple(self.dotfiles)) ^ + hash(tuple(self.included_profiles))) + + def __str__(self): + msg = 'key:"{}"' + return msg.format(self.key) + + def __repr__(self): + return 'profile({!s})'.format(self) diff --git a/dotdrop/settings.py b/dotdrop/settings.py new file mode 100644 index 0000000..1d2a5dc --- /dev/null +++ b/dotdrop/settings.py @@ -0,0 +1,96 @@ +""" +author: deadc0de6 (https://github.com/deadc0de6) +Copyright (c) 2019, deadc0de6 + +settings block +""" + +# local imports +from dotdrop.linktypes import LinkTypes +from dotdrop.dictparser import DictParser + + +class Settings(DictParser): + # key in yaml file + key_yaml = 'config' + + # settings item keys + key_backup = 'backup' + key_banner = 'banner' + key_cmpignore = 'cmpignore' + key_create = 'create' + key_default_actions = 'default_actions' + key_dotpath = 'dotpath' + key_ignoreempty = 'ignoreempty' + key_keepdot = 'keepdot' + key_longkey = 'longkey' + key_link_dotfile_default = 'link_dotfile_default' + key_link_on_import = 'link_on_import' + key_showdiff = 'showdiff' + key_upignore = 'upignore' + key_workdir = 'workdir' + + # import keys + key_import_actions = 'import_actions' + key_import_configs = 'import_configs' + key_import_variables = 'import_variables' + + def __init__(self, backup=True, banner=True, cmpignore=[], + create=True, default_actions=[], dotpath='dotfiles', + ignoreempty=True, import_actions=[], import_configs=[], + import_variables=[], keepdot=False, + link_dotfile_default=LinkTypes.NOLINK, + link_on_import=LinkTypes.NOLINK, longkey=False, + showdiff=False, upignore=[], workdir='~/.config/dotdrop'): + self.backup = backup + self.banner = banner + self.create = create + self.cmpignore = cmpignore + self.default_actions = default_actions + self.dotpath = dotpath + self.ignoreempty = ignoreempty + self.import_actions = import_actions + self.import_configs = import_configs + self.import_variables = import_variables + self.keepdot = keepdot + self.longkey = longkey + self.showdiff = showdiff + self.upignore = upignore + self.workdir = workdir + self.link_dotfile_default = LinkTypes.get(link_dotfile_default) + self.link_on_import = LinkTypes.get(link_on_import) + + def resolve_paths(self, resolver): + """resolve path using resolver function""" + self.dotpath = resolver(self.dotpath) + self.workdir = resolver(self.workdir) + + def _serialize_seq(self, name, dic): + """serialize attribute 'name' into 'dic'""" + seq = getattr(self, name) + dic[name] = seq + + def serialize(self): + """Return key-value pair representation of the settings""" + # Tedious, but less error-prone than introspection + dic = { + self.key_backup: self.backup, + self.key_banner: self.banner, + self.key_create: self.create, + self.key_dotpath: self.dotpath, + self.key_ignoreempty: self.ignoreempty, + self.key_keepdot: self.keepdot, + self.key_link_dotfile_default: str(self.link_dotfile_default), + self.key_link_on_import: str(self.link_on_import), + self.key_longkey: self.longkey, + self.key_showdiff: self.showdiff, + self.key_workdir: self.workdir, + } + self._serialize_seq(self.key_cmpignore, dic) + self._serialize_seq(self.key_default_actions, dic) + self._serialize_seq(self.key_import_actions, dic) + self._serialize_seq(self.key_import_configs, dic) + self._serialize_seq(self.key_import_variables, dic) + self._serialize_seq(self.key_upignore, dic) + + return {self.key_yaml: dic} diff --git a/dotdrop/templategen.py b/dotdrop/templategen.py index 3b22d80..e532a5f 100644 --- a/dotdrop/templategen.py +++ b/dotdrop/templategen.py @@ -52,6 +52,8 @@ class Templategen: self.env.globals['exists_in_path'] = jhelpers.exists_in_path self.env.globals['basename'] = jhelpers.basename self.env.globals['dirname'] = jhelpers.dirname + if self.debug: + self.log.dbg('template additional variables: {}'.format(variables)) def generate(self, src): """render template from path""" diff --git a/dotdrop/updater.py b/dotdrop/updater.py index 65b39e1..5524d5c 100644 --- a/dotdrop/updater.py +++ b/dotdrop/updater.py @@ -81,12 +81,11 @@ class Updater: if self._ignore([path, dtpath]): self.log.sub('\"{}\" ignored'.format(dotfile.key)) return True - if dotfile.trans_w: - # apply write transformation if any - new_path = self._apply_trans_w(path, dotfile) - if not new_path: - return False - path = new_path + # apply write transformation if any + new_path = self._apply_trans_w(path, dotfile) + if not new_path: + return False + path = new_path if os.path.isdir(path): ret = self._handle_dir(path, dtpath) else: @@ -98,7 +97,9 @@ class Updater: def _apply_trans_w(self, path, dotfile): """apply write transformation to dotfile""" - trans = dotfile.trans_w + trans = dotfile.get_trans_w() + if not trans: + return path if self.debug: self.log.dbg('executing write transformation {}'.format(trans)) tmp = utils.get_unique_tmp_name() diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 70b1a61..88dbf75 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -66,7 +66,7 @@ def get_tmpdir(): def get_tmpfile(): """create a temporary file""" - (fd, path) = tempfile.mkstemp(prefix='dotdrop-') + (_, path) = tempfile.mkstemp(prefix='dotdrop-') return path diff --git a/dotdrop/version.py b/dotdrop/version.py index e35d6cc..a658c2b 100644 --- a/dotdrop/version.py +++ b/dotdrop/version.py @@ -3,4 +3,4 @@ author: deadc0de6 (https://github.com/deadc0de6) Copyright (c) 2018, deadc0de6 """ -__version__ = '0.28.0' +__version__ = '0.27.0' diff --git a/packages/arch-dotdrop/.SRCINFO b/packages/arch-dotdrop/.SRCINFO index 26b449c..4da2e16 100644 --- a/packages/arch-dotdrop/.SRCINFO +++ b/packages/arch-dotdrop/.SRCINFO @@ -1,6 +1,6 @@ pkgbase = dotdrop pkgdesc = Save your dotfiles once, deploy them everywhere - pkgver = 0.28.0 + pkgver = 0.27.0 pkgrel = 1 url = https://github.com/deadc0de6/dotdrop arch = any @@ -11,7 +11,7 @@ pkgbase = dotdrop depends = python-jinja depends = python-docopt depends = python-pyaml - source = git+https://github.com/deadc0de6/dotdrop.git#tag=v0.28.0 + source = git+https://github.com/deadc0de6/dotdrop.git#tag=v0.27.0 md5sums = SKIP pkgname = dotdrop diff --git a/packages/arch-dotdrop/PKGBUILD b/packages/arch-dotdrop/PKGBUILD index 5891d6a..20da6a6 100644 --- a/packages/arch-dotdrop/PKGBUILD +++ b/packages/arch-dotdrop/PKGBUILD @@ -1,7 +1,7 @@ # Maintainer: deadc0de6 pkgname=dotdrop -pkgver=0.28.0 +pkgver=0.27.0 pkgrel=1 pkgdesc="Save your dotfiles once, deploy them everywhere " arch=('any') diff --git a/scripts/change-link.py b/scripts/change-link.py index 8911d1b..f2ee974 100755 --- a/scripts/change-link.py +++ b/scripts/change-link.py @@ -42,7 +42,7 @@ def main(): ignores = args['--ignore'] with open(path, 'r') as f: - content = yaml.load(f) + content = yaml.safe_load(f) for k, v in content[key].items(): if k in ignores: continue diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh index e5fadfe..0460c91 100755 --- a/tests-ng/compare.sh +++ b/tests-ng/compare.sh @@ -93,6 +93,7 @@ create_conf ${cfg} # sets token echo "[+] import" cd ${ddpath} | ${bin} import -c ${cfg} ${tmpd}/dir1 cd ${ddpath} | ${bin} import -c ${cfg} ${tmpd}/uniquefile +cat ${cfg} # let's see the dotpath #tree ${basedir}/dotfiles diff --git a/tests-ng/dotdrop-variables.sh b/tests-ng/dotdrop-variables.sh index 3978512..3659f5f 100755 --- a/tests-ng/dotdrop-variables.sh +++ b/tests-ng/dotdrop-variables.sh @@ -79,9 +79,9 @@ echo "cfgpath: {{@@ _dotdrop_cfgpath @@}}" >> ${tmps}/dotfiles/abc echo "workdir: {{@@ _dotdrop_workdir @@}}" >> ${tmps}/dotfiles/abc # install -cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V -#cat ${tmpd}/abc +cat ${tmpd}/abc grep "^dotpath: ${tmps}/dotfiles$" ${tmpd}/abc >/dev/null grep "^cfgpath: ${tmps}/config.yaml$" ${tmpd}/abc >/dev/null diff --git a/tests-ng/dotfile-variables.sh b/tests-ng/dotfile-variables.sh index e23b0aa..a33c98f 100755 --- a/tests-ng/dotfile-variables.sh +++ b/tests-ng/dotfile-variables.sh @@ -81,7 +81,7 @@ cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V # checks [ ! -e ${tmpd}/abc ] && echo 'dotfile not installed' && exit 1 -#cat ${tmpd}/abc +cat ${tmpd}/abc grep "src:${tmps}/dotfiles/abc" ${tmpd}/abc >/dev/null grep "dst:${tmpd}/abc" ${tmpd}/abc >/dev/null grep "key:f_abc" ${tmpd}/abc >/dev/null diff --git a/tests-ng/ext-actions.sh b/tests-ng/ext-actions.sh index 550d69b..f753dbf 100755 --- a/tests-ng/ext-actions.sh +++ b/tests-ng/ext-actions.sh @@ -96,7 +96,7 @@ _EOF echo "test" > ${tmps}/dotfiles/abc # install -cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V # checks [ ! -e ${tmpa}/pre ] && exit 1 diff --git a/tests-ng/import-configs.sh b/tests-ng/import-configs.sh new file mode 100755 index 0000000..1507e2d --- /dev/null +++ b/tests-ng/import-configs.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2019, deadc0de6 +# +# import config testing +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "\e[96m\e[1m==> RUNNING $(basename $BASH_SOURCE) <==\e[0m" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests'` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests'` + +# create the config file +cfg1="${tmps}/config1.yaml" +cfg2="${tmps}/config2.yaml" + +cat > ${cfg1} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + import_configs: + - ${cfg2} +dotfiles: + f_abc: + dst: ${tmpd}/abc + src: abc + f_zzz: + dst: ${tmpd}/zzz + src: zzz + f_sub: + dst: ${tmpd}/sub + src: sub +profiles: + p0: + include: + - p2 + p1: + dotfiles: + - f_abc + p3: + dotfiles: + - f_zzz + pup: + include: + - psubsub +_EOF + +cat > ${cfg2} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: + f_def: + dst: ${tmpd}/def + src: def + f_ghi: + dst: ${tmpd}/ghi + src: ghi +profiles: + p2: + dotfiles: + - f_def + psubsub: + dotfiles: + - f_sub +_EOF + +# create the source +mkdir -p ${tmps}/dotfiles/ +echo "abc" > ${tmps}/dotfiles/abc +echo "def" > ${tmps}/dotfiles/def +echo "ghi" > ${tmps}/dotfiles/ghi +echo "zzz" > ${tmps}/dotfiles/zzz +echo "sub" > ${tmps}/dotfiles/sub + +# install +cd ${ddpath} | ${bin} listfiles -c ${cfg1} -p p0 -V | grep f_def +cd ${ddpath} | ${bin} listfiles -c ${cfg1} -p p1 -V | grep f_abc +cd ${ddpath} | ${bin} listfiles -c ${cfg1} -p p2 -V | grep f_def +cd ${ddpath} | ${bin} listfiles -c ${cfg1} -p p3 -V | grep f_zzz +cd ${ddpath} | ${bin} listfiles -c ${cfg1} -p pup -V | grep f_sub +cd ${ddpath} | ${bin} listfiles -c ${cfg1} -p psubsub -V | grep f_sub + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 diff --git a/tests-ng/import-profile-dotfiles.sh b/tests-ng/import-profile-dotfiles.sh new file mode 100755 index 0000000..402b041 --- /dev/null +++ b/tests-ng/import-profile-dotfiles.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2017, deadc0de6 +# +# test the use of the keyword "import" in profiles +# returns 1 in case of error +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "\e[96m\e[1m==> RUNNING $(basename $BASH_SOURCE) <==\e[0m" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests'` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests'` +extdotfiles="${tmps}/df_p1.yaml" + +dynextdotfiles_name="d_uid_dynvar" +dynextdotfiles="${tmps}/ext_${dynextdotfiles_name}" + +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dynvariables: + d_uid: "echo ${dynextdotfiles_name}" +dotfiles: + f_abc: + dst: ${tmpd}/abc + src: abc + f_def: + dst: ${tmpd}/def + src: def + f_xyz: + dst: ${tmpd}/xyz + src: xyz + f_dyn: + dst: ${tmpd}/dyn + src: dyn +profiles: + p1: + dotfiles: + - f_abc + import: + - $(basename ${extdotfiles}) + - "ext_{{@@ d_uid @@}}" +_EOF + +# create the external dotfile file +cat > ${extdotfiles} << _EOF +dotfiles: + - f_def + - f_xyz +_EOF + +cat > ${dynextdotfiles} << _EOF +dotfiles: + - f_dyn +_EOF + +# create the source +mkdir -p ${tmps}/dotfiles/ +echo "abc" > ${tmps}/dotfiles/abc +echo "def" > ${tmps}/dotfiles/def +echo "xyz" > ${tmps}/dotfiles/xyz +echo "dyn" > ${tmps}/dotfiles/dyn + +# install +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V + +# checks +[ ! -e ${tmpd}/abc ] && exit 1 +[ ! -e ${tmpd}/def ] && exit 1 +[ ! -e ${tmpd}/xyz ] && exit 1 +[ ! -e ${tmpd}/dyn ] && exit 1 +echo 'file found' +grep 'abc' ${tmpd}/abc >/dev/null 2>&1 +grep 'def' ${tmpd}/def >/dev/null 2>&1 +grep 'xyz' ${tmpd}/xyz >/dev/null 2>&1 +grep 'dyn' ${tmpd}/dyn >/dev/null 2>&1 + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 diff --git a/tests-ng/import.sh b/tests-ng/import.sh index c71972a..6d7c0af 100755 --- a/tests-ng/import.sh +++ b/tests-ng/import.sh @@ -1,9 +1,8 @@ #!/usr/bin/env bash # author: deadc0de6 (https://github.com/deadc0de6) -# Copyright (c) 2017, deadc0de6 +# Copyright (c) 2019, deadc0de6 # -# test the use of the keyword "import" in profiles -# returns 1 in case of error +# test basic import # # exit on first error @@ -50,10 +49,13 @@ tmps=`mktemp -d --suffix='-dotdrop-tests'` mkdir -p ${tmps}/dotfiles # the dotfile destination tmpd=`mktemp -d --suffix='-dotdrop-tests'` -extdotfiles="${tmps}/df_p1.yaml" +#echo "dotfile destination: ${tmpd}" -dynextdotfiles_name="d_uid_dynvar" -dynextdotfiles="${tmps}/ext_${dynextdotfiles_name}" +# create the dotfile +mkdir -p ${tmpd}/adir +echo "adir/file1" > ${tmpd}/adir/file1 +echo "adir/fil2" > ${tmpd}/adir/file2 +echo "file3" > ${tmpd}/file3 # create the config file cfg="${tmps}/config.yaml" @@ -63,61 +65,30 @@ config: backup: true create: true dotpath: dotfiles -dynvariables: - d_uid: "echo ${dynextdotfiles_name}" dotfiles: - f_abc: - dst: ${tmpd}/abc - src: abc - f_def: - dst: ${tmpd}/def - src: def - f_xyz: - dst: ${tmpd}/xyz - src: xyz - f_dyn: - dst: ${tmpd}/dyn - src: dyn profiles: - p1: - dotfiles: - - f_abc - import: - - $(basename ${extdotfiles}) - - "ext_{{@@ d_uid @@}}" _EOF +#cat ${cfg} -# create the external dotfile file -cat > ${extdotfiles} << _EOF -dotfiles: - - f_def - - f_xyz -_EOF +# import +cd ${ddpath} | ${bin} import -c ${cfg} -p p1 -V ${tmpd}/adir +cd ${ddpath} | ${bin} import -c ${cfg} -p p1 -V ${tmpd}/file3 -cat > ${dynextdotfiles} << _EOF -dotfiles: - - f_dyn -_EOF +cat ${cfg} -# create the source -mkdir -p ${tmps}/dotfiles/ -echo "abc" > ${tmps}/dotfiles/abc -echo "def" > ${tmps}/dotfiles/def -echo "xyz" > ${tmps}/dotfiles/xyz -echo "dyn" > ${tmps}/dotfiles/dyn +# ensure exists and is not link +[ ! -d ${tmps}/dotfiles/${tmpd}/adir ] && echo "not a directory" && exit 1 +[ ! -e ${tmps}/dotfiles/${tmpd}/adir/file1 ] && echo "not exist" && exit 1 +[ ! -e ${tmps}/dotfiles/${tmpd}/adir/file2 ] && echo "not exist" && exit 1 +[ ! -e ${tmps}/dotfiles/${tmpd}/file3 ] && echo "not a file" && exit 1 -# install -cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V +cat ${cfg} | grep ${tmpd}/adir >/dev/null 2>&1 +cat ${cfg} | grep ${tmpd}/file3 >/dev/null 2>&1 -# checks -[ ! -e ${tmpd}/abc ] && exit 1 -[ ! -e ${tmpd}/def ] && exit 1 -[ ! -e ${tmpd}/xyz ] && exit 1 -[ ! -e ${tmpd}/dyn ] && exit 1 -grep 'abc' ${tmpd}/abc >/dev/null 2>&1 -grep 'def' ${tmpd}/def >/dev/null 2>&1 -grep 'xyz' ${tmpd}/xyz >/dev/null 2>&1 -grep 'dyn' ${tmpd}/dyn >/dev/null 2>&1 +nb=`cat ${cfg} | grep d_adir | wc -l` +[ "${nb}" != "2" ] && echo 'bad config1' && exit 1 +nb=`cat ${cfg} | grep f_file3 | wc -l` +[ "${nb}" != "2" ] && echo 'bad config2' && exit 1 ## CLEANING rm -rf ${tmps} ${tmpd} diff --git a/tests-ng/include.sh b/tests-ng/include.sh index c91c05d..e5dd12e 100755 --- a/tests-ng/include.sh +++ b/tests-ng/include.sh @@ -64,12 +64,18 @@ dotfiles: dst: ${tmpd}/abc src: abc profiles: + p0: + include: + - p3 p1: dotfiles: - f_abc p2: include: - p1 + p3: + include: + - p2 _EOF # create the source @@ -82,6 +88,14 @@ cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 # compare cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 cd ${ddpath} | ${bin} compare -c ${cfg} -p p2 +cd ${ddpath} | ${bin} compare -c ${cfg} -p p3 +cd ${ddpath} | ${bin} compare -c ${cfg} -p p0 + +# list +cd ${ddpath} | ${bin} listfiles -c ${cfg} -p p1 | grep f_abc +cd ${ddpath} | ${bin} listfiles -c ${cfg} -p p2 | grep f_abc +cd ${ddpath} | ${bin} listfiles -c ${cfg} -p p3 | grep f_abc +cd ${ddpath} | ${bin} listfiles -c ${cfg} -p p0 | grep f_abc # count cnt=`cd ${ddpath} | ${bin} listfiles -c ${cfg} -p p1 -b | grep '^f_' | wc -l` diff --git a/tests.sh b/tests.sh index d59efdb..3d2088b 100755 --- a/tests.sh +++ b/tests.sh @@ -8,7 +8,7 @@ set -ev # PEP8 tests which pycodestyle 2>/dev/null [ "$?" != "0" ] && echo "Install pycodestyle" && exit 1 -pycodestyle --ignore=W605 dotdrop/ +pycodestyle --ignore=W503,W504,W605 dotdrop/ pycodestyle tests/ pycodestyle scripts/ @@ -35,7 +35,17 @@ PYTHONPATH=dotdrop ${nosebin} -s --with-coverage --cover-package=dotdrop ## execute bash script tests [ "$1" = '--python-only' ] || { - for scr in tests-ng/*.sh; do - ${scr} - done + log=`mktemp` + for scr in tests-ng/*.sh; do + ${scr} 2>&1 | tee ${log} + set +e + if grep Traceback ${log}; then + echo "crash found in logs" + rm -f ${log} + exit 1 + fi + set -e + done + rm -f ${log} } + diff --git a/tests/helpers.py b/tests/helpers.py index eeaa5bd..6656a4b 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -171,8 +171,9 @@ def get_dotfile_from_yaml(dic, path): """Return the dotfile from the yaml dictionary""" # path is not the file in dotpath but on the FS dotfiles = dic['dotfiles'] - src = get_path_strip_version(path) - return [d for d in dotfiles.values() if d['src'] == src][0] + # src = get_path_strip_version(path) + dotfile = [d for d in dotfiles.values() if d['dst'] == path][0] + return dotfile def yaml_dashed_list(items, indent=0): @@ -256,10 +257,10 @@ def file_in_yaml(yaml_file, path, link=False): dotfiles = yaml_conf['dotfiles'].values() - in_src = strip in (x['src'] for x in dotfiles) + in_src = any([x['src'].endswith(strip) for x in dotfiles]) in_dst = path in (os.path.expanduser(x['dst']) for x in dotfiles) if link: - has_link = get_dotfile_from_yaml(yaml_conf, path)['link'] + has_link = 'link' in get_dotfile_from_yaml(yaml_conf, path) return in_src and in_dst and has_link return in_src and in_dst diff --git a/tests/test_import.py b/tests/test_import.py index 5e712f1..25bb861 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -33,7 +33,7 @@ class TestImport(unittest.TestCase): self.assertTrue(os.path.exists(path)) content = '' with open(path, 'r') as f: - content = yaml.load(f) + content = yaml.safe_load(f) return content def assert_file(self, path, o, profile): @@ -45,7 +45,7 @@ class TestImport(unittest.TestCase): def assert_in_yaml(self, path, dic, link=False): """Make sure "path" is in the "dic" representing the yaml file""" - self.assertTrue(file_in_yaml(dic, path, link)) + self.assertTrue(file_in_yaml(dic, path, link=link)) def test_import(self): """Test the import function""" @@ -117,7 +117,7 @@ class TestImport(unittest.TestCase): o = load_options(confpath, profile) # test dotfiles in config class - self.assertTrue(profile in o.profiles) + self.assertTrue(profile in [p.key for p in o.profiles]) self.assert_file(dotfile1, o, profile) self.assert_file(dotfile2, o, profile) self.assert_file(dotfile3, o, profile) @@ -218,9 +218,10 @@ class TestImport(unittest.TestCase): self.assertTrue(os.path.exists(dotdrop_home)) self.addCleanup(clean, dotdrop_home) + dotpath_ed = 'imported' imported = { 'config': { - 'dotpath': 'imported', + 'dotpath': dotpath_ed, }, 'dotfiles': {}, 'profiles': { @@ -250,9 +251,10 @@ class TestImport(unittest.TestCase): 'dv_log_ed': 'echo 5', }, } + dotpath_ing = 'importing' importing = { 'config': { - 'dotpath': 'importing', + 'dotpath': dotpath_ing, }, 'dotfiles': {}, 'profiles': { @@ -293,7 +295,7 @@ class TestImport(unittest.TestCase): # create the importing base config file importing_path = create_fake_config(dotdrop_home, configname='config.yaml', - import_configs=('config-*.yaml',), + import_configs=['config-2.yaml'], **importing['config']) # edit the imported config @@ -326,8 +328,10 @@ class TestImport(unittest.TestCase): y = self.load_yaml(imported_path) # testing dotfiles - self.assertTrue(all(file_in_yaml(y, df) for df in dotfiles_ed)) - self.assertFalse(any(file_in_yaml(y, df) for df in dotfiles_ing)) + self.assertTrue(all(file_in_yaml(y, df) + for df in dotfiles_ed)) + self.assertFalse(any(file_in_yaml(y, df) + for df in dotfiles_ing)) # testing profiles profiles = y['profiles'].keys() @@ -355,7 +359,7 @@ class TestImport(unittest.TestCase): self.assertFalse(any(t.endswith('ing') for t in transformations)) # testing variables - variables = y['variables'].keys() + variables = self._remove_priv_vars(y['variables'].keys()) self.assertTrue(all(v.endswith('ed') for v in variables)) self.assertFalse(any(v.endswith('ing') for v in variables)) dyn_variables = y['dynvariables'].keys() @@ -366,8 +370,10 @@ class TestImport(unittest.TestCase): y = self.load_yaml(importing_path) # testing dotfiles - self.assertTrue(all(file_in_yaml(y, df) for df in dotfiles_ing)) - self.assertFalse(any(file_in_yaml(y, df) for df in dotfiles_ed)) + self.assertTrue(all(file_in_yaml(y, df) + for df in dotfiles_ing)) + self.assertFalse(any(file_in_yaml(y, df) + for df in dotfiles_ed)) # testing profiles profiles = y['profiles'].keys() @@ -395,13 +401,19 @@ class TestImport(unittest.TestCase): self.assertFalse(any(t.endswith('ed') for t in transformations)) # testing variables - variables = y['variables'].keys() + variables = self._remove_priv_vars(y['variables'].keys()) self.assertTrue(all(v.endswith('ing') for v in variables)) self.assertFalse(any(v.endswith('ed') for v in variables)) dyn_variables = y['dynvariables'].keys() self.assertTrue(all(dv.endswith('ing') for dv in dyn_variables)) self.assertFalse(any(dv.endswith('ed') for dv in dyn_variables)) + def _remove_priv_vars(self, variables_keys): + variables = [v for v in variables_keys if not v.startswith('_')] + if 'profile' in variables: + variables.remove('profile') + return variables + def main(): unittest.main() diff --git a/tests/test_install.py b/tests/test_install.py index b4cd587..80e1b4f 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -9,7 +9,7 @@ import unittest from unittest.mock import MagicMock, patch import filecmp -from dotdrop.config import Cfg +from dotdrop.cfg_aggregator import CfgAggregator as Cfg from tests.helpers import (clean, create_dir, create_fake_config, create_random_file, get_string, get_tempdir, load_options, populate_fake_config) @@ -89,7 +89,7 @@ exec bspwm f1, c1 = create_random_file(tmp) dst1 = os.path.join(dst, get_string(6)) d1 = Dotfile(get_string(5), dst1, os.path.basename(f1)) - # fake a print + # fake a __str__ self.assertTrue(str(d1) != '') f2, c2 = create_random_file(tmp) dst2 = os.path.join(dst, get_string(6)) @@ -178,7 +178,7 @@ exec bspwm dotfiles = [d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, ddot] self.fake_config(confpath, dotfiles, profile, tmp, [act1], [tr]) - conf = Cfg(confpath) + conf = Cfg(confpath, profile) self.assertTrue(conf is not None) # install them @@ -305,7 +305,7 @@ exec bspwm # create the importing base config file importing_path = create_fake_config(tmp, configname='config.yaml', - import_configs=('config-*.yaml',), + import_configs=['config-2.yaml'], **importing['config']) # edit the imported config diff --git a/tests/test_update.py b/tests/test_update.py index 4b30621..2fdee34 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -117,7 +117,7 @@ class TestUpdate(unittest.TestCase): # retrieve the path of the sub in the dotpath d1indotpath = os.path.join(o.dotpath, dotfile.src) d1indotpath = os.path.expanduser(d1indotpath) - dotfile.trans_w = trans + dotfile.trans_w = [trans] # update template o.update_path = [d3t] diff --git a/tests/test_config.py b/tests/test_yamlcfg.py similarity index 80% rename from tests/test_config.py rename to tests/test_yamlcfg.py index 2b23d7f..a2cd152 100644 --- a/tests/test_config.py +++ b/tests/test_yamlcfg.py @@ -10,7 +10,7 @@ from unittest.mock import patch import os import yaml -from dotdrop.config import Cfg +from dotdrop.cfg_yaml import CfgYaml as Cfg from dotdrop.options import Options from dotdrop.linktypes import LinkTypes from tests.helpers import (SubsetTestCase, _fake_args, clean, @@ -41,14 +41,12 @@ class TestConfig(SubsetTestCase): conf = Cfg(confpath) self.assertTrue(conf is not None) - opts = conf.get_settings() + opts = conf.settings self.assertTrue(opts is not None) self.assertTrue(opts != {}) self.assertTrue(opts['backup'] == self.CONFIG_BACKUP) self.assertTrue(opts['create'] == self.CONFIG_CREATE) - dotpath = os.path.join(tmp, self.CONFIG_DOTPATH) - self.assertTrue(opts['dotpath'] == dotpath) - self.assertTrue(conf._is_valid()) + self.assertTrue(opts['dotpath'] == self.CONFIG_DOTPATH) self.assertTrue(conf.dump() != '') def test_def_link(self): @@ -68,8 +66,8 @@ class TestConfig(SubsetTestCase): 'link_children') self._test_link_import_fail('whatever') - @patch('dotdrop.config.open', create=True) - @patch('dotdrop.config.os.path.exists', create=True) + @patch('dotdrop.cfg_yaml.open', create=True) + @patch('dotdrop.cfg_yaml.os.path.exists', create=True) def _test_link_import(self, cfgstring, expected, cliargs, mock_exists, mock_open): data = ''' @@ -99,8 +97,8 @@ profiles: self.assertTrue(o.import_link == expected) - @patch('dotdrop.config.open', create=True) - @patch('dotdrop.config.os.path.exists', create=True) + @patch('dotdrop.cfg_yaml.open', create=True) + @patch('dotdrop.cfg_yaml.os.path.exists', create=True) def _test_link_import_fail(self, value, mock_exists, mock_open): data = ''' config: @@ -125,7 +123,7 @@ profiles: args['--profile'] = 'p1' args['--cfg'] = 'mocked' - with self.assertRaisesRegex(ValueError, 'config is not valid'): + with self.assertRaises(ValueError): o = Options(args=args) print(o.import_link) @@ -143,7 +141,7 @@ profiles: # edit the config with open(confpath, 'r') as f: - content = yaml.load(f) + content = yaml.safe_load(f) # adding dotfiles df1key = 'f_vimrc' @@ -171,22 +169,22 @@ profiles: self.assertTrue(conf is not None) # test profile - profiles = conf.get_profiles() + profiles = conf.profiles self.assertTrue(pf1key in profiles) self.assertTrue(pf2key in profiles) # test dotfiles - dotfiles = conf._get_dotfiles(pf1key) - self.assertTrue(df1key in [x.key for x in dotfiles]) - self.assertTrue(df2key in [x.key for x in dotfiles]) - dotfiles = conf._get_dotfiles(pf2key) - self.assertTrue(df1key in [x.key for x in dotfiles]) - self.assertFalse(df2key in [x.key for x in dotfiles]) + dotfiles = conf.profiles[pf1key]['dotfiles'] + self.assertTrue(df1key in dotfiles) + self.assertTrue(df2key in dotfiles) + dotfiles = conf.profiles[pf2key]['dotfiles'] + self.assertTrue(df1key in dotfiles) + self.assertFalse(df2key in dotfiles) # test not existing included profile # edit the config with open(confpath, 'r') as f: - content = yaml.load(f) + content = yaml.safe_load(f) content['profiles'] = { pf1key: {'dotfiles': [df2key], 'include': ['host2']}, pf2key: {'dotfiles': [df1key], 'include': ['host3']} @@ -227,22 +225,26 @@ profiles: vars_ing_file = create_yaml_keyval(vars_ing, tmp) actions_ed = { - 'pre': { - 'a_pre_action_ed': 'echo pre 22', - }, - 'post': { - 'a_post_action_ed': 'echo post 22', - }, - 'a_action_ed': 'echo 22', + 'actions': { + 'pre': { + 'a_pre_action_ed': 'echo pre 22', + }, + 'post': { + 'a_post_action_ed': 'echo post 22', + }, + 'a_action_ed': 'echo 22', + } } actions_ing = { - 'pre': { - 'a_pre_action_ing': 'echo pre aa', - }, - 'post': { - 'a_post_action_ing': 'echo post aa', - }, - 'a_action_ing': 'echo aa', + 'actions': { + 'pre': { + 'a_pre_action_ing': 'echo pre aa', + }, + 'post': { + 'a_post_action_ing': 'echo post aa', + }, + 'a_action_ing': 'echo aa', + } } actions_ed_file = create_yaml_keyval(actions_ed, tmp) actions_ing_file = create_yaml_keyval(actions_ing, tmp) @@ -328,7 +330,9 @@ profiles: # create the importing base config file importing_path = create_fake_config(tmp, configname=self.CONFIG_NAME, - import_configs=('config-*.yaml',), + import_configs=[ + self.CONFIG_NAME_2 + ], **importing['config']) # edit the imported config @@ -352,17 +356,28 @@ profiles: self.assertIsNotNone(imported_cfg) # test profiles - self.assertIsSubset(imported_cfg.lnk_profiles, - importing_cfg.lnk_profiles) + self.assertIsSubset(imported_cfg.profiles, + importing_cfg.profiles) # test dotfiles self.assertIsSubset(imported_cfg.dotfiles, importing_cfg.dotfiles) # test actions - self.assertIsSubset(imported_cfg.actions['pre'], - importing_cfg.actions['pre']) - self.assertIsSubset(imported_cfg.actions['post'], - importing_cfg.actions['post']) + pre_ed = post_ed = pre_ing = post_ing = {} + for k, v in imported_cfg.actions.items(): + kind, _ = v + if kind == 'pre': + pre_ed[k] = v + elif kind == 'post': + post_ed[k] = v + for k, v in importing_cfg.actions.items(): + kind, _ = v + if kind == 'pre': + pre_ing[k] = v + elif kind == 'post': + post_ing[k] = v + self.assertIsSubset(pre_ed, pre_ing) + self.assertIsSubset(post_ed, post_ing) # test transactions self.assertIsSubset(imported_cfg.trans_r, importing_cfg.trans_r) @@ -371,18 +386,18 @@ profiles: # test variables imported_vars = { k: v - for k, v in imported_cfg.get_variables(None).items() + for k, v in imported_cfg.variables.items() if not k.startswith('_') } importing_vars = { k: v - for k, v in importing_cfg.get_variables(None).items() + for k, v in importing_cfg.variables.items() if not k.startswith('_') } self.assertIsSubset(imported_vars, importing_vars) # test prodots - self.assertIsSubset(imported_cfg.prodots, importing_cfg.prodots) + self.assertIsSubset(imported_cfg.profiles, importing_cfg.profiles) def test_import_configs_override(self): """Test import_configs when some config keys overlap.""" @@ -410,22 +425,26 @@ profiles: vars_ing_file = create_yaml_keyval(vars_ing, tmp) actions_ed = { - 'pre': { - 'a_pre_action': 'echo pre 22', - }, - 'post': { - 'a_post_action': 'echo post 22', - }, - 'a_action': 'echo 22', + 'actions': { + 'pre': { + 'a_pre_action': 'echo pre 22', + }, + 'post': { + 'a_post_action': 'echo post 22', + }, + 'a_action': 'echo 22', + } } actions_ing = { - 'pre': { - 'a_pre_action': 'echo pre aa', - }, - 'post': { - 'a_post_action': 'echo post aa', - }, - 'a_action': 'echo aa', + 'actions': { + 'pre': { + 'a_pre_action': 'echo pre aa', + }, + 'post': { + 'a_post_action': 'echo post aa', + }, + 'a_action': 'echo aa', + } } actions_ed_file = create_yaml_keyval(actions_ed, tmp) actions_ing_file = create_yaml_keyval(actions_ing, tmp) @@ -542,8 +561,8 @@ profiles: self.assertIsNotNone(imported_cfg) # test profiles - self.assertIsSubset(imported_cfg.lnk_profiles, - importing_cfg.lnk_profiles) + self.assertIsSubset(imported_cfg.profiles, + importing_cfg.profiles) # test dotfiles self.assertEqual(importing_cfg.dotfiles['f_vimrc'], @@ -553,14 +572,9 @@ profiles: # test actions self.assertFalse(any( - (imported_cfg.actions['pre'][key] - == importing_cfg.actions['pre'][key]) - for key in imported_cfg.actions['pre'] - )) - self.assertFalse(any( - (imported_cfg.actions['post'][key] - == importing_cfg.actions['post'][key]) - for key in imported_cfg.actions['post'] + (imported_cfg.actions[key] + == importing_cfg.actions[key]) + for key in imported_cfg.actions )) # test transactions @@ -574,20 +588,20 @@ profiles: )) # test variables - imported_vars = imported_cfg.get_variables(None) + imported_vars = imported_cfg.variables self.assertFalse(any( imported_vars[k] == v - for k, v in importing_cfg.get_variables(None).items() + for k, v in importing_cfg.variables.items() if not k.startswith('_') )) - # test prodots - self.assertEqual(imported_cfg.prodots['host1'], - importing_cfg.prodots['host1']) - self.assertNotEqual(imported_cfg.prodots['host2'], - importing_cfg.prodots['host2']) - self.assertTrue(set(imported_cfg.prodots['host1']) - < set(importing_cfg.prodots['host2'])) + # test profiles dotfiles + self.assertEqual(imported_cfg.profiles['host1']['dotfiles'], + importing_cfg.profiles['host1']['dotfiles']) + self.assertNotEqual(imported_cfg.profiles['host2']['dotfiles'], + importing_cfg.profiles['host2']['dotfiles']) + self.assertTrue(set(imported_cfg.profiles['host1']['dotfiles']) + < set(importing_cfg.profiles['host2']['dotfiles'])) def main(): From 015874ea25753e0a59c77300dc65ded0a6fa05cf Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 31 May 2019 18:38:09 +0200 Subject: [PATCH 02/58] make it python3.4 compatible --- dotdrop/cfg_yaml.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 29c4ee2..c4a49b7 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -457,9 +457,9 @@ class CfgYaml: self.log.dbg('found: {}'.format(new)) if isinstance(current, dict) and isinstance(new, dict): # imported entries get more priority than current - current = {**current, **new} + current = self._merge_dict(new, current) elif isinstance(current, list) and isinstance(new, list): - current = [*current, *new] + current = current + new else: raise Exception('invalid import {} from {}'.format(key, path)) if self.debug: @@ -468,7 +468,11 @@ class CfgYaml: def _merge_dict(self, high, low): """merge low into high""" - return {**low, **high} + # won't work in python3.4 + # return {**low, **high} + new = low.copy() + new.update(high) + return new def _get_entry(self, yaml_dict, key, mandatory=True): """return entry from yaml dictionary""" From 3a0d0869a7bd6d6b184173f52850525e22352eff Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 31 May 2019 18:44:22 +0200 Subject: [PATCH 03/58] make it python3.4 compatible --- dotdrop/cfg_aggregator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotdrop/cfg_aggregator.py b/dotdrop/cfg_aggregator.py index 5e35bf5..3c07891 100644 --- a/dotdrop/cfg_aggregator.py +++ b/dotdrop/cfg_aggregator.py @@ -198,7 +198,7 @@ class CfgAggregator: """ dirs = self._split_path_for_key(path) prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix - key = self.key_sep.join([prefix, *dirs]) + key = self.key_sep.join([prefix] + dirs) return self._uniq_key(key, keys) def _get_short_key(self, path, keys): From 05072f2f38696627d3d26c5ee2081f8cb5e8b2a7 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 31 May 2019 18:51:43 +0200 Subject: [PATCH 04/58] make it python3.4 compatible --- dotdrop/cfg_aggregator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotdrop/cfg_aggregator.py b/dotdrop/cfg_aggregator.py index 3c07891..3370282 100644 --- a/dotdrop/cfg_aggregator.py +++ b/dotdrop/cfg_aggregator.py @@ -212,7 +212,7 @@ class CfgAggregator: entries = [] for d in dirs: entries.insert(0, d) - key = self.key_sep.join([prefix, *entries]) + key = self.key_sep.join([prefix] + entries) if key not in keys: return key return self._uniq_key(key, keys) From a90fc32c15bb3c4954c173eb8310296ab50863ca Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 31 May 2019 19:25:48 +0200 Subject: [PATCH 05/58] coverage --- tests/helpers.py | 1 + tests/test_compare.py | 3 ++- tests/test_install.py | 2 +- tests/test_yamlcfg.py | 15 ++++++++------- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index 6656a4b..e880160 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -144,6 +144,7 @@ def load_options(confpath, profile): args = _fake_args() args['--cfg'] = confpath args['--profile'] = profile + args['--debug'] = True # and get the options o = Options(args=args) o.profile = profile diff --git a/tests/test_compare.py b/tests/test_compare.py index 43553d5..7bb10be 100644 --- a/tests/test_compare.py +++ b/tests/test_compare.py @@ -29,7 +29,7 @@ class TestCompare(unittest.TestCase): def compare(self, o, tmp, nbdotfiles): dotfiles = o.dotfiles self.assertTrue(len(dotfiles) == nbdotfiles) - t = Templategen(base=o.dotpath, debug=o.debug) + t = Templategen(base=o.dotpath, debug=True) inst = Installer(create=o.create, backup=o.backup, dry=o.dry, base=o.dotpath, debug=o.debug) comp = Comparator() @@ -109,6 +109,7 @@ class TestCompare(unittest.TestCase): self.assertTrue(os.path.exists(confpath)) o = load_options(confpath, profile) o.longkey = True + o.debug = True dfiles = [d1, d2, d3, d4, d5, d9] # import the files diff --git a/tests/test_install.py b/tests/test_install.py index 80e1b4f..2c82e45 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -178,7 +178,7 @@ exec bspwm dotfiles = [d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, ddot] self.fake_config(confpath, dotfiles, profile, tmp, [act1], [tr]) - conf = Cfg(confpath, profile) + conf = Cfg(confpath, profile, debug=True) self.assertTrue(conf is not None) # install them diff --git a/tests/test_yamlcfg.py b/tests/test_yamlcfg.py index a2cd152..78f8dd8 100644 --- a/tests/test_yamlcfg.py +++ b/tests/test_yamlcfg.py @@ -38,7 +38,7 @@ class TestConfig(SubsetTestCase): dotpath=self.CONFIG_DOTPATH, backup=self.CONFIG_BACKUP, create=self.CONFIG_CREATE) - conf = Cfg(confpath) + conf = Cfg(confpath, debug=True) self.assertTrue(conf is not None) opts = conf.settings @@ -93,6 +93,7 @@ profiles: args['--profile'] = 'p1' args['--cfg'] = 'mocked' args['--link'] = cliargs + args['--verbose'] = True o = Options(args=args) self.assertTrue(o.import_link == expected) @@ -165,7 +166,7 @@ profiles: indent=2) # do the tests - conf = Cfg(confpath) + conf = Cfg(confpath, debug=True) self.assertTrue(conf is not None) # test profile @@ -196,7 +197,7 @@ profiles: indent=2) # do the tests - conf = Cfg(confpath) + conf = Cfg(confpath, debug=True) self.assertTrue(conf is not None) def test_import_configs_merge(self): @@ -350,8 +351,8 @@ profiles: }) # do the tests - importing_cfg = Cfg(importing_path) - imported_cfg = Cfg(imported_path) + importing_cfg = Cfg(importing_path, debug=True) + imported_cfg = Cfg(imported_path, debug=True) self.assertIsNotNone(importing_cfg) self.assertIsNotNone(imported_cfg) @@ -555,8 +556,8 @@ profiles: }) # do the tests - importing_cfg = Cfg(importing_path) - imported_cfg = Cfg(imported_path) + importing_cfg = Cfg(importing_path, debug=True) + imported_cfg = Cfg(imported_path, debug=True) self.assertIsNotNone(importing_cfg) self.assertIsNotNone(imported_cfg) From 6f31432f22a761832b1b3262eb44b95b9db57394 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 31 May 2019 20:15:33 +0200 Subject: [PATCH 06/58] update design doc --- CONTRIBUTING.md | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6a8f1c9..6aeaac4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,8 +15,10 @@ Dotdrop's code base is located in the [dotdrop directory](/dotdrop). Here's an overview of the different files and their role: * **action.py**: represent the actions and transformations +* **cfg_yaml.py**: the lower level config parser +* **cfg_aggregator.py**: the higher level config parser * **comparator.py**: the class handling the comparison for `compare` -* **config.py**: the config file (*config.yaml*) parser +* **dictparser.py**: abstract class for parsing dictionaries * **dotdrop.py**: the entry point and where the different cli commands are executed * **dotfile.py**: represent a dotfile * **installer.py**: the class handling the installation of dotfile for `install` @@ -24,10 +26,54 @@ Here's an overview of the different files and their role: * **linktypes.py**: enum for the three types of linking (none, symlink, children) * **logger.py**: the custom logger * **options.py**: the class embedding all the different options across dotdrop +* **profile.py**: represent a profile +* **settings.py**: represent the config settings * **templategen.py**: the jinja2 templating class * **updater.py**: the class handling the update of dotfiles for `update` * **utils.py**: some useful methods +## Config parsing + +The configuration file (yaml) is parsed in two layers: + + * the lower layer in `cfg_yaml.py` + * the higher layer in `cfg_aggregator.py` + +Only the higher layer is accessible to other classes of dotdrop. + +The lower layer part is only taking care of basic types and +does the following: + * normalize all config entries + * resolve paths (dotfiles src, dotpath, etc) + * refactor actions to a common format + * etc + * import any data from external files (configs, variables, etc) + * apply variable substitutions + * complete any data if needed (add the "profile" variable, etc) + * execute intrepreted variables through the shell + * write new entries (dotfile, profile) into the dictionary and save it to a file + * fix any deprecated entries (link_by_default, etc) + * clear empty entries + +In the end it makes sure the dictionary (or parts of it) accessed +by the higher layer is clean and normalize. + +The higher layer will transform the dictionary parsed by the lower layer +into objects (profiles, dotfiles, actions, etc). +The higher layer has no notion of inclusion (profile included for example) or +file importing (import actions, etc) or even interpreted variables +(it only sees variables that have already been interpreted). + +It does the following: + * transform dictionaries into objects + * patch list of keys with its corresponding object (for example dotfile's actions) + * provide getters for every other classes of dotdrop needing to access elements + +Note that any change to the yaml dictionary (adding a new profile or a new dotfile for +example) won't be *seen* by the higher layer until the config is reloaded. Consider the +`dirty` flag as a sign the file needs to be written and its representation in higher +levels in not accurate anymore. + # Testing Dotdrop is tested with the use of the [tests.sh](/tests.sh) script. From 806b4690b25d2f20eabafd333dc3805b6a96c068 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 31 May 2019 20:20:25 +0200 Subject: [PATCH 07/58] coverage --- tests/helpers.py | 6 ++---- tests/test_import.py | 1 + tests/test_update.py | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index e880160..5ebc6c1 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -13,7 +13,7 @@ from unittest import TestCase import yaml -from dotdrop.options import Options, ENV_NODEBUG +from dotdrop.options import Options from dotdrop.linktypes import LinkTypes from dotdrop.utils import strip_home @@ -144,7 +144,7 @@ def load_options(confpath, profile): args = _fake_args() args['--cfg'] = confpath args['--profile'] = profile - args['--debug'] = True + args['--verbose'] = True # and get the options o = Options(args=args) o.profile = profile @@ -154,8 +154,6 @@ def load_options(confpath, profile): o.import_link = LinkTypes.NOLINK o.install_showdiff = True o.debug = True - if ENV_NODEBUG in os.environ: - o.debug = False o.compare_dopts = '' o.variables = {} return o diff --git a/tests/test_import.py b/tests/test_import.py index 25bb861..98efdea 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -194,6 +194,7 @@ class TestImport(unittest.TestCase): edit_content(dotfile1, editcontent) o.safe = False o.update_path = [dotfile1] + o.debug = True cmd_update(o) c2 = open(indt1, 'r').read() self.assertTrue(editcontent == c2) diff --git a/tests/test_update.py b/tests/test_update.py index 2fdee34..885fbd6 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -105,6 +105,7 @@ class TestUpdate(unittest.TestCase): o = load_options(confpath, profile) o.safe = False o.update_showpatch = True + o.debug = True trans = Transform('trans', 'cp -r {0} {1}') d3tb = os.path.basename(d3t) for dotfile in o.dotfiles: From 8f8c39ce9c76b994151767263e9647493cc56fc7 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 31 May 2019 20:31:24 +0200 Subject: [PATCH 08/58] add src=key for dotfile if not present (for #150) --- dotdrop/cfg_yaml.py | 23 ++++++++-- tests-ng/dotfile-no-src.sh | 94 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 4 deletions(-) create mode 100755 tests-ng/dotfile-no-src.sh diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index c4a49b7..3977864 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -103,6 +103,7 @@ class CfgYaml: # dotfiles self.dotfiles = self._get_entry(self.yaml_dict, self.key_dotfiles) + self.dotfiles = self._norm_dotfiles(self.dotfiles) if self.debug: self.log.dbg('dotfiles: {}'.format(self.dotfiles)) @@ -114,7 +115,7 @@ class CfgYaml: # actions self.actions = self._get_entry(self.yaml_dict, self.key_actions, mandatory=False) - self.actions = self._patch_actions(self.actions) + self.actions = self._norm_actions(self.actions) if self.debug: self.log.dbg('actions: {}'.format(self.actions)) @@ -246,7 +247,7 @@ class CfgYaml: return allvars - def _patch_actions(self, actions): + def _norm_actions(self, actions): """ ensure each action is either pre or post explicitely action entry of the form {action_key: (pre|post, action)} @@ -262,6 +263,19 @@ class CfgYaml: new[k] = (self.action_pre, v) return new + def _norm_dotfiles(self, dotfiles): + """add 'src' as 'key if not present""" + if not dotfiles: + return dotfiles + new = {} + for k, v in dotfiles.items(): + if self.key_dotfile_src not in v: + v[self.key_dotfile_src] = k + new[k] = v + else: + new[k] = v + return new + def _get_variables_dict(self, profile, seen, sub=False): """return enriched variables""" variables = {} @@ -352,7 +366,7 @@ class CfgYaml: self.log.dbg('import actions from {}'.format(path)) self.actions = self._import_sub(path, self.key_actions, self.actions, mandatory=False, - patch_func=self._patch_actions) + patch_func=self._norm_actions) # profiles -> import for k, v in self.profiles.items(): @@ -365,7 +379,8 @@ class CfgYaml: current = v.get(self.key_dotfiles, []) path = self.resolve_path(p) current = self._import_sub(path, self.key_dotfiles, - current, mandatory=False) + current, mandatory=False, + path_func=self._norm_dotfiles) v[self.key_dotfiles] = current def _resolve_import_configs(self): diff --git a/tests-ng/dotfile-no-src.sh b/tests-ng/dotfile-no-src.sh new file mode 100755 index 0000000..4e3ffa3 --- /dev/null +++ b/tests-ng/dotfile-no-src.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2019, deadc0de6 +# +# test dotfiles with no 'src' +# returns 1 in case of error +# + +# exit on first error +set -e +#set -v + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "\e[96m\e[1m==> RUNNING $(basename $BASH_SOURCE) <==\e[0m" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests'` +mkdir -p ${tmps}/dotfiles +echo "dotfiles source (dotpath): ${tmps}" +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests'` +echo "dotfiles destination: ${tmpd}" + +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: + abc: + dst: ${tmpd}/abc +profiles: + p1: + dotfiles: + - ALL +_EOF +#cat ${cfg} + +# create the dotfiles +echo "abc" > ${tmps}/dotfiles/abc + +########################### +# test install and compare +########################### + +# install +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -b -V +[ "$?" != "0" ] && exit 1 + +# checks +[ ! -e ${tmpd}/abc ] && exit 1 +grep 'abc' ${tmpd}/abc + +## CLEANING +rm -rf ${tmps} ${tmpd} ${tmpx} ${tmpy} + +echo "OK" +exit 0 From 8bd0f4a40d998c15ac11332bbda9bba607ec0145 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 31 May 2019 20:37:26 +0200 Subject: [PATCH 09/58] ensure all actions are parsed --- tests-ng/actions.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests-ng/actions.sh b/tests-ng/actions.sh index f05566d..d21ab03 100755 --- a/tests-ng/actions.sh +++ b/tests-ng/actions.sh @@ -60,8 +60,10 @@ cat > ${cfg} << _EOF actions: pre: preaction: echo 'pre' > ${tmpa}/pre + preaction2: echo 'pre2' > ${tmpa}/pre2 post: postaction: echo 'post' > ${tmpa}/post + postaction2: echo 'post2' > ${tmpa}/post2 nakedaction: echo 'naked' > ${tmpa}/naked config: backup: true @@ -75,6 +77,8 @@ dotfiles: - preaction - postaction - nakedaction + - preaction2 + - postaction2 profiles: p1: dotfiles: @@ -86,7 +90,7 @@ _EOF echo "test" > ${tmps}/dotfiles/abc # install -cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V # checks [ ! -e ${tmpa}/pre ] && exit 1 @@ -95,6 +99,10 @@ grep pre ${tmpa}/pre >/dev/null grep post ${tmpa}/post >/dev/null [ ! -e ${tmpa}/naked ] && exit 1 grep naked ${tmpa}/naked >/dev/null +[ ! -e ${tmpa}/pre2 ] && exit 1 +grep pre2 ${tmpa}/pre2 >/dev/null +[ ! -e ${tmpa}/post2 ] && exit 1 +grep post2 ${tmpa}/post2 >/dev/null ## CLEANING rm -rf ${tmps} ${tmpd} ${tmpa} From fca28aad800420ecabe88d7054dd41ca797979c8 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 31 May 2019 20:48:01 +0200 Subject: [PATCH 10/58] typo --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6aeaac4..ccc629b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,7 +56,7 @@ does the following: * clear empty entries In the end it makes sure the dictionary (or parts of it) accessed -by the higher layer is clean and normalize. +by the higher layer is clean and normalized. The higher layer will transform the dictionary parsed by the lower layer into objects (profiles, dotfiles, actions, etc). From 805f15f7a2f73230ae709f4ab2f4d6f722f0628a Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 31 May 2019 21:40:01 +0200 Subject: [PATCH 11/58] handle glob in imports --- dotdrop/cfg_yaml.py | 92 +++++++++++++++++++++++++----------- tests-ng/globs.sh | 113 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+), 27 deletions(-) create mode 100755 tests-ng/globs.sh diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 3977864..8410d51 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -7,6 +7,7 @@ handle lower level of the config file import os import yaml +import glob # local imports from dotdrop.settings import Settings @@ -85,7 +86,7 @@ class CfgYaml: allvars = self._merge_and_apply_variables() self.variables.update(allvars) # process imported configs - self._resolve_import_configs() + self._import_configs() # process other imports self._resolve_imports() # process diverse options @@ -342,31 +343,64 @@ class CfgYaml: return variables + def _is_glob(self, path): + """quick test if path is a glob""" + return '*' in path or '?' in path + + def _glob_paths(self, paths): + """glob a list of paths""" + if not isinstance(paths, list): + paths = [paths] + res = [] + for p in paths: + if not self._is_glob(p): + res.extend([p]) + continue + p = os.path.expanduser(p) + new = glob.glob(p) + if not new: + raise Exception('bad path: {}'.format(p)) + res.extend(glob.glob(p)) + return res + + def _import_variables(self, paths): + """import external variables from paths""" + if not paths: + return + paths = self._glob_paths(paths) + for p in paths: + path = self.resolve_path(p) + if self.debug: + self.log.dbg('import variables from {}'.format(path)) + self.variables = self._import_sub(path, self.key_variables, + self.variables, + mandatory=False) + self.dvariables = self._import_sub(path, self.key_dvariables, + self.dvariables, + mandatory=False) + + def _import_actions(self, paths): + """import external actions from paths""" + if not paths: + return + paths = self._glob_paths(paths) + for p in paths: + path = self.resolve_path(p) + if self.debug: + self.log.dbg('import actions from {}'.format(path)) + self.actions = self._import_sub(path, self.key_actions, + self.actions, mandatory=False, + patch_func=self._norm_actions) + def _resolve_imports(self): """handle all the imports""" # settings -> import_variables imp = self.settings.get(self.key_import_variables, None) - if imp: - for p in imp: - path = self.resolve_path(p) - if self.debug: - self.log.dbg('import variables from {}'.format(path)) - self.variables = self._import_sub(path, self.key_variables, - self.variables, - mandatory=False) - self.dvariables = self._import_sub(path, self.key_dvariables, - self.dvariables, - mandatory=False) + self._import_variables(imp) + # settings -> import_actions imp = self.settings.get(self.key_import_actions, None) - if imp: - for p in imp: - path = self.resolve_path(p) - if self.debug: - self.log.dbg('import actions from {}'.format(path)) - self.actions = self._import_sub(path, self.key_actions, - self.actions, mandatory=False, - patch_func=self._norm_actions) + self._import_actions(imp) # profiles -> import for k, v in self.profiles.items(): @@ -375,7 +409,8 @@ class CfgYaml: continue if self.debug: self.log.dbg('import dotfiles for profile {}'.format(k)) - for p in imp: + paths = self._glob_paths(imp) + for p in paths: current = v.get(self.key_dotfiles, []) path = self.resolve_path(p) current = self._import_sub(path, self.key_dotfiles, @@ -383,14 +418,15 @@ class CfgYaml: path_func=self._norm_dotfiles) v[self.key_dotfiles] = current - def _resolve_import_configs(self): - """resolve import_configs""" + def _import_configs(self): + """import configs from external file""" # settings -> import_configs imp = self.settings.get(self.key_import_configs, None) if not imp: return - for p in imp: - path = self.resolve_path(p) + paths = self._glob_paths(imp) + for path in paths: + path = self.resolve_path(path) if self.debug: self.log.dbg('import config from {}'.format(path)) sub = CfgYaml(path, debug=self.debug) @@ -400,8 +436,10 @@ class CfgYaml: self.actions = self._merge_dict(self.actions, sub.actions) self.trans_r = self._merge_dict(self.trans_r, sub.trans_r) self.trans_w = self._merge_dict(self.trans_w, sub.trans_w) - self.variables = self._merge_dict(self.variables, sub.variables) - self.dvariables = self._merge_dict(self.dvariables, sub.dvariables) + self.variables = self._merge_dict(self.variables, + sub.variables) + self.dvariables = self._merge_dict(self.dvariables, + sub.dvariables) def _resolve_rest(self): """resolve some other parts of the config""" diff --git a/tests-ng/globs.sh b/tests-ng/globs.sh new file mode 100755 index 0000000..f8b1925 --- /dev/null +++ b/tests-ng/globs.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2019, deadc0de6 +# +# ensure imports allow globs +# - import_actions +# - import_configs +# - import_variables +# - profile import +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "\e[96m\e[1m==> RUNNING $(basename $BASH_SOURCE) <==\e[0m" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests'` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests'` +# temporary +tmpa=`mktemp -d --suffix='-dotdrop-tests'` + +########### +# test globs in import_actions +########### +# create the action files +actionsd="${tmps}/actions" +mkdir -p ${actionsd} +cat > ${actionsd}/action1.yaml << _EOF +actions: + fromaction1: echo "fromaction1" > ${tmpa}/fromaction1 +_EOF +cat > ${actionsd}/action2.yaml << _EOF +actions: + fromaction2: echo "fromaction2" > ${tmpa}/fromaction2 +_EOF + +cfg="${tmps}/config.yaml" +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + import_actions: + - ${actionsd}/* +dotfiles: + f_abc: + dst: ${tmpd}/abc + src: abc + actions: + - fromaction1 + - fromaction2 +profiles: + p1: + dotfiles: + - f_abc +_EOF + +# create the source +mkdir -p ${tmps}/dotfiles/ +echo "abc" > ${tmps}/dotfiles/abc + +# install +cd ${ddpath} | ${bin} install -c ${cfg} -p p1 -V + +# checks +[ ! -e ${tmpd}/abc ] && echo "dotfile not installed" && exit 1 +[ ! -e ${tmpa}/fromaction1 ] && echo "action1 not executed" && exit 1 +grep fromaction1 ${tmpa}/fromaction1 +[ ! -e ${tmpa}/fromaction2 ] && echo "action2 not executed" && exit 1 +grep fromaction2 ${tmpa}/fromaction2 + +## CLEANING +rm -rf ${tmps} ${tmpd} ${tmpa} + +echo "OK" +exit 0 From 27e6a0da587779e7db9b1df25a935d1c6261e724 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 11:26:30 +0200 Subject: [PATCH 12/58] refactor resolve_path --- dotdrop/cfg_aggregator.py | 1 - dotdrop/cfg_yaml.py | 25 +++++++++++++++---------- dotdrop/settings.py | 5 ----- tests/test_yamlcfg.py | 3 ++- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/dotdrop/cfg_aggregator.py b/dotdrop/cfg_aggregator.py index 3370282..3134c30 100644 --- a/dotdrop/cfg_aggregator.py +++ b/dotdrop/cfg_aggregator.py @@ -46,7 +46,6 @@ class CfgAggregator: # settings self.settings = Settings.parse(None, self.cfgyaml.settings) - self.settings.resolve_paths(self.cfgyaml.resolve_path) if self.debug: self.log.dbg('settings: {}'.format(self.settings)) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 8410d51..b0314a3 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -99,6 +99,11 @@ class CfgYaml: self.ori_settings = self._get_entry(self.yaml_dict, self.key_settings) self.settings = Settings(None).serialize().get(self.key_settings) self.settings.update(self.ori_settings) + # resolve settings paths + p = self._resolve_path(self.settings[self.key_settings_dotpath]) + self.settings[self.key_settings_dotpath] = p + p = self._resolve_path(self.settings[self.key_settings_workdir]) + self.settings[self.key_settings_workdir] = p if self.debug: self.log.dbg('settings: {}'.format(self.settings)) @@ -149,9 +154,9 @@ class CfgYaml: for dotfile in self.dotfiles.values(): src = dotfile[self.key_dotfile_src] src = os.path.join(self.settings[self.key_settings_dotpath], src) - dotfile[self.key_dotfile_src] = self.resolve_path(src) + dotfile[self.key_dotfile_src] = self._resolve_path(src) dst = dotfile[self.key_dotfile_dst] - dotfile[self.key_dotfile_dst] = self.resolve_path(dst) + dotfile[self.key_dotfile_dst] = self._resolve_path(dst) def _merge_and_apply_variables(self): """ @@ -286,11 +291,11 @@ class CfgYaml: variables['profile'] = profile # add some more variables p = self.settings.get(self.key_settings_dotpath) - p = self.resolve_path(p) + p = self._resolve_path(p) variables['_dotdrop_dotpath'] = p - variables['_dotdrop_cfgpath'] = self.resolve_path(self.path) + variables['_dotdrop_cfgpath'] = self._resolve_path(self.path) p = self.settings.get(self.key_settings_workdir) - p = self.resolve_path(p) + p = self._resolve_path(p) variables['_dotdrop_workdir'] = p # variables @@ -369,7 +374,7 @@ class CfgYaml: return paths = self._glob_paths(paths) for p in paths: - path = self.resolve_path(p) + path = self._resolve_path(p) if self.debug: self.log.dbg('import variables from {}'.format(path)) self.variables = self._import_sub(path, self.key_variables, @@ -385,7 +390,7 @@ class CfgYaml: return paths = self._glob_paths(paths) for p in paths: - path = self.resolve_path(p) + path = self._resolve_path(p) if self.debug: self.log.dbg('import actions from {}'.format(path)) self.actions = self._import_sub(path, self.key_actions, @@ -412,7 +417,7 @@ class CfgYaml: paths = self._glob_paths(imp) for p in paths: current = v.get(self.key_dotfiles, []) - path = self.resolve_path(p) + path = self._resolve_path(p) current = self._import_sub(path, self.key_dotfiles, current, mandatory=False, path_func=self._norm_dotfiles) @@ -426,7 +431,7 @@ class CfgYaml: return paths = self._glob_paths(imp) for path in paths: - path = self.resolve_path(path) + path = self._resolve_path(path) if self.debug: self.log.dbg('import config from {}'.format(path)) sub = CfgYaml(path, debug=self.debug) @@ -481,7 +486,7 @@ class CfgYaml: values[self.key_profiles_dotfiles] = list(set(current)) return values.get(self.key_profiles_dotfiles, []) - def resolve_path(self, path): + def _resolve_path(self, path): """resolve a path either absolute or relative to config path""" path = os.path.expanduser(path) if not os.path.isabs(path): diff --git a/dotdrop/settings.py b/dotdrop/settings.py index 1d2a5dc..51ef59f 100644 --- a/dotdrop/settings.py +++ b/dotdrop/settings.py @@ -60,11 +60,6 @@ class Settings(DictParser): self.link_dotfile_default = LinkTypes.get(link_dotfile_default) self.link_on_import = LinkTypes.get(link_on_import) - def resolve_paths(self, resolver): - """resolve path using resolver function""" - self.dotpath = resolver(self.dotpath) - self.workdir = resolver(self.workdir) - def _serialize_seq(self, name, dic): """serialize attribute 'name' into 'dic'""" seq = getattr(self, name) diff --git a/tests/test_yamlcfg.py b/tests/test_yamlcfg.py index 78f8dd8..0a22420 100644 --- a/tests/test_yamlcfg.py +++ b/tests/test_yamlcfg.py @@ -46,7 +46,8 @@ class TestConfig(SubsetTestCase): self.assertTrue(opts != {}) self.assertTrue(opts['backup'] == self.CONFIG_BACKUP) self.assertTrue(opts['create'] == self.CONFIG_CREATE) - self.assertTrue(opts['dotpath'] == self.CONFIG_DOTPATH) + dpath = os.path.basename(opts['dotpath']) + self.assertTrue(dpath == self.CONFIG_DOTPATH) self.assertTrue(conf.dump() != '') def test_def_link(self): From a90995d42711bec2f90896e7141e5007559e4f61 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 11:34:29 +0200 Subject: [PATCH 13/58] rename trans to trans_read --- dotdrop/cfg_yaml.py | 9 ++++++++- tests/test_import.py | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index b0314a3..1088d53 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -24,7 +24,8 @@ class CfgYaml: key_dotfiles = 'dotfiles' key_profiles = 'profiles' key_actions = 'actions' - key_trans_r = 'trans' + old_key_trans_r = 'trans' + key_trans_r = 'trans_read' key_trans_w = 'trans_write' key_variables = 'variables' key_dvariables = 'dynvariables' @@ -126,6 +127,12 @@ class CfgYaml: self.log.dbg('actions: {}'.format(self.actions)) # trans_r + if self.old_key_trans_r in self.yaml_dict: + self.log.warn('\"trans\" is deprecated, please use \"trans_read\"') + self.yaml_dict[self.key_trans_r] = self.yaml_dict.pop( + self.old_key_trans_r + ) + self.dirty = True self.trans_r = self._get_entry(self.yaml_dict, self.key_trans_r, mandatory=False) if self.debug: diff --git a/tests/test_import.py b/tests/test_import.py index 98efdea..bbbfb75 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -352,7 +352,7 @@ class TestImport(unittest.TestCase): self.assertFalse(any(a.endswith('ing') for a in actions)) # testing transformations - transformations = y['trans'].keys() + transformations = y['trans_read'].keys() self.assertTrue(all(t.endswith('ed') for t in transformations)) self.assertFalse(any(t.endswith('ing') for t in transformations)) transformations = y['trans_write'].keys() @@ -394,7 +394,7 @@ class TestImport(unittest.TestCase): self.assertFalse(any(action.endswith('ed') for action in actions)) # testing transformations - transformations = y['trans'].keys() + transformations = y['trans_read'].keys() self.assertTrue(all(t.endswith('ing') for t in transformations)) self.assertFalse(any(t.endswith('ed') for t in transformations)) transformations = y['trans_write'].keys() From 44ed1a4d1f5209a84f88e3ce38f1a346f5ffd2a9 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 11:43:20 +0200 Subject: [PATCH 14/58] ensure only one trans_r and one trans_w --- dotdrop/dotfile.py | 4 ++++ tests/test_install.py | 9 +++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/dotdrop/dotfile.py b/dotdrop/dotfile.py index 8c4ab24..7f3f5c3 100644 --- a/dotdrop/dotfile.py +++ b/dotdrop/dotfile.py @@ -42,7 +42,11 @@ class Dotfile(DictParser): self.noempty = noempty self.src = src self.trans_r = trans_r + if trans_r and len(self.trans_r) > 1: + raise Exception('only one trans_read allowed') self.trans_w = trans_w + if trans_w and len(self.trans_w) > 1: + raise Exception('only one trans_write allowed') self.upignore = upignore def get_dotfile_variables(self): diff --git a/tests/test_install.py b/tests/test_install.py index 2c82e45..988ae3a 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -47,8 +47,8 @@ exec bspwm for action in actions: f.write(' {}: {}\n'.format(action.key, action.action)) f.write('trans:\n') - for action in trans: - f.write(' {}: {}\n'.format(action.key, action.action)) + for tr in trans: + f.write(' {}: {}\n'.format(tr.key, tr.action)) f.write('config:\n') f.write(' backup: true\n') f.write(' create: true\n') @@ -64,7 +64,8 @@ exec bspwm for action in d.actions: f.write(' - {}\n'.format(action.key)) if d.trans_r: - f.write(' trans: {}\n'.format(d.trans_r.key)) + for tr in d.trans_r: + f.write(' trans: {}\n'.format(tr.key)) f.write('profiles:\n') f.write(' {}:\n'.format(profile)) f.write(' dotfiles:\n') @@ -165,7 +166,7 @@ exec bspwm tr = Action('testtrans', 'post', cmd) f9, c9 = create_random_file(tmp, content=trans1) dst9 = os.path.join(dst, get_string(6)) - d9 = Dotfile(get_string(6), dst9, os.path.basename(f9), trans_r=tr) + d9 = Dotfile(get_string(6), dst9, os.path.basename(f9), trans_r=[tr]) # to test template f10, _ = create_random_file(tmp, content='{{@@ header() @@}}') From 239f80222818a5cbecbe697ba6abbcf26567708f Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 11:57:57 +0200 Subject: [PATCH 15/58] disable trans_{r,w} when linked --- dotdrop/dotfile.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dotdrop/dotfile.py b/dotdrop/dotfile.py index 7f3f5c3..ea6a384 100644 --- a/dotdrop/dotfile.py +++ b/dotdrop/dotfile.py @@ -49,6 +49,18 @@ class Dotfile(DictParser): raise Exception('only one trans_write allowed') self.upignore = upignore + if link != LinkTypes.NOLINK and \ + ( + (trans_r and len(trans_r) > 0) + or + (trans_w and len(trans_w) > 0) + ): + msg = '[{}] transformations disabled'.format(key) + msg += ' because dotfile is linked' + self.log.warn(msg) + trans_r = [] + trans_w = [] + def get_dotfile_variables(self): """return this dotfile specific variables""" return { From 936f9190625d0863168c5023c09581529f9276a8 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 12:01:09 +0200 Subject: [PATCH 16/58] refactor --- dotdrop/cfg_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 1088d53..cb7267b 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -366,7 +366,7 @@ class CfgYaml: res = [] for p in paths: if not self._is_glob(p): - res.extend([p]) + res.append(p) continue p = os.path.expanduser(p) new = glob.glob(p) From 8af57adfab2443c4c310e39e0b16fb7fbd643fd5 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 12:32:58 +0200 Subject: [PATCH 17/58] adding --force-action for #149 --- completion/_dotdrop-completion.zsh | 2 + completion/_dotdrop.sh-completion.zsh | 2 + completion/dotdrop-completion.bash | 2 +- completion/dotdrop.sh-completion.bash | 2 +- dotdrop/dotdrop.py | 16 +++- dotdrop/options.py | 20 +++-- tests-ng/force-actions.sh | 123 ++++++++++++++++++++++++++ 7 files changed, 154 insertions(+), 13 deletions(-) mode change 100755 => 100644 completion/dotdrop-completion.bash create mode 100755 tests-ng/force-actions.sh diff --git a/completion/_dotdrop-completion.zsh b/completion/_dotdrop-completion.zsh index d70dbd8..2f694da 100644 --- a/completion/_dotdrop-completion.zsh +++ b/completion/_dotdrop-completion.zsh @@ -96,6 +96,8 @@ _dotdrop-install () '(--dry)--dry' \ '(-D)-D' \ '(--showdiff)--showdiff' \ + '(-a)-a' \ + '(--force-action)--force-action' \ '(-c=-)-c=-' \ '(--cfg=-)--cfg=-' \ '(-p=-)-p=-' \ diff --git a/completion/_dotdrop.sh-completion.zsh b/completion/_dotdrop.sh-completion.zsh index bee9bb7..3357e75 100644 --- a/completion/_dotdrop.sh-completion.zsh +++ b/completion/_dotdrop.sh-completion.zsh @@ -96,6 +96,8 @@ _dotdrop.sh-install () '(--dry)--dry' \ '(-D)-D' \ '(--showdiff)--showdiff' \ + '(-a)-a' \ + '(--force-action)--force-action' \ '(-c=-)-c=-' \ '(--cfg=-)--cfg=-' \ '(-p=-)-p=-' \ diff --git a/completion/dotdrop-completion.bash b/completion/dotdrop-completion.bash old mode 100755 new mode 100644 index ce4e511..261a2be --- a/completion/dotdrop-completion.bash +++ b/completion/dotdrop-completion.bash @@ -40,7 +40,7 @@ _dotdrop_install() cur="${COMP_WORDS[COMP_CWORD]}" if [ $COMP_CWORD -ge 2 ]; then - COMPREPLY=( $( compgen -fW '-V --verbose -b --no-banner -t --temp -f --force -n --nodiff -d --dry -D --showdiff -c= --cfg= -p= --profile= ' -- $cur) ) + COMPREPLY=( $( compgen -fW '-V --verbose -b --no-banner -t --temp -f --force -n --nodiff -d --dry -D --showdiff -a --force-action -c= --cfg= -p= --profile= ' -- $cur) ) fi } diff --git a/completion/dotdrop.sh-completion.bash b/completion/dotdrop.sh-completion.bash index 5357e11..98ece6c 100644 --- a/completion/dotdrop.sh-completion.bash +++ b/completion/dotdrop.sh-completion.bash @@ -40,7 +40,7 @@ _dotdropsh_install() cur="${COMP_WORDS[COMP_CWORD]}" if [ $COMP_CWORD -ge 2 ]; then - COMPREPLY=( $( compgen -fW '-V --verbose -b --no-banner -t --temp -f --force -n --nodiff -d --dry -D --showdiff -c= --cfg= -p= --profile= ' -- $cur) ) + COMPREPLY=( $( compgen -fW '-V --verbose -b --no-banner -t --temp -f --force -n --nodiff -d --dry -D --showdiff -a --force-action -c= --cfg= -p= --profile= ' -- $cur) ) fi } diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index b734eae..497caa0 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -129,6 +129,7 @@ def cmd_install(o): if os.path.exists(tmp): remove(tmp) if r: + # dotfile was installed if not o.install_temporary: defactions = o.install_default_actions_post postactions = dotfile.get_post_actions() @@ -136,8 +137,19 @@ def cmd_install(o): defactions, t, post=True) post_actions_exec() installed += 1 - elif not r and err: - LOG.err('installing \"{}\" failed: {}'.format(dotfile.key, err)) + elif not r: + # dotfile was NOT installed + if o.install_force_action: + LOG.dbg('force pre action execution ...') + pre_actions_exec() + LOG.dbg('force post pre action execution ...') + postactions = dotfile.get_post_actions() + post_actions_exec = action_executor(o, dotfile, postactions, + defactions, t, post=True) + post_actions_exec() + if err: + LOG.err('installing \"{}\" failed: {}'.format(dotfile.key, + err)) if o.install_temporary: LOG.log('\ninstalled to tmp \"{}\".'.format(tmpdir)) LOG.log('\n{} dotfile(s) installed.'.format(installed)) diff --git a/dotdrop/options.py b/dotdrop/options.py index 5771789..97683df 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -50,15 +50,15 @@ USAGE = """ {} Usage: - dotdrop install [-VbtfndD] [-c ] [-p ] [...] - dotdrop import [-Vbd] [-c ] [-p ] [-l ] ... - dotdrop compare [-Vb] [-c ] [-p ] - [-o ] [-C ...] [-i ...] - dotdrop update [-VbfdkP] [-c ] [-p ] - [-i ...] [...] - dotdrop listfiles [-VbT] [-c ] [-p ] - dotdrop detail [-Vb] [-c ] [-p ] [...] - dotdrop list [-Vb] [-c ] + dotdrop install [-VbtfndDa] [-c ] [-p ] [...] + dotdrop import [-Vbd] [-c ] [-p ] [-l ] ... + dotdrop compare [-Vb] [-c ] [-p ] + [-o ] [-C ...] [-i ...] + dotdrop update [-VbfdkP] [-c ] [-p ] + [-i ...] [...] + dotdrop listfiles [-VbT] [-c ] [-p ] + dotdrop detail [-Vb] [-c ] [-p ] [...] + dotdrop list [-Vb] [-c ] dotdrop --help dotdrop --version @@ -75,6 +75,7 @@ Options: -D --showdiff Show a diff before overwriting. -P --show-patch Provide a one-liner to manually patch template. -f --force Do not ask user confirmation for anything. + -a --force-action Execute all actions even if no dotfile is installed. -k --key Treat as a dotfile key. -V --verbose Be verbose. -d --dry Dry run. @@ -209,6 +210,7 @@ class Options(AttrMonitor): # "listfiles" specifics self.listfiles_templateonly = self.args['--template'] # "install" specifics + self.install_force_action = self.args['--force-action'] self.install_temporary = self.args['--temp'] self.install_keys = self.args[''] self.install_diff = not self.args['--nodiff'] diff --git a/tests-ng/force-actions.sh b/tests-ng/force-actions.sh new file mode 100755 index 0000000..363afc6 --- /dev/null +++ b/tests-ng/force-actions.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2019, deadc0de6 +# +# force actions +# returns 1 in case of error +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "\e[96m\e[1m==> RUNNING $(basename $BASH_SOURCE) <==\e[0m" + +################################################################ +# this is the test +################################################################ + +# the action temp +tmpa=`mktemp -d --suffix='-dotdrop-tests'` +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests'` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests'` + +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +actions: + pre: + preaction: echo 'pre' > ${tmpa}/pre + preaction2: echo 'pre2' > ${tmpa}/pre2 + post: + postaction: echo 'post' > ${tmpa}/post + postaction2: echo 'post2' > ${tmpa}/post2 + nakedaction: echo 'naked' > ${tmpa}/naked +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: + f_abc: + dst: ${tmpd}/abc + src: abc + actions: + - preaction + - postaction + - nakedaction + - preaction2 + - postaction2 +profiles: + p1: + dotfiles: + - f_abc +_EOF +#cat ${cfg} + +# create the dotfile +echo "test" > ${tmps}/dotfiles/abc +# deploy the dotfile +cp ${tmps}/dotfiles/abc ${tmpd}/abc + +# install +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V + +# checks +[ -e ${tmpa}/pre ] && exit 1 +[ -e ${tmpa}/post ] && exit 1 +[ -e ${tmpa}/naked ] && exit 1 +[ -e ${tmpa}/pre2 ] && exit 1 +[ -e ${tmpa}/post2 ] && exit 1 + +# install and force +cd ${ddpath} | ${bin} install -f -a -c ${cfg} -p p1 -V + +# checks +[ ! -e ${tmpa}/pre ] && exit 1 +grep pre ${tmpa}/pre >/dev/null +[ ! -e ${tmpa}/post ] && exit 1 +grep post ${tmpa}/post >/dev/null +[ ! -e ${tmpa}/naked ] && exit 1 +grep naked ${tmpa}/naked >/dev/null +[ ! -e ${tmpa}/pre2 ] && exit 1 +grep pre2 ${tmpa}/pre2 >/dev/null +[ ! -e ${tmpa}/post2 ] && exit 1 +grep post2 ${tmpa}/post2 >/dev/null + +## CLEANING +rm -rf ${tmps} ${tmpd} ${tmpa} + +echo "OK" +exit 0 From bebe6f5eaee2d938721eabbc26582b73b0ee1934 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 13:31:36 +0200 Subject: [PATCH 18/58] keep order of included profiles for #149 --- dotdrop/cfg_yaml.py | 7 +- dotdrop/comparator.py | 2 +- dotdrop/dotdrop.py | 8 ++- dotdrop/utils.py | 9 +++ tests-ng/include-order.sh | 142 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 162 insertions(+), 6 deletions(-) create mode 100755 tests-ng/include-order.sh diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index cb7267b..4199391 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -14,7 +14,7 @@ from dotdrop.settings import Settings from dotdrop.logger import Logger from dotdrop.templategen import Templategen from dotdrop.linktypes import LinkTypes -from dotdrop.utils import shell +from dotdrop.utils import shell, uniq_list class CfgYaml: @@ -490,7 +490,10 @@ class CfgYaml: others.extend(self._rec_resolve_profile_include(i)) current.extend(others) # unique them - values[self.key_profiles_dotfiles] = list(set(current)) + values[self.key_profiles_dotfiles] = uniq_list(current) + if self.debug: + dfs = values[self.key_profiles_dotfiles] + self.log.dbg('profile dfs after include: {}'.format(dfs)) return values.get(self.key_profiles_dotfiles, []) def _resolve_path(self, path): diff --git a/dotdrop/comparator.py b/dotdrop/comparator.py index 66a0ae2..992674d 100644 --- a/dotdrop/comparator.py +++ b/dotdrop/comparator.py @@ -95,7 +95,7 @@ class Comparator: # content is different funny = comp.diff_files funny.extend(comp.funny_files) - funny = list(set(funny)) + funny = utils.uniq_list(funny) for i in funny: lfile = os.path.join(left, i) rfile = os.path.join(right, i) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 497caa0..d90eab3 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -15,7 +15,7 @@ from dotdrop.templategen import Templategen from dotdrop.installer import Installer from dotdrop.updater import Updater from dotdrop.comparator import Comparator -from dotdrop.utils import get_tmpdir, remove, strip_home, run +from dotdrop.utils import get_tmpdir, remove, strip_home, run, uniq_list from dotdrop.linktypes import LinkTypes LOG = Logger() @@ -71,7 +71,8 @@ def cmd_install(o): dotfiles = o.dotfiles if o.install_keys: # filtered dotfiles to install - dotfiles = [d for d in dotfiles if d.key in set(o.install_keys)] + uniq = uniq_list(o.install_keys) + dotfiles = [d for d in dotfiles if d.key in uniq] if not dotfiles: msg = 'no dotfile to install for this profile (\"{}\")' LOG.warn(msg.format(o.profile)) @@ -388,7 +389,8 @@ def cmd_detail(o): dotfiles = o.dotfiles if o.detail_keys: # filtered dotfiles to install - dotfiles = [d for d in dotfiles if d.key in set(o.details_keys)] + uniq = uniq_list(o.details_keys) + dotfiles = [d for d in dotfiles if d.key in uniq] LOG.emph('dotfiles details for profile \"{}\":\n'.format(o.profile)) for d in dotfiles: _detail(o.dotpath, d) diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 88dbf75..2954a96 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -130,3 +130,12 @@ def must_ignore(paths, ignores, debug=False): LOG.dbg('ignore \"{}\" match: {}'.format(i, p)) return True return False + + +def uniq_list(a_list): + """unique elements of a list while preserving order""" + new = [] + for a in a_list: + if a not in new: + new.append(a) + return new diff --git a/tests-ng/include-order.sh b/tests-ng/include-order.sh new file mode 100755 index 0000000..eca2187 --- /dev/null +++ b/tests-ng/include-order.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2019, deadc0de6 +# +# test the use of the keyword "include" +# that has to be ordered +# returns 1 in case of error +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "\e[96m\e[1m==> RUNNING $(basename $BASH_SOURCE) <==\e[0m" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests'` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests'` +# temporary +tmpa=`mktemp -d --suffix='-dotdrop-tests'` + +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +actions: + pre: + first: 'echo first > ${tmpa}/cookie' + second: 'echo second >> ${tmpa}/cookie' + third: 'echo third >> ${tmpa}/cookie' +dotfiles: + f_first: + dst: ${tmpd}/first + src: first + actions: + - first + f_second: + dst: ${tmpd}/second + src: second + actions: + - second + f_third: + dst: ${tmpd}/third + src: third + actions: + - third +profiles: + p0: + dotfiles: + - f_first + include: + - second + - third + second: + dotfiles: + - f_second + third: + dotfiles: + - f_third +_EOF + +# create the source +mkdir -p ${tmps}/dotfiles/ +echo "first" > ${tmps}/dotfiles/first +echo "second" > ${tmps}/dotfiles/second +echo "third" > ${tmps}/dotfiles/third + +attempts="3" +for ((i=0;i<${attempts};i++)); do + # install + cd ${ddpath} | ${bin} install -f -c ${cfg} -p p0 -V + + # checks timestamp + echo "first timestamp: `stat -c %y ${tmpd}/first`" + echo "second timestamp: `stat -c %y ${tmpd}/second`" + echo "third timestamp: `stat -c %y ${tmpd}/third`" + + ts_first=`date "+%S%N" -d "$(stat -c %y ${tmpd}/first)"` + ts_second=`date "+%S%N" -d "$(stat -c %y ${tmpd}/second)"` + ts_third=`date "+%S%N" -d "$(stat -c %y ${tmpd}/third)"` + + #echo "first ts: ${ts_first}" + #echo "second ts: ${ts_second}" + #echo "third ts: ${ts_third}" + + [ "${ts_first}" -ge "${ts_second}" ] && echo "second created before first" && exit 1 + [ "${ts_second}" -ge "${ts_third}" ] && echo "third created before second" && exit 1 + + # check cookie + cat ${tmpa}/cookie + content=`cat ${tmpa}/cookie | xargs` + [ "${content}" != "first second third" ] && echo "bad cookie" && exit 1 + + # clean + rm ${tmpa}/cookie + rm ${tmpd}/first ${tmpd}/second ${tmpd}/third +done + +## CLEANING +rm -rf ${tmps} ${tmpd} ${tmpa} + +echo "OK" +exit 0 From 58dd284118b49397a03f5f1c54ebecaad46b7df2 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 13:50:31 +0200 Subject: [PATCH 19/58] adding profile actions for #141 --- dotdrop/cfg_aggregator.py | 12 +++- dotdrop/dotdrop.py | 6 ++ dotdrop/options.py | 3 +- dotdrop/profile.py | 9 +++ tests-ng/profile-actions.sh | 118 ++++++++++++++++++++++++++++++++++++ 5 files changed, 145 insertions(+), 3 deletions(-) create mode 100755 tests-ng/profile-actions.sh diff --git a/dotdrop/cfg_aggregator.py b/dotdrop/cfg_aggregator.py index 3134c30..142d98d 100644 --- a/dotdrop/cfg_aggregator.py +++ b/dotdrop/cfg_aggregator.py @@ -83,13 +83,14 @@ class CfgAggregator: self._patch_keys_to_objs(self.profiles, "dotfiles", self.get_dotfile) - # patch action in actions + # patch action in dotfiles actions self._patch_keys_to_objs(self.dotfiles, "actions", self._get_action_w_args) + # patch action in profiles actions self._patch_keys_to_objs(self.profiles, "actions", self._get_action_w_args) - # patch default actions in settings + # patch actions in settings default_actions self._patch_keys_to_objs([self.settings], "default_actions", self._get_action_w_args) if self.debug: @@ -254,6 +255,13 @@ class CfgAggregator: """return profiles""" return self.profiles + def get_profile(self, key): + """return profile by key""" + try: + return next(x for x in self.profiles if x.key == key) + except StopIteration: + return None + def get_dotfiles(self, profile=None): """return dotfiles dict for this profile key""" if not profile: diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index d90eab3..b95cc64 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -101,6 +101,8 @@ def cmd_install(o): preactions = [] if not o.install_temporary: preactions.extend(dotfile.get_pre_actions()) + prof = o.conf.get_profile(o.profile) + preactions.extend(prof.get_pre_actions()) defactions = o.install_default_actions_pre pre_actions_exec = action_executor(o, dotfile, preactions, defactions, t, post=False) @@ -134,6 +136,8 @@ def cmd_install(o): if not o.install_temporary: defactions = o.install_default_actions_post postactions = dotfile.get_post_actions() + prof = o.conf.get_profile(o.profile) + postactions.extend(prof.get_post_actions()) post_actions_exec = action_executor(o, dotfile, postactions, defactions, t, post=True) post_actions_exec() @@ -145,6 +149,8 @@ def cmd_install(o): pre_actions_exec() LOG.dbg('force post pre action execution ...') postactions = dotfile.get_post_actions() + prof = o.conf.get_profile(o.profile) + postactions.extend(prof.get_post_actions()) post_actions_exec = action_executor(o, dotfile, postactions, defactions, t, post=True) post_actions_exec() diff --git a/dotdrop/options.py b/dotdrop/options.py index 97683df..da3def8 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -51,7 +51,8 @@ USAGE = """ Usage: dotdrop install [-VbtfndDa] [-c ] [-p ] [...] - dotdrop import [-Vbd] [-c ] [-p ] [-l ] ... + dotdrop import [-Vbd] [-c ] [-p ] + [-l ] ... dotdrop compare [-Vb] [-c ] [-p ] [-o ] [-C ...] [-i ...] dotdrop update [-VbfdkP] [-c ] [-p ] diff --git a/dotdrop/profile.py b/dotdrop/profile.py index 5a1e671..c1147ee 100644 --- a/dotdrop/profile.py +++ b/dotdrop/profile.py @@ -6,6 +6,7 @@ represent a profile in dotdrop """ from dotdrop.dictparser import DictParser +from dotdrop.action import Action class Profile(DictParser): @@ -27,6 +28,14 @@ class Profile(DictParser): self.dotfiles = dotfiles self.variables = variables + def get_pre_actions(self): + """return all 'pre' actions""" + return [a for a in self.actions if a.kind == Action.pre] + + def get_post_actions(self): + """return all 'post' actions""" + return [a for a in self.actions if a.kind == Action.post] + @classmethod def _adjust_yaml_keys(cls, value): """patch dict""" diff --git a/tests-ng/profile-actions.sh b/tests-ng/profile-actions.sh new file mode 100755 index 0000000..88d124a --- /dev/null +++ b/tests-ng/profile-actions.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2019, deadc0de6 +# +# test actions per profile +# returns 1 in case of error +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "\e[96m\e[1m==> RUNNING $(basename $BASH_SOURCE) <==\e[0m" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests'` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests'` +#echo "dotfile destination: ${tmpd}" +# the action temp +tmpa=`mktemp -d --suffix='-dotdrop-tests'` + +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +actions: + pre: + preaction: echo 'pre' >> ${tmpa}/pre + preaction2: echo 'pre2' >> ${tmpa}/pre2 + post: + postaction: echo 'post' >> ${tmpa}/post + postaction2: echo 'post2' >> ${tmpa}/post2 + nakedaction: echo 'naked' >> ${tmpa}/naked +dotfiles: + f_abc: + dst: ${tmpd}/abc + src: abc +profiles: + p0: + actions: + - preaction2 + - postaction2 + - nakedaction + dotfiles: + - f_abc +_EOF +#cat ${cfg} + +# create the dotfile +echo "test" > ${tmps}/dotfiles/abc + +# install +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p0 -V + +# check actions executed +[ ! -e ${tmpa}/pre2 ] && echo 'action not executed' && exit 1 +[ ! -e ${tmpa}/post2 ] && echo 'action not executed' && exit 1 +[ ! -e ${tmpa}/naked ] && echo 'action not executed' && exit 1 + +grep pre2 ${tmpa}/pre2 +grep post2 ${tmpa}/post2 +grep naked ${tmpa}/naked + +# install again +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p0 -V + +# check actions not executed twice +nb=`wc -l ${tmpa}/pre2 | awk '{print $1}'` +[ "${nb}" -gt "1" ] && echo "action executed twice" && exit 1 +nb=`wc -l ${tmpa}/post2 | awk '{print $1}'` +[ "${nb}" -gt "1" ] && echo "action executed twice" && exit 1 +nb=`wc -l ${tmpa}/naked | awk '{print $1}'` +[ "${nb}" -gt "1" ] && echo "action executed twice" && exit 1 + +## CLEANING +rm -rf ${tmps} ${tmpd} ${tmpa} + +echo "OK" +exit 0 From 44db88ce747f4ef671421da1de5b0b0501bb3f20 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 13:53:00 +0200 Subject: [PATCH 20/58] fix --force-actions in tests --- dotdrop/options.py | 4 ++-- tests/helpers.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dotdrop/options.py b/dotdrop/options.py index da3def8..8e69670 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -76,7 +76,7 @@ Options: -D --showdiff Show a diff before overwriting. -P --show-patch Provide a one-liner to manually patch template. -f --force Do not ask user confirmation for anything. - -a --force-action Execute all actions even if no dotfile is installed. + -a --force-actions Execute all actions even if no dotfile is installed. -k --key Treat as a dotfile key. -V --verbose Be verbose. -d --dry Dry run. @@ -211,7 +211,7 @@ class Options(AttrMonitor): # "listfiles" specifics self.listfiles_templateonly = self.args['--template'] # "install" specifics - self.install_force_action = self.args['--force-action'] + self.install_force_action = self.args['--force-actions'] self.install_temporary = self.args['--temp'] self.install_keys = self.args[''] self.install_diff = not self.args['--nodiff'] diff --git a/tests/helpers.py b/tests/helpers.py index 5ebc6c1..45ab61e 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -127,6 +127,7 @@ def _fake_args(): args['--key'] = False args['--ignore'] = [] args['--show-patch'] = False + args['--force-actions'] = False # cmds args['list'] = False args['listfiles'] = False From 4ed7b4d78c8efd3b9521d744466766eb037e6bbd Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 18:18:52 +0200 Subject: [PATCH 21/58] adding "remove" option for #47 and fix a few issues/bugs --- completion/_dotdrop-completion.zsh | 35 ++++- completion/_dotdrop.sh-completion.zsh | 35 ++++- completion/dotdrop-completion.bash | 17 ++- completion/dotdrop.sh-completion.bash | 17 ++- dotdrop/cfg_aggregator.py | 60 +++++++-- dotdrop/cfg_yaml.py | 104 ++++++++++----- dotdrop/dotdrop.py | 78 ++++++++++- dotdrop/options.py | 5 + dotdrop/updater.py | 59 ++------- tests-ng/remove.sh | 179 ++++++++++++++++++++++++++ tests/helpers.py | 17 ++- tests/test_import.py | 4 +- tests/test_install.py | 1 - 13 files changed, 509 insertions(+), 102 deletions(-) create mode 100755 tests-ng/remove.sh diff --git a/completion/_dotdrop-completion.zsh b/completion/_dotdrop-completion.zsh index 2f694da..7062241 100644 --- a/completion/_dotdrop-completion.zsh +++ b/completion/_dotdrop-completion.zsh @@ -38,6 +38,7 @@ _dotdrop () 'import' 'compare' 'update' + 'remove' 'listfiles' 'detail' 'list' @@ -59,6 +60,9 @@ _dotdrop () update) _dotdrop-update ;; + remove) + _dotdrop-remove + ;; listfiles) _dotdrop-listfiles ;; @@ -97,7 +101,7 @@ _dotdrop-install () '(-D)-D' \ '(--showdiff)--showdiff' \ '(-a)-a' \ - '(--force-action)--force-action' \ + '(--force-actions)--force-actions' \ '(-c=-)-c=-' \ '(--cfg=-)--cfg=-' \ '(-p=-)-p=-' \ @@ -193,6 +197,35 @@ _dotdrop-update () fi } +_dotdrop-remove () +{ + local context state state_descr line + typeset -A opt_args + + if [[ $words[$CURRENT] == -* ]] ; then + _arguments -C \ + ':command:->command' \ + '(-V)-V' \ + '(--verbose)--verbose' \ + '(-b)-b' \ + '(--no-banner)--no-banner' \ + '(-f)-f' \ + '(--force)--force' \ + '(-d)-d' \ + '(--dry)--dry' \ + '(-k)-k' \ + '(--key)--key' \ + '(-c=-)-c=-' \ + '(--cfg=-)--cfg=-' \ + '(-p=-)-p=-' \ + '(--profile=-)--profile=-' \ + + else + myargs=('') + _message_next_arg + fi +} + _dotdrop-listfiles () { local context state state_descr line diff --git a/completion/_dotdrop.sh-completion.zsh b/completion/_dotdrop.sh-completion.zsh index 3357e75..a123621 100644 --- a/completion/_dotdrop.sh-completion.zsh +++ b/completion/_dotdrop.sh-completion.zsh @@ -38,6 +38,7 @@ _dotdrop.sh () 'import' 'compare' 'update' + 'remove' 'listfiles' 'detail' 'list' @@ -59,6 +60,9 @@ _dotdrop.sh () update) _dotdrop.sh-update ;; + remove) + _dotdrop.sh-remove + ;; listfiles) _dotdrop.sh-listfiles ;; @@ -97,7 +101,7 @@ _dotdrop.sh-install () '(-D)-D' \ '(--showdiff)--showdiff' \ '(-a)-a' \ - '(--force-action)--force-action' \ + '(--force-actions)--force-actions' \ '(-c=-)-c=-' \ '(--cfg=-)--cfg=-' \ '(-p=-)-p=-' \ @@ -193,6 +197,35 @@ _dotdrop.sh-update () fi } +_dotdrop.sh-remove () +{ + local context state state_descr line + typeset -A opt_args + + if [[ $words[$CURRENT] == -* ]] ; then + _arguments -C \ + ':command:->command' \ + '(-V)-V' \ + '(--verbose)--verbose' \ + '(-b)-b' \ + '(--no-banner)--no-banner' \ + '(-f)-f' \ + '(--force)--force' \ + '(-d)-d' \ + '(--dry)--dry' \ + '(-k)-k' \ + '(--key)--key' \ + '(-c=-)-c=-' \ + '(--cfg=-)--cfg=-' \ + '(-p=-)-p=-' \ + '(--profile=-)--profile=-' \ + + else + myargs=('') + _message_next_arg + fi +} + _dotdrop.sh-listfiles () { local context state state_descr line diff --git a/completion/dotdrop-completion.bash b/completion/dotdrop-completion.bash index 261a2be..92f2ffb 100644 --- a/completion/dotdrop-completion.bash +++ b/completion/dotdrop-completion.bash @@ -5,7 +5,7 @@ _dotdrop() cur="${COMP_WORDS[COMP_CWORD]}" if [ $COMP_CWORD -eq 1 ]; then - COMPREPLY=( $( compgen -W '-h --help -v --version install import compare update listfiles detail list' -- $cur) ) + COMPREPLY=( $( compgen -W '-h --help -v --version install import compare update remove listfiles detail list' -- $cur) ) else case ${COMP_WORDS[1]} in install) @@ -19,6 +19,9 @@ _dotdrop() ;; update) _dotdrop_update + ;; + remove) + _dotdrop_remove ;; listfiles) _dotdrop_listfiles @@ -40,7 +43,7 @@ _dotdrop_install() cur="${COMP_WORDS[COMP_CWORD]}" if [ $COMP_CWORD -ge 2 ]; then - COMPREPLY=( $( compgen -fW '-V --verbose -b --no-banner -t --temp -f --force -n --nodiff -d --dry -D --showdiff -a --force-action -c= --cfg= -p= --profile= ' -- $cur) ) + COMPREPLY=( $( compgen -fW '-V --verbose -b --no-banner -t --temp -f --force -n --nodiff -d --dry -D --showdiff -a --force-actions -c= --cfg= -p= --profile= ' -- $cur) ) fi } @@ -74,6 +77,16 @@ _dotdrop_update() fi } +_dotdrop_remove() +{ + local cur + cur="${COMP_WORDS[COMP_CWORD]}" + + if [ $COMP_CWORD -ge 2 ]; then + COMPREPLY=( $( compgen -fW '-V --verbose -b --no-banner -f --force -d --dry -k --key -c= --cfg= -p= --profile= ' -- $cur) ) + fi +} + _dotdrop_listfiles() { local cur diff --git a/completion/dotdrop.sh-completion.bash b/completion/dotdrop.sh-completion.bash index 98ece6c..c6e0760 100644 --- a/completion/dotdrop.sh-completion.bash +++ b/completion/dotdrop.sh-completion.bash @@ -5,7 +5,7 @@ _dotdropsh() cur="${COMP_WORDS[COMP_CWORD]}" if [ $COMP_CWORD -eq 1 ]; then - COMPREPLY=( $( compgen -W '-h --help -v --version install import compare update listfiles detail list' -- $cur) ) + COMPREPLY=( $( compgen -W '-h --help -v --version install import compare update remove listfiles detail list' -- $cur) ) else case ${COMP_WORDS[1]} in install) @@ -19,6 +19,9 @@ _dotdropsh() ;; update) _dotdropsh_update + ;; + remove) + _dotdropsh_remove ;; listfiles) _dotdropsh_listfiles @@ -40,7 +43,7 @@ _dotdropsh_install() cur="${COMP_WORDS[COMP_CWORD]}" if [ $COMP_CWORD -ge 2 ]; then - COMPREPLY=( $( compgen -fW '-V --verbose -b --no-banner -t --temp -f --force -n --nodiff -d --dry -D --showdiff -a --force-action -c= --cfg= -p= --profile= ' -- $cur) ) + COMPREPLY=( $( compgen -fW '-V --verbose -b --no-banner -t --temp -f --force -n --nodiff -d --dry -D --showdiff -a --force-actions -c= --cfg= -p= --profile= ' -- $cur) ) fi } @@ -74,6 +77,16 @@ _dotdropsh_update() fi } +_dotdropsh_remove() +{ + local cur + cur="${COMP_WORDS[COMP_CWORD]}" + + if [ $COMP_CWORD -ge 2 ]; then + COMPREPLY=( $( compgen -fW '-V --verbose -b --no-banner -f --force -d --dry -k --key -c= --cfg= -p= --profile= ' -- $cur) ) + fi +} + _dotdropsh_listfiles() { local cur diff --git a/dotdrop/cfg_aggregator.py b/dotdrop/cfg_aggregator.py index 142d98d..07a4407 100644 --- a/dotdrop/cfg_aggregator.py +++ b/dotdrop/cfg_aggregator.py @@ -19,6 +19,9 @@ from dotdrop.logger import Logger from dotdrop.utils import strip_home +TILD = '~' + + class CfgAggregator: file_prefix = 'f' @@ -99,9 +102,9 @@ class CfgAggregator: # patch trans_w/trans_r in dotfiles self._patch_keys_to_objs(self.dotfiles, - "trans_r", self.get_trans_r) + "trans_r", self._get_trans_r) self._patch_keys_to_objs(self.dotfiles, - "trans_w", self.get_trans_w) + "trans_w", self._get_trans_w) def _patch_keys_to_objs(self, containers, keys, get_by_key): """ @@ -128,6 +131,14 @@ class CfgAggregator: self.log.dbg('patching {}.{} with {}'.format(c, keys, objects)) setattr(c, keys, objects) + def del_dotfile(self, dotfile): + """remove this dotfile from the config""" + return self.cfgyaml.del_dotfile(dotfile.key) + + def del_dotfile_from_profile(self, dotfile, profile): + """remove this dotfile from this profile""" + return self.cfgyaml.del_dotfile_from_profile(dotfile.key, profile.key) + def new(self, src, dst, link, profile_key): """ import a new dotfile @@ -136,10 +147,9 @@ class CfgAggregator: @link: LinkType @profile_key: to which profile """ - home = os.path.expanduser('~') - dst = dst.replace(home, '~', 1) + dst = self.path_to_dotfile_dst(dst) - dotfile = self._get_dotfile_by_dst(dst) + dotfile = self.get_dotfile_by_dst(dst) if not dotfile: # get a new dotfile with a unique key key = self._get_new_dotfile_key(dst) @@ -228,8 +238,22 @@ class CfgAggregator: cnt += 1 return newkey - def _get_dotfile_by_dst(self, dst): + def path_to_dotfile_dst(self, path): + """normalize the path to match dotfile dst""" + path = os.path.expanduser(path) + path = os.path.expandvars(path) + path = os.path.abspath(path) + home = os.path.expanduser(TILD) + os.sep + + # normalize the path + if path.startswith(home): + path = path[len(home):] + path = os.path.join(TILD, path) + return path + + def get_dotfile_by_dst(self, dst): """get a dotfile by dst""" + dst = self.path_to_dotfile_dst(dst) try: return next(d for d in self.dotfiles if d.dst == dst) except StopIteration: @@ -262,12 +286,24 @@ class CfgAggregator: except StopIteration: return None + def get_profiles_by_dotfile_key(self, key): + """return all profiles having this dotfile""" + res = [] + for p in self.profiles: + keys = [d.key for d in p.dotfiles] + if key in keys: + res.append(p) + return res + def get_dotfiles(self, profile=None): """return dotfiles dict for this profile key""" if not profile: return self.dotfiles try: - return next(x.dotfiles for x in self.profiles if x.key == profile) + pro = self.get_profile(profile) + if not pro: + return [] + return pro.dotfiles except StopIteration: return [] @@ -278,7 +314,7 @@ class CfgAggregator: except StopIteration: return None - def get_action(self, key): + def _get_action(self, key): """return action by key""" try: return next(x for x in self.actions if x.key == key) @@ -293,19 +329,19 @@ class CfgAggregator: key, *args = fields if self.debug: self.log.dbg('action with parm: {} and {}'.format(key, args)) - action = self.get_action(key).copy(args) + action = self._get_action(key).copy(args) else: - action = self.get_action(key) + action = self._get_action(key) return action - def get_trans_r(self, key): + def _get_trans_r(self, key): """return the trans_r with this key""" try: return next(x for x in self.trans_r if x.key == key) except StopIteration: return None - def get_trans_w(self, key): + def _get_trans_w(self, key): """return the trans_w with this key""" try: return next(x for x in self.trans_w if x.key == key) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 4199391..e7bea99 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -8,6 +8,7 @@ handle lower level of the config file import os import yaml import glob +from copy import deepcopy # local imports from dotdrop.settings import Settings @@ -97,9 +98,10 @@ class CfgYaml: def _parse_main_yaml(self, dic): """parse the different blocks""" - self.ori_settings = self._get_entry(self.yaml_dict, self.key_settings) + self.ori_settings = self._get_entry(dic, self.key_settings) self.settings = Settings(None).serialize().get(self.key_settings) self.settings.update(self.ori_settings) + # resolve settings paths p = self._resolve_path(self.settings[self.key_settings_dotpath]) self.settings[self.key_settings_dotpath] = p @@ -109,50 +111,60 @@ class CfgYaml: self.log.dbg('settings: {}'.format(self.settings)) # dotfiles - self.dotfiles = self._get_entry(self.yaml_dict, self.key_dotfiles) + self.ori_dotfiles = self._get_entry(dic, self.key_dotfiles) + self.dotfiles = deepcopy(self.ori_dotfiles) + keys = self.dotfiles.keys() + if len(keys) != len(list(set(keys))): + dups = [x for x in keys if x not in list(set(keys))] + raise Exception('duplicate dotfile keys found: {}'.format(dups)) self.dotfiles = self._norm_dotfiles(self.dotfiles) if self.debug: self.log.dbg('dotfiles: {}'.format(self.dotfiles)) # profiles - self.profiles = self._get_entry(self.yaml_dict, self.key_profiles) + self.ori_profiles = self._get_entry(dic, self.key_profiles) + self.profiles = deepcopy(self.ori_profiles) if self.debug: self.log.dbg('profiles: {}'.format(self.profiles)) # actions - self.actions = self._get_entry(self.yaml_dict, self.key_actions, - mandatory=False) + self.ori_actions = self._get_entry(dic, self.key_actions, + mandatory=False) + self.actions = deepcopy(self.ori_actions) self.actions = self._norm_actions(self.actions) if self.debug: self.log.dbg('actions: {}'.format(self.actions)) # trans_r - if self.old_key_trans_r in self.yaml_dict: + key = self.key_trans_r + if self.old_key_trans_r in dic: self.log.warn('\"trans\" is deprecated, please use \"trans_read\"') - self.yaml_dict[self.key_trans_r] = self.yaml_dict.pop( - self.old_key_trans_r - ) - self.dirty = True - self.trans_r = self._get_entry(self.yaml_dict, self.key_trans_r, - mandatory=False) + key = self.old_key_trans_r + self.ori_trans_r = self._get_entry(dic, key, mandatory=False) + self.trans_r = deepcopy(self.ori_trans_r) if self.debug: self.log.dbg('trans_r: {}'.format(self.trans_r)) # trans_w - self.trans_w = self._get_entry(self.yaml_dict, self.key_trans_w, - mandatory=False) + self.ori_trans_w = self._get_entry(dic, self.key_trans_w, + mandatory=False) + self.trans_w = deepcopy(self.ori_trans_w) if self.debug: self.log.dbg('trans_w: {}'.format(self.trans_w)) # variables - self.variables = self._get_entry(self.yaml_dict, self.key_variables, - mandatory=False) + self.ori_variables = self._get_entry(dic, + self.key_variables, + mandatory=False) + self.variables = deepcopy(self.ori_variables) if self.debug: self.log.dbg('variables: {}'.format(self.variables)) # dynvariables - self.dvariables = self._get_entry(self.yaml_dict, self.key_dvariables, - mandatory=False) + self.ori_dvariables = self._get_entry(dic, + self.key_dvariables, + mandatory=False) + self.dvariables = deepcopy(self.ori_dvariables) if self.debug: self.log.dbg('dvariables: {}'.format(self.dvariables)) @@ -493,7 +505,7 @@ class CfgYaml: values[self.key_profiles_dotfiles] = uniq_list(current) if self.debug: dfs = values[self.key_profiles_dotfiles] - self.log.dbg('profile dfs after include: {}'.format(dfs)) + self.log.dbg('{} dfs after include: {}'.format(profile, dfs)) return values.get(self.key_profiles_dotfiles, []) def _resolve_path(self, path): @@ -538,21 +550,21 @@ class CfgYaml: """merge low into high""" # won't work in python3.4 # return {**low, **high} - new = low.copy() + new = deepcopy(low) new.update(high) return new - def _get_entry(self, yaml_dict, key, mandatory=True): + def _get_entry(self, dic, key, mandatory=True): """return entry from yaml dictionary""" - if key not in yaml_dict: + if key not in dic: if mandatory: raise Exception('invalid config: no {} found'.format(key)) - yaml_dict[key] = {} - return yaml_dict[key] - if mandatory and not yaml_dict[key]: + dic[key] = {} + return dic[key] + if mandatory and not dic[key]: # ensure is not none - yaml_dict[key] = {} - return yaml_dict[key] + dic[key] = {} + return dic[key] def _load_yaml(self, path): """load a yaml file to a dict""" @@ -609,6 +621,40 @@ class CfgYaml: self.yaml_dict[self.key_dotfiles][key] = df_dict self.dirty = True + def del_dotfile(self, key): + """remove this dotfile from config""" + if key not in self.yaml_dict[self.key_dotfiles]: + self.log.err('key not in dotfiles: {}'.format(key)) + return False + if self.debug: + self.log.dbg('remove dotfile: {}'.format(key)) + del self.yaml_dict[self.key_dotfiles][key] + if self.debug: + dfs = self.yaml_dict[self.key_dotfiles] + self.log.dbg('new dotfiles: {}'.format(dfs)) + self.dirty = True + return True + + def del_dotfile_from_profile(self, df_key, pro_key): + """remove this dotfile from that profile""" + if df_key not in self.dotfiles.keys(): + self.log.err('key not in dotfiles: {}'.format(df_key)) + return False + if pro_key not in self.profiles.keys(): + self.log.err('key not in profiles: {}'.format(pro_key)) + return False + profiles = self.yaml_dict[self.key_profiles][pro_key] + if self.debug: + dfs = profiles[self.key_profiles_dotfiles] + self.log.dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) + self.log.dbg('remove {} from profile {}'.format(df_key, pro_key)) + profiles[self.key_profiles_dotfiles].remove(df_key) + if self.debug: + dfs = profiles[self.key_profiles_dotfiles] + self.log.dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) + self.dirty = True + return True + def _fix_deprecated(self, yamldict): """fix deprecated entries""" self._fix_deprecated_link_by_default(yamldict) @@ -671,9 +717,9 @@ class CfgYaml: newv = v if isinstance(v, dict): newv = self._clear_none(v) - if v is None: + if newv is None: continue - if not v: + if not newv: continue new[k] = newv return new diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index b95cc64..1721d46 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -268,7 +268,10 @@ def cmd_update(o): if o.debug: LOG.dbg('dotfile to update: {}'.format(paths)) - updater = Updater(o.dotpath, o.dotfiles, o.variables, + updater = Updater(o.dotpath, o.variables, + o.conf.get_dotfile, + o.conf.get_dotfile_by_dst, + o.conf.path_to_dotfile_dst, dry=o.dry, safe=o.safe, debug=o.debug, ignore=ignore, showpatch=showpatch) if not iskey: @@ -403,6 +406,67 @@ def cmd_detail(o): LOG.log('') +def cmd_remove(o): + """remove dotfile from dotpath and from config""" + paths = o.remove_path + iskey = o.remove_iskey + + if not paths: + LOG.log('no dotfile to remove') + return False + if o.debug: + LOG.dbg('dotfile to remove: {}'.format(paths)) + + removed = [] + for key in paths: + if o.debug: + LOG.dbg('removing {}'.format(key)) + if not iskey: + # by path + dotfile = o.conf.get_dotfile_by_dst(key) + if not dotfile: + LOG.warn('{} ignored, does not exist'.format(key)) + continue + k = dotfile.key + else: + # by key + dotfile = o.conf.get_dotfile(key) + k = key + # make sure is part of the profile + if dotfile.key not in [d.key for d in o.dotfiles]: + LOG.warn('{} ignored, not associated to this profile'.format(key)) + continue + profiles = o.conf.get_profiles_by_dotfile_key(k) + pkeys = ','.join([p.key for p in profiles]) + if o.dry: + LOG.dry('would remove {} from {}'.format(dotfile, pkeys)) + continue + msg = 'Remove dotfile from all these profiles: {}'.format(pkeys) + if o.safe and not LOG.ask(msg): + return False + if o.debug: + LOG.dbg('remove dotfile: {}'.format(dotfile)) + + for profile in profiles: + if not o.conf.del_dotfile_from_profile(dotfile, profile): + return False + if not o.conf.del_dotfile(dotfile): + return False + + # remove dotfile from dotpath + dtpath = os.path.join(o.dotpath, dotfile.src) + remove(dtpath) + removed.append(dotfile.key) + + if o.dry: + LOG.dry('new config file would be:') + LOG.raw(o.conf.dump()) + else: + o.conf.save() + LOG.log('\ndotfile(s) removed: {}'.format(','.join(removed))) + return True + + ########################################################### # helpers ########################################################### @@ -444,8 +508,10 @@ def _select(selections, dotfiles): def apply_trans(dotpath, dotfile, debug=False): - """apply the read transformation to the dotfile - return None if fails and new source if succeed""" + """ + apply the read transformation to the dotfile + return None if fails and new source if succeed + """ src = dotfile.src new_src = '{}.{}'.format(src, TRANS_SUFFIX) for trans in dotfile.trans_r: @@ -523,6 +589,12 @@ def main(): LOG.dbg('running cmd: detail') cmd_detail(o) + elif o.cmd_remove: + # remove dotfile + if o.debug: + LOG.dbg('running cmd: remove') + cmd_remove(o) + except KeyboardInterrupt: LOG.err('interrupted') ret = False diff --git a/dotdrop/options.py b/dotdrop/options.py index 8e69670..6ba7d6c 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -57,6 +57,7 @@ Usage: [-o ] [-C ...] [-i ...] dotdrop update [-VbfdkP] [-c ] [-p ] [-i ...] [...] + dotdrop remove [-Vbfdk] [-c ] [-p ] [...] dotdrop listfiles [-VbT] [-c ] [-p ] dotdrop detail [-Vb] [-c ] [-p ] [...] dotdrop list [-Vb] [-c ] @@ -193,6 +194,7 @@ class Options(AttrMonitor): self.cmd_import = self.args['import'] self.cmd_update = self.args['update'] self.cmd_detail = self.args['detail'] + self.cmd_remove = self.args['remove'] # adapt attributes based on arguments self.dry = self.args['--dry'] @@ -236,6 +238,9 @@ class Options(AttrMonitor): self.update_showpatch = self.args['--show-patch'] # "detail" specifics self.detail_keys = self.args[''] + # "remove" specifics + self.remove_path = self.args[''] + self.remove_iskey = self.args['--key'] def _fill_attr(self): """create attributes from conf""" diff --git a/dotdrop/updater.py b/dotdrop/updater.py index 5524d5c..1ee2148 100644 --- a/dotdrop/updater.py +++ b/dotdrop/updater.py @@ -20,12 +20,17 @@ TILD = '~' class Updater: - def __init__(self, dotpath, dotfiles, variables, dry=False, safe=True, + def __init__(self, dotpath, variables, + dotfile_key_getter, dotfile_dst_getter, + dotfile_path_normalizer, + dry=False, safe=True, debug=False, ignore=[], showpatch=False): """constructor @dotpath: path where dotfiles are stored - @dotfiles: dotfiles for this profile @variables: dictionary of variables for the templates + @dotfile_key_getter: func to get a dotfile by key + @dotfile_dst_getter: func to get a dotfile by dst + @dotfile_path_normalizer: func to normalize dotfile dst @dry: simulate @safe: ask for overwrite if True @debug: enable debug @@ -33,8 +38,10 @@ class Updater: @showpatch: show patch if dotfile to update is a template """ self.dotpath = dotpath - self.dotfiles = dotfiles self.variables = variables + self.dotfile_key_getter = dotfile_key_getter + self.dotfile_dst_getter = dotfile_dst_getter + self.dotfile_path_normalizer = dotfile_path_normalizer self.dry = dry self.safe = safe self.debug = debug @@ -48,8 +55,7 @@ class Updater: if not os.path.lexists(path): self.log.err('\"{}\" does not exist!'.format(path)) return False - path = self._normalize(path) - dotfile = self._get_dotfile_by_path(path) + dotfile = self.dotfile_dst_getter(path) if not dotfile: return False if self.debug: @@ -58,12 +64,12 @@ class Updater: def update_key(self, key): """update the dotfile referenced by key""" - dotfile = self._get_dotfile_by_key(key) + dotfile = self.dotfile_key_getter(key) if not dotfile: return False if self.debug: self.log.dbg('updating {} from key \"{}\"'.format(dotfile, key)) - path = self._normalize(dotfile.dst) + path = self.dotfile_path_normalizer(dotfile.dst) return self._update(path, dotfile) def _update(self, path, dotfile): @@ -111,45 +117,6 @@ class Updater: return None return tmp - def _normalize(self, path): - """normalize the path to match dotfile""" - path = os.path.expanduser(path) - path = os.path.expandvars(path) - path = os.path.abspath(path) - home = os.path.expanduser(TILD) + os.sep - - # normalize the path - if path.startswith(home): - path = path[len(home):] - path = os.path.join(TILD, path) - return path - - def _get_dotfile_by_key(self, key): - """get the dotfile matching this key""" - dotfiles = self.dotfiles - subs = [d for d in dotfiles if d.key == key] - if not subs: - self.log.err('key \"{}\" not found!'.format(key)) - return None - if len(subs) > 1: - found = ','.join([d.src for d in dotfiles]) - self.log.err('multiple dotfiles found: {}'.format(found)) - return None - return subs[0] - - def _get_dotfile_by_path(self, path): - """get the dotfile matching this path""" - dotfiles = self.dotfiles - subs = [d for d in dotfiles if d.dst == path] - if not subs: - self.log.err('\"{}\" is not managed!'.format(path)) - return None - if len(subs) > 1: - found = ','.join([d.src for d in dotfiles]) - self.log.err('multiple dotfiles found: {}'.format(found)) - return None - return subs[0] - def _is_template(self, path): if not Templategen.is_template(path): if self.debug: diff --git a/tests-ng/remove.sh b/tests-ng/remove.sh new file mode 100755 index 0000000..01d4966 --- /dev/null +++ b/tests-ng/remove.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2019, deadc0de6 +# +# test remove +# returns 1 in case of error +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "\e[96m\e[1m==> RUNNING $(basename $BASH_SOURCE) <==\e[0m" + +################################################################ +# this is the test +################################################################ + +# dotdrop directory +tmps=`mktemp -d --suffix='-dotdrop-tests'` +mkdir -p ${tmps}/dotfiles +# the dotfile to be imported +tmpd=`mktemp -d --suffix='-dotdrop-tests'` + +# create the config file +cfg="${tmps}/config.yaml" +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: + f_abc: + dst: ${tmpd}/abc + src: abc + f_def: + dst: ${tmpd}/def + src: def + f_last: + dst: ${tmpd}/last + src: last +profiles: + p1: + dotfiles: + - f_abc + - f_def + p2: + dotfiles: + - f_def + last: + dotfiles: + - f_last +_EOF +cfgbak="${tmps}/config.yaml.bak" +cp ${cfg} ${cfgbak} + +# create the dotfile +echo "abc" > ${tmps}/dotfiles/abc +echo "abc" > ${tmpd}/abc + +echo "def" > ${tmps}/dotfiles/def +echo "def" > ${tmpd}/def + +# remove with bad profile +cd ${ddpath} | ${bin} remove -f -k -p empty -c ${cfg} f_abc -V +[ ! -e ${tmps}/dotfiles/abc ] && echo "dotfile in dotpath deleted" && exit 1 +[ ! -e ${tmpd}/abc ] && echo "source dotfile deleted" && exit 1 +[ ! -e ${tmps}/dotfiles/def ] && echo "dotfile in dotpath deleted" && exit 1 +[ ! -e ${tmpd}/def ] && echo "source dotfile deleted" && exit 1 +# ensure config not altered +diff ${cfg} ${cfgbak} + +# remove by key +echo "[+] remove f_abc by key" +cd ${ddpath} | ${bin} remove -p p1 -f -k -c ${cfg} f_abc -V +cat ${cfg} +echo "[+] remove f_def by key" +cd ${ddpath} | ${bin} remove -p p2 -f -k -c ${cfg} f_def -V +cat ${cfg} + +# checks +[ -e ${tmps}/dotfiles/abc ] && echo "dotfile in dotpath not deleted" && exit 1 +[ ! -e ${tmpd}/abc ] && echo "source dotfile deleted" && exit 1 + +[ -e ${tmps}/dotfiles/def ] && echo "dotfile in dotpath not deleted" && exit 1 +[ ! -e ${tmpd}/def ] && echo "source dotfile deleted" && exit 1 + +echo "[+] =========" + +# create the config file +cfg="${tmps}/config.yaml" +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: + f_abc: + dst: ${tmpd}/abc + src: abc + f_def: + dst: ${tmpd}/def + src: def + f_last: + dst: ${tmpd}/last + src: last +profiles: + p1: + dotfiles: + - f_abc + - f_def + p2: + dotfiles: + - f_def + last: + dotfiles: + - f_last +_EOF +cat ${cfg} + +# create the dotfile +echo "abc" > ${tmps}/dotfiles/abc +echo "abc" > ${tmpd}/abc + +echo "def" > ${tmps}/dotfiles/def +echo "def" > ${tmpd}/def + +# remove by key +echo "[+] remove f_abc by path" +cd ${ddpath} | ${bin} remove -p p1 -f -c ${cfg} ${tmpd}/abc -V +cat ${cfg} +echo "[+] remove f_def by path" +cd ${ddpath} | ${bin} remove -p p2 -f -c ${cfg} ${tmpd}/def -V +cat ${cfg} + +# checks +[ -e ${tmps}/dotfiles/abc ] && echo "(2) dotfile in dotpath not deleted" && exit 1 +[ ! -e ${tmpd}/abc ] && echo "(2) source dotfile deleted" && exit 1 + +[ -e ${tmps}/dotfiles/def ] && echo "(2) dotfile in dotpath not deleted" && exit 1 +[ ! -e ${tmpd}/def ] && echo "(2) source dotfile deleted" && exit 1 + + +cat ${cfg} + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 diff --git a/tests/helpers.py b/tests/helpers.py index 45ab61e..7bd57f6 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -136,6 +136,7 @@ def _fake_args(): args['import'] = False args['update'] = False args['detail'] = False + args['remove'] = False return args @@ -172,8 +173,13 @@ def get_dotfile_from_yaml(dic, path): # path is not the file in dotpath but on the FS dotfiles = dic['dotfiles'] # src = get_path_strip_version(path) - dotfile = [d for d in dotfiles.values() if d['dst'] == path][0] - return dotfile + home = os.path.expanduser('~') + if path.startswith(home): + path = path.replace(home, '~') + dotfile = [d for d in dotfiles.values() if d['dst'] == path] + if dotfile: + return dotfile[0] + return None def yaml_dashed_list(items, indent=0): @@ -261,6 +267,11 @@ def file_in_yaml(yaml_file, path, link=False): in_dst = path in (os.path.expanduser(x['dst']) for x in dotfiles) if link: - has_link = 'link' in get_dotfile_from_yaml(yaml_conf, path) + df = get_dotfile_from_yaml(yaml_conf, path) + has_link = False + if df: + has_link = 'link' in df + else: + return False return in_src and in_dst and has_link return in_src and in_dst diff --git a/tests/test_import.py b/tests/test_import.py index bbbfb75..98efdea 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -352,7 +352,7 @@ class TestImport(unittest.TestCase): self.assertFalse(any(a.endswith('ing') for a in actions)) # testing transformations - transformations = y['trans_read'].keys() + transformations = y['trans'].keys() self.assertTrue(all(t.endswith('ed') for t in transformations)) self.assertFalse(any(t.endswith('ing') for t in transformations)) transformations = y['trans_write'].keys() @@ -394,7 +394,7 @@ class TestImport(unittest.TestCase): self.assertFalse(any(action.endswith('ed') for action in actions)) # testing transformations - transformations = y['trans_read'].keys() + transformations = y['trans'].keys() self.assertTrue(all(t.endswith('ing') for t in transformations)) self.assertFalse(any(t.endswith('ed') for t in transformations)) transformations = y['trans_write'].keys() diff --git a/tests/test_install.py b/tests/test_install.py index 988ae3a..af68d97 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -486,7 +486,6 @@ exec bspwm # ensure dst is link self.assertTrue(os.path.islink(dst)) # ensure dst not directly linked to src - # TODO: maybe check that its actually linked to template folder self.assertNotEqual(os.path.realpath(dst), src) From e486fbc8d5427cf6c3409c07653dcd03d37ad95d Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 18:26:33 +0200 Subject: [PATCH 22/58] rebase version and arch package --- packages/arch-dotdrop/.SRCINFO | 4 ++-- packages/arch-dotdrop/PKGBUILD | 2 +- version.py | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 version.py diff --git a/packages/arch-dotdrop/.SRCINFO b/packages/arch-dotdrop/.SRCINFO index 4da2e16..26b449c 100644 --- a/packages/arch-dotdrop/.SRCINFO +++ b/packages/arch-dotdrop/.SRCINFO @@ -1,6 +1,6 @@ pkgbase = dotdrop pkgdesc = Save your dotfiles once, deploy them everywhere - pkgver = 0.27.0 + pkgver = 0.28.0 pkgrel = 1 url = https://github.com/deadc0de6/dotdrop arch = any @@ -11,7 +11,7 @@ pkgbase = dotdrop depends = python-jinja depends = python-docopt depends = python-pyaml - source = git+https://github.com/deadc0de6/dotdrop.git#tag=v0.27.0 + source = git+https://github.com/deadc0de6/dotdrop.git#tag=v0.28.0 md5sums = SKIP pkgname = dotdrop diff --git a/packages/arch-dotdrop/PKGBUILD b/packages/arch-dotdrop/PKGBUILD index 20da6a6..5891d6a 100644 --- a/packages/arch-dotdrop/PKGBUILD +++ b/packages/arch-dotdrop/PKGBUILD @@ -1,7 +1,7 @@ # Maintainer: deadc0de6 pkgname=dotdrop -pkgver=0.27.0 +pkgver=0.28.0 pkgrel=1 pkgdesc="Save your dotfiles once, deploy them everywhere " arch=('any') diff --git a/version.py b/version.py new file mode 100644 index 0000000..e35d6cc --- /dev/null +++ b/version.py @@ -0,0 +1,6 @@ +""" +author: deadc0de6 (https://github.com/deadc0de6) +Copyright (c) 2018, deadc0de6 +""" + +__version__ = '0.28.0' From 827b860cc90bef1fa2ecac3b831c7f3ee5a8ac31 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 18:40:27 +0200 Subject: [PATCH 23/58] rebase version --- dotdrop/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotdrop/version.py b/dotdrop/version.py index a658c2b..e35d6cc 100644 --- a/dotdrop/version.py +++ b/dotdrop/version.py @@ -3,4 +3,4 @@ author: deadc0de6 (https://github.com/deadc0de6) Copyright (c) 2018, deadc0de6 """ -__version__ = '0.27.0' +__version__ = '0.28.0' From 61053f29166955315e25bb2dc435ca4efde92f26 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 18:41:09 +0200 Subject: [PATCH 24/58] remove version file in root --- version.py | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 version.py diff --git a/version.py b/version.py deleted file mode 100644 index e35d6cc..0000000 --- a/version.py +++ /dev/null @@ -1,6 +0,0 @@ -""" -author: deadc0de6 (https://github.com/deadc0de6) -Copyright (c) 2018, deadc0de6 -""" - -__version__ = '0.28.0' From dbd1e9a942ce98cfdbc8fdeb3e7041b4c0955db5 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 19:06:33 +0200 Subject: [PATCH 25/58] update default config file --- config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.yaml b/config.yaml index cca9aab..bfb00e5 100644 --- a/config.yaml +++ b/config.yaml @@ -5,7 +5,7 @@ config: banner: true longkey: false keepdot: false - link_import_default: nolink + link_on_import: nolink link_dotfile_default: nolink dotfiles: profiles: From 53e3921e6495432bcc1bc39174218142374fc330 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 19:08:35 +0200 Subject: [PATCH 26/58] order config alpha --- config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.yaml b/config.yaml index bfb00e5..43e2b97 100644 --- a/config.yaml +++ b/config.yaml @@ -1,10 +1,10 @@ config: backup: true + banner: true create: true dotpath: dotfiles - banner: true - longkey: false keepdot: false + longkey: false link_on_import: nolink link_dotfile_default: nolink dotfiles: From 55778f79597fd4d31b0d1815753f3992a9ec4f49 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 19:12:22 +0200 Subject: [PATCH 27/58] fix config saving bug --- config.yaml | 4 ++-- dotdrop/cfg_yaml.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config.yaml b/config.yaml index 43e2b97..9e2e1ea 100644 --- a/config.yaml +++ b/config.yaml @@ -4,8 +4,8 @@ config: create: true dotpath: dotfiles keepdot: false - longkey: false - link_on_import: nolink link_dotfile_default: nolink + link_on_import: nolink + longkey: false dotfiles: profiles: diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index e7bea99..6b69d52 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -717,10 +717,10 @@ class CfgYaml: newv = v if isinstance(v, dict): newv = self._clear_none(v) + if not newv: + continue if newv is None: continue - if not newv: - continue new[k] = newv return new From ba93547ea090f30b57838b221f5a10a722d3cba4 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 2 Jun 2019 19:42:53 +0200 Subject: [PATCH 28/58] fix diverse bugs --- dotdrop/cfg_aggregator.py | 1 - dotdrop/cfg_yaml.py | 49 ++++++++++++++++++++++++++++----------- dotdrop/dotdrop.py | 9 ++++--- dotdrop/exceptions.py | 11 +++++++++ dotdrop/profile.py | 5 +++- 5 files changed, 56 insertions(+), 19 deletions(-) create mode 100644 dotdrop/exceptions.py diff --git a/dotdrop/cfg_aggregator.py b/dotdrop/cfg_aggregator.py index 07a4407..fc751d6 100644 --- a/dotdrop/cfg_aggregator.py +++ b/dotdrop/cfg_aggregator.py @@ -253,7 +253,6 @@ class CfgAggregator: def get_dotfile_by_dst(self, dst): """get a dotfile by dst""" - dst = self.path_to_dotfile_dst(dst) try: return next(d for d in self.dotfiles if d.dst == dst) except StopIteration: diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 6b69d52..b98b48c 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -16,6 +16,7 @@ from dotdrop.logger import Logger from dotdrop.templategen import Templategen from dotdrop.linktypes import LinkTypes from dotdrop.utils import shell, uniq_list +from dotdrop.exceptions import YamlException class CfgYaml: @@ -116,7 +117,8 @@ class CfgYaml: keys = self.dotfiles.keys() if len(keys) != len(list(set(keys))): dups = [x for x in keys if x not in list(set(keys))] - raise Exception('duplicate dotfile keys found: {}'.format(dups)) + err = 'duplicate dotfile keys found: {}'.format(dups) + raise YamlException(err) self.dotfiles = self._norm_dotfiles(self.dotfiles) if self.debug: self.log.dbg('dotfiles: {}'.format(self.dotfiles)) @@ -329,7 +331,7 @@ class CfgYaml: # inherite profile variables for inherited_profile in pentry.get(self.key_profile_include, []): if inherited_profile == profile or inherited_profile in seen: - raise Exception('\"include\" loop') + raise YamlException('\"include\" loop') seen.append(inherited_profile) new = self._get_variables_dict(inherited_profile, seen, sub=True) variables.update(new) @@ -356,7 +358,7 @@ class CfgYaml: # inherite profile dynvariables for inherited_profile in pentry.get(self.key_profile_include, []): if inherited_profile == profile or inherited_profile in seen: - raise Exception('\"include loop\"') + raise YamlException('\"include loop\"') seen.append(inherited_profile) new = self._get_dvariables_dict(inherited_profile, seen, sub=True) variables.update(new) @@ -383,7 +385,7 @@ class CfgYaml: p = os.path.expanduser(p) new = glob.glob(p) if not new: - raise Exception('bad path: {}'.format(p)) + raise YamlException('bad path: {}'.format(p)) res.extend(glob.glob(p)) return res @@ -438,8 +440,7 @@ class CfgYaml: current = v.get(self.key_dotfiles, []) path = self._resolve_path(p) current = self._import_sub(path, self.key_dotfiles, - current, mandatory=False, - path_func=self._norm_dotfiles) + current, mandatory=False) v[self.key_dotfiles] = current def _import_configs(self): @@ -491,7 +492,7 @@ class CfgYaml: seen = [] for i in inc: if i in seen: - raise Exception('\"include loop\"') + raise YamlException('\"include loop\"') seen.append(i) if i not in self.profiles.keys(): self.log.warn('include unknown profile: {}'.format(i)) @@ -541,7 +542,7 @@ class CfgYaml: elif isinstance(current, list) and isinstance(new, list): current = current + new else: - raise Exception('invalid import {} from {}'.format(key, path)) + raise YamlException('invalid import {} from {}'.format(key, path)) if self.debug: self.log.dbg('new \"{}\": {}'.format(key, current)) return current @@ -558,7 +559,7 @@ class CfgYaml: """return entry from yaml dictionary""" if key not in dic: if mandatory: - raise Exception('invalid config: no {} found'.format(key)) + raise YamlException('invalid config: no {} found'.format(key)) dic[key] = {} return dic[key] if mandatory and not dic[key]: @@ -570,13 +571,13 @@ class CfgYaml: """load a yaml file to a dict""" content = {} if not os.path.exists(path): - raise Exception('config path not found: {}'.format(path)) + raise YamlException('config path not found: {}'.format(path)) with open(path, 'r') as f: try: content = yaml.safe_load(f) except Exception as e: self.log.err(e) - raise Exception('invalid config: {}'.format(path)) + raise YamlException('invalid config: {}'.format(path)) return content def _new_profile(self, key): @@ -718,8 +719,13 @@ class CfgYaml: if isinstance(v, dict): newv = self._clear_none(v) if not newv: + # no empty dict continue if newv is None: + # no None value + continue + if isinstance(newv, list) and not newv: + # no empty list continue new[k] = newv return new @@ -730,12 +736,27 @@ class CfgYaml: return False content = self._clear_none(self.dump()) + + # make sure we have the base entries + if self.key_settings not in content: + content[self.key_settings] = None + if self.key_dotfiles not in content: + content[self.key_dotfiles] = None + if self.key_profiles not in content: + content[self.key_profiles] = None + + # ensure no null are displayed + data = yaml.safe_dump(content, + default_flow_style=False, + indent=2) + data = data.replace('null', '') + + # save to file if self.debug: self.log.dbg('saving: {}'.format(content)) with open(self.path, 'w') as f: - yaml.safe_dump(content, f, - default_flow_style=False, - indent=2) + f.write(data) + self.dirty = False return True diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 1721d46..7f49186 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -17,6 +17,7 @@ from dotdrop.updater import Updater from dotdrop.comparator import Comparator from dotdrop.utils import get_tmpdir, remove, strip_home, run, uniq_list from dotdrop.linktypes import LinkTypes +from dotdrop.exceptions import YamlException LOG = Logger() TRANS_SUFFIX = 'trans' @@ -423,7 +424,9 @@ def cmd_remove(o): LOG.dbg('removing {}'.format(key)) if not iskey: # by path + print(key) dotfile = o.conf.get_dotfile_by_dst(key) + print(dotfile) if not dotfile: LOG.warn('{} ignored, does not exist'.format(key)) continue @@ -441,7 +444,7 @@ def cmd_remove(o): if o.dry: LOG.dry('would remove {} from {}'.format(dotfile, pkeys)) continue - msg = 'Remove dotfile from all these profiles: {}'.format(pkeys) + msg = 'Remove \"{}\" from all these profiles: {}'.format(k, pkeys) if o.safe and not LOG.ask(msg): return False if o.debug: @@ -537,8 +540,8 @@ def main(): """entry point""" try: o = Options() - except Exception as e: - LOG.err('options error: {}'.format(str(e))) + except YamlException as e: + LOG.err('config file error: {}'.format(str(e))) return False ret = True diff --git a/dotdrop/exceptions.py b/dotdrop/exceptions.py new file mode 100644 index 0000000..a327696 --- /dev/null +++ b/dotdrop/exceptions.py @@ -0,0 +1,11 @@ +""" +author: deadc0de6 (https://github.com/deadc0de6) +Copyright (c) 2019, deadc0de6 + +diverse exceptions +""" + + +class YamlException(Exception): + """exception in CfgYaml""" + pass diff --git a/dotdrop/profile.py b/dotdrop/profile.py index c1147ee..fd7bd3f 100644 --- a/dotdrop/profile.py +++ b/dotdrop/profile.py @@ -15,18 +15,21 @@ class Profile(DictParser): key_include = 'include' key_import = 'import' - def __init__(self, key, actions=[], dotfiles=[], variables=[]): + def __init__(self, key, actions=[], dotfiles=[], + variables=[], dynvariables=[]): """ constructor @key: profile key @actions: list of action keys @dotfiles: list of dotfile keys @variables: list of variable keys + @dynvariables: list of interpreted variable keys """ self.key = key self.actions = actions self.dotfiles = dotfiles self.variables = variables + self.dynvariables = dynvariables def get_pre_actions(self): """return all 'pre' actions""" From 1d3d8953eebf9edfa4ee4bde1ad0adb4a4c858f0 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 4 Jun 2019 15:40:22 +0200 Subject: [PATCH 29/58] backport issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index a4b661a..3cb74cd 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -7,7 +7,7 @@ assignees: '' --- -Dotdrop version: v0.xxx +Dotdrop version (and git commit if run from source): v0.xxx Using dotdrop: as a submodule, from pypi, '...' **Describe the bug** From d4b3ed22298755edf15b87c2ee9a68adae2042ab Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 4 Jun 2019 21:50:11 +0200 Subject: [PATCH 30/58] implement relative upignore/cmpignore for #149 and some import refactoring --- dotdrop/comparator.py | 32 +++--- dotdrop/dotdrop.py | 4 +- dotdrop/updater.py | 20 ++-- dotdrop/utils.py | 19 ++++ tests-ng/compare-ignore-relative.sh | 159 ++++++++++++++++++++++++++++ tests-ng/update-ignore-relative.sh | 107 +++++++++++++++++++ 6 files changed, 315 insertions(+), 26 deletions(-) create mode 100755 tests-ng/compare-ignore-relative.sh create mode 100755 tests-ng/update-ignore-relative.sh diff --git a/dotdrop/comparator.py b/dotdrop/comparator.py index 992674d..eda8387 100644 --- a/dotdrop/comparator.py +++ b/dotdrop/comparator.py @@ -10,7 +10,7 @@ import filecmp # local imports from dotdrop.logger import Logger -import dotdrop.utils as utils +from dotdrop.utils import must_ignore, uniq_list, diff class Comparator: @@ -43,7 +43,7 @@ class Comparator: """compare a file""" if self.debug: self.log.dbg('compare file {} with {}'.format(left, right)) - if utils.must_ignore([left, right], ignore, debug=self.debug): + if must_ignore([left, right], ignore, debug=self.debug): if self.debug: self.log.dbg('ignoring diff {} and {}'.format(left, right)) return '' @@ -55,7 +55,7 @@ class Comparator: self.log.dbg('compare directory {} with {}'.format(left, right)) if not os.path.exists(right): return '' - if utils.must_ignore([left, right], ignore, debug=self.debug): + if must_ignore([left, right], ignore, debug=self.debug): if self.debug: self.log.dbg('ignoring diff {} and {}'.format(left, right)) return '' @@ -68,15 +68,15 @@ class Comparator: # handle files only in deployed dir for i in comp.left_only: - if utils.must_ignore([os.path.join(left, i)], - ignore, debug=self.debug): + if must_ignore([os.path.join(left, i)], + ignore, debug=self.debug): continue ret.append('=> \"{}\" does not exist on local\n'.format(i)) # handle files only in dotpath dir for i in comp.right_only: - if utils.must_ignore([os.path.join(right, i)], - ignore, debug=self.debug): + if must_ignore([os.path.join(right, i)], + ignore, debug=self.debug): continue ret.append('=> \"{}\" does not exist in dotdrop\n'.format(i)) @@ -85,8 +85,8 @@ class Comparator: for i in funny: lfile = os.path.join(left, i) rfile = os.path.join(right, i) - if utils.must_ignore([lfile, rfile], - ignore, debug=self.debug): + if must_ignore([lfile, rfile], + ignore, debug=self.debug): continue short = os.path.basename(lfile) # file vs dir @@ -95,12 +95,12 @@ class Comparator: # content is different funny = comp.diff_files funny.extend(comp.funny_files) - funny = utils.uniq_list(funny) + funny = uniq_list(funny) for i in funny: lfile = os.path.join(left, i) rfile = os.path.join(right, i) - if utils.must_ignore([lfile, rfile], - ignore, debug=self.debug): + if must_ignore([lfile, rfile], + ignore, debug=self.debug): continue diff = self._diff(lfile, rfile, header=True) ret.append(diff) @@ -115,9 +115,9 @@ class Comparator: def _diff(self, left, right, header=False): """diff using the unix tool diff""" - diff = utils.diff(left, right, raw=False, - opts=self.diffopts, debug=self.debug) + d = diff(left, right, raw=False, + opts=self.diffopts, debug=self.debug) if header: lshort = os.path.basename(left) - diff = '=> diff \"{}\":\n{}'.format(lshort, diff) - return diff + d = '=> diff \"{}\":\n{}'.format(lshort, diff) + return d diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 7f49186..5e575d5 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -15,7 +15,8 @@ from dotdrop.templategen import Templategen from dotdrop.installer import Installer from dotdrop.updater import Updater from dotdrop.comparator import Comparator -from dotdrop.utils import get_tmpdir, remove, strip_home, run, uniq_list +from dotdrop.utils import get_tmpdir, remove, strip_home, \ + run, uniq_list, patch_ignores from dotdrop.linktypes import LinkTypes from dotdrop.exceptions import YamlException @@ -225,6 +226,7 @@ def cmd_compare(o, tmp): same = False continue ignores = list(set(o.compare_ignore + dotfile.cmpignore)) + ignores = patch_ignores(ignores, dotfile.src) diff = comp.compare(insttmp, dotfile.dst, ignore=ignores) if tmpsrc: # clean tmp transformed dotfile if any diff --git a/dotdrop/updater.py b/dotdrop/updater.py index 1ee2148..48df214 100644 --- a/dotdrop/updater.py +++ b/dotdrop/updater.py @@ -12,7 +12,8 @@ import filecmp # local imports from dotdrop.logger import Logger from dotdrop.templategen import Templategen -import dotdrop.utils as utils +from dotdrop.utils import patch_ignores, remove, get_unique_tmp_name, \ + write_to_tmpfile, must_ignore TILD = '~' @@ -76,7 +77,8 @@ class Updater: """update dotfile from file pointed by path""" ret = False new_path = None - self.ignores = list(set(self.ignore + dotfile.upignore)) + ignores = list(set(self.ignore + dotfile.upignore)) + self.ignores = patch_ignores(ignores, dotfile.dst) if self.debug: self.log.dbg('ignore pattern(s): {}'.format(self.ignores)) @@ -98,7 +100,7 @@ class Updater: ret = self._handle_file(path, dtpath) # clean temporary files if new_path and os.path.exists(new_path): - utils.remove(new_path) + remove(new_path) return ret def _apply_trans_w(self, path, dotfile): @@ -108,12 +110,12 @@ class Updater: return path if self.debug: self.log.dbg('executing write transformation {}'.format(trans)) - tmp = utils.get_unique_tmp_name() + tmp = get_unique_tmp_name() if not trans.transform(path, tmp): msg = 'transformation \"{}\" failed for {}' self.log.err(msg.format(trans.key, dotfile.key)) if os.path.exists(tmp): - utils.remove(tmp) + remove(tmp) return None return tmp @@ -128,7 +130,7 @@ class Updater: def _show_patch(self, fpath, tpath): """provide a way to manually patch the template""" content = self._resolve_template(tpath) - tmp = utils.write_to_tmpfile(content) + tmp = write_to_tmpfile(content) cmds = ['diff', '-u', tmp, fpath, '|', 'patch', tpath] self.log.warn('try patching with: \"{}\"'.format(' '.join(cmds))) return False @@ -231,7 +233,7 @@ class Updater: self.log.dbg('rm -r {}'.format(old)) if not self._confirm_rm_r(old): continue - utils.remove(old) + remove(old) self.log.sub('\"{}\" dir removed'.format(old)) # handle files diff @@ -283,7 +285,7 @@ class Updater: continue if self.debug: self.log.dbg('rm {}'.format(new)) - utils.remove(new) + remove(new) self.log.sub('\"{}\" removed'.format(new)) # Recursively decent into common subdirectories. @@ -308,7 +310,7 @@ class Updater: return True def _ignore(self, paths): - if utils.must_ignore(paths, self.ignores, debug=self.debug): + if must_ignore(paths, self.ignores, debug=self.debug): if self.debug: self.log.dbg('ignoring update for {}'.format(paths)) return True diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 2954a96..ac36a74 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -17,6 +17,7 @@ from shutil import rmtree from dotdrop.logger import Logger LOG = Logger() +STAR = '*' def run(cmd, raw=True, debug=False, checkerr=False): @@ -139,3 +140,21 @@ def uniq_list(a_list): if a not in new: new.append(a) return new + + +def patch_ignores(ignores, prefix): + """allow relative ignore pattern""" + new = [] + for ignore in ignores: + if STAR in ignore: + # is glob + new.append(ignore) + continue + if os.path.isabs(ignore): + # is absolute + new.append(ignore) + continue + # patch upignore + path = os.path.join(prefix, ignore) + new.append(path) + return new diff --git a/tests-ng/compare-ignore-relative.sh b/tests-ng/compare-ignore-relative.sh new file mode 100755 index 0000000..519d8bf --- /dev/null +++ b/tests-ng/compare-ignore-relative.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2019, deadc0de6 +# +# test compare ignore relative +# returns 1 in case of error +# + +# exit on first error +#set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "\e[96m\e[1m==> RUNNING $(basename $BASH_SOURCE) <==\e[0m" + +################################################################ +# this is the test +################################################################ + +# dotdrop directory +basedir=`mktemp -d --suffix='-dotdrop-tests'` +echo "[+] dotdrop dir: ${basedir}" +echo "[+] dotpath dir: ${basedir}/dotfiles" + +# the dotfile to be imported +tmpd=`mktemp -d --suffix='-dotdrop-tests'` + +# some files +mkdir -p ${tmpd}/{program,config} +touch ${tmpd}/program/a +touch ${tmpd}/config/a +mkdir ${tmpd}/vscode +touch ${tmpd}/vscode/extensions.txt +touch ${tmpd}/vscode/keybindings.json + +# create the config file +cfg="${basedir}/config.yaml" +create_conf ${cfg} # sets token + +# import +echo "[+] import" +cd ${ddpath} | ${bin} import -c ${cfg} ${tmpd}/program +cd ${ddpath} | ${bin} import -c ${cfg} ${tmpd}/config +cd ${ddpath} | ${bin} import -c ${cfg} ${tmpd}/vscode + +# add files +echo "[+] add files" +touch ${tmpd}/program/b +touch ${tmpd}/config/b + +# expects diff +echo "[+] comparing normal - 2 diffs" +set +e +cd ${ddpath} | ${bin} compare -c ${cfg} --verbose +[ "$?" = "0" ] && exit 1 +set -e + +# expects one diff +patt="${tmpd}/config/b" +echo "[+] comparing with ignore (pattern: ${patt}) - 1 diff" +set +e +cd ${ddpath} | ${bin} compare -c ${cfg} --verbose --ignore=${patt} +[ "$?" = "0" ] && exit 1 +set -e + +# expects no diff +patt="*b" +echo "[+] comparing with ignore (pattern: ${patt}) - 0 diff" +set +e +cd ${ddpath} | ${bin} compare -c ${cfg} --verbose --ignore=${patt} +[ "$?" != "0" ] && exit 1 +set -e + +# expects one diff +patt="*/config/*b" +echo "[+] comparing with ignore (pattern: ${patt}) - 1 diff" +set +e +cd ${ddpath} | ${bin} compare -c ${cfg} --verbose --ignore=${patt} +[ "$?" = "0" ] && exit 1 +set -e + +#cat ${cfg} + +# adding ignore in dotfile +cfg2="${basedir}/config2.yaml" +sed '/d_config:/a \ \ \ \ cmpignore:\n\ \ \ \ - "config/b"' ${cfg} > ${cfg2} +#cat ${cfg2} + +# expects one diff +echo "[+] comparing with ignore in dotfile - 1 diff" +set +e +cd ${ddpath} | ${bin} compare -c ${cfg2} --verbose +[ "$?" = "0" ] && exit 1 +set -e + +# adding ignore in dotfile +cfg2="${basedir}/config2.yaml" +sed '/d_config:/a \ \ \ \ cmpignore:\n\ \ \ \ - "b"' ${cfg} > ${cfg2} +sed -i '/d_program:/a \ \ \ \ cmpignore:\n\ \ \ \ - "b"' ${cfg2} +#cat ${cfg2} + +# expects no diff +patt="*b" +echo "[+] comparing with ignore in dotfile - 0 diff" +set +e +cd ${ddpath} | ${bin} compare -c ${cfg2} --verbose +[ "$?" != "0" ] && exit 1 +set -e + +# update files +echo touched > ${tmpd}/vscode/extensions.txt +echo touched > ${tmpd}/vscode/keybindings.json + +# expect two diffs +set +e +cd ${ddpath} | ${bin} compare -c ${cfg} --verbose -C ${tmpd}/vscode +[ "$?" = "0" ] && exit 1 +set -e + +# expects no diff +sed '/d_vscode:/a \ \ \ \ cmpignore:\n\ \ \ \ - "extensions.txt"\n\ \ \ \ - "keybindings.json"' ${cfg} > ${cfg2} +set +e +cd ${ddpath} | ${bin} compare -c ${cfg2} --verbose -C ${tmpd}/vscode +[ "$?" != "0" ] && exit 1 +set -e + +## CLEANING +rm -rf ${basedir} ${tmpd} + +echo "OK" +exit 0 diff --git a/tests-ng/update-ignore-relative.sh b/tests-ng/update-ignore-relative.sh new file mode 100755 index 0000000..1dc36f1 --- /dev/null +++ b/tests-ng/update-ignore-relative.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2019, deadc0de6 +# +# test ignore update relative pattern +# returns 1 in case of error +# + +# exit on first error +#set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "\e[96m\e[1m==> RUNNING $(basename $BASH_SOURCE) <==\e[0m" + +################################################################ +# this is the test +################################################################ + +# dotdrop directory +tmps=`mktemp -d --suffix='-dotdrop-tests'` +dt="${tmps}/dotfiles" +mkdir -p ${dt} +mkdir -p ${dt}/a/{b,c} +echo 'a' > ${dt}/a/b/abfile +echo 'a' > ${dt}/a/c/acfile + +# fs dotfiles +tmpd=`mktemp -d --suffix='-dotdrop-tests'` +cp -r ${dt}/a ${tmpd}/ + +# create the config file +cfg="${tmps}/config.yaml" +cat > ${cfg} << _EOF +config: + backup: false + create: true + dotpath: dotfiles +dotfiles: + f_abc: + dst: ${tmpd}/a + src: a + upignore: + - "cfile" + - "newfile" + - "newdir" +profiles: + p1: + dotfiles: + - f_abc +_EOF +#cat ${cfg} + +#tree ${dt} + +# edit/add files +echo "[+] edit/add files" +touch ${tmpd}/a/newfile +echo 'b' > ${tmpd}/a/c/acfile +mkdir -p ${tmpd}/a/newdir/b +touch ${tmpd}/a/newdir/b/c + +#tree ${tmpd}/a + +# update +echo "[+] update" +cd ${ddpath} | ${bin} update -f -c ${cfg} --verbose --profile=p1 --key f_abc + +#tree ${dt} + +# check files haven't been updated +grep 'b' ${dt}/a/c/acfile >/dev/null +[ -e ${dt}/a/newfile ] && exit 1 + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 From 9c9a4c6d2b4a939e89963fca2c6e26cd0a3cb18f Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 4 Jun 2019 22:25:16 +0200 Subject: [PATCH 31/58] fix bug with remove and ALL in profiles --- dotdrop/cfg_yaml.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index b98b48c..8b84972 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -642,16 +642,19 @@ class CfgYaml: self.log.err('key not in dotfiles: {}'.format(df_key)) return False if pro_key not in self.profiles.keys(): - self.log.err('key not in profiles: {}'.format(pro_key)) + self.log.err('key not in profile: {}'.format(pro_key)) return False - profiles = self.yaml_dict[self.key_profiles][pro_key] + # get the profile dictionary + profile = self.yaml_dict[self.key_profiles][pro_key] + if df_key not in profile[self.key_profiles_dotfiles]: + return True if self.debug: - dfs = profiles[self.key_profiles_dotfiles] + dfs = profile[self.key_profiles_dotfiles] self.log.dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) self.log.dbg('remove {} from profile {}'.format(df_key, pro_key)) - profiles[self.key_profiles_dotfiles].remove(df_key) + profile[self.key_profiles_dotfiles].remove(df_key) if self.debug: - dfs = profiles[self.key_profiles_dotfiles] + dfs = profile[self.key_profiles_dotfiles] self.log.dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) self.dirty = True return True From 274a613b998f96fc8f383c2d5224c355976681f9 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 4 Jun 2019 22:40:44 +0200 Subject: [PATCH 32/58] avoid yaml keys to be re-ordered --- dotdrop/cfg_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 8b84972..630f291 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -751,7 +751,7 @@ class CfgYaml: # ensure no null are displayed data = yaml.safe_dump(content, default_flow_style=False, - indent=2) + indent=2, sort_keys=False) data = data.replace('null', '') # save to file From 4452140200479b8bc6f1ec76af5cb1a6fdcae08b Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 4 Jun 2019 22:52:16 +0200 Subject: [PATCH 33/58] backport issue template --- .github/ISSUE_TEMPLATE/need-help.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/need-help.md diff --git a/.github/ISSUE_TEMPLATE/need-help.md b/.github/ISSUE_TEMPLATE/need-help.md new file mode 100644 index 0000000..5298952 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/need-help.md @@ -0,0 +1,12 @@ +--- +name: Need help +about: Get help +title: "[help]" +labels: help +assignees: '' + +--- + +**What I am trying to achieve** + +**What I have tried so far** From f5a0c65df264ae4a04c00ec90b5e1f8e612970ee Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 4 Jun 2019 23:01:11 +0200 Subject: [PATCH 34/58] typo --- dotdrop/dotdrop.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 5e575d5..86a964b 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -147,9 +147,11 @@ def cmd_install(o): elif not r: # dotfile was NOT installed if o.install_force_action: + # pre-actions LOG.dbg('force pre action execution ...') pre_actions_exec() - LOG.dbg('force post pre action execution ...') + # post-actions + LOG.dbg('force post action execution ...') postactions = dotfile.get_post_actions() prof = o.conf.get_profile(o.profile) postactions.extend(prof.get_post_actions()) From b7f5495f0902b52a98d4530db01d9b8659403232 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 4 Jun 2019 23:42:49 +0200 Subject: [PATCH 35/58] add remove tests and fix bugs --- dotdrop/dotdrop.py | 12 ++-- tests/test_remove.py | 136 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 5 deletions(-) create mode 100644 tests/test_remove.py diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 86a964b..31eb1cf 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -420,17 +420,13 @@ def cmd_remove(o): LOG.log('no dotfile to remove') return False if o.debug: - LOG.dbg('dotfile to remove: {}'.format(paths)) + LOG.dbg('dotfile(s) to remove: {}'.format(','.join(paths))) removed = [] for key in paths: - if o.debug: - LOG.dbg('removing {}'.format(key)) if not iskey: # by path - print(key) dotfile = o.conf.get_dotfile_by_dst(key) - print(dotfile) if not dotfile: LOG.warn('{} ignored, does not exist'.format(key)) continue @@ -438,7 +434,13 @@ def cmd_remove(o): else: # by key dotfile = o.conf.get_dotfile(key) + if not dotfile: + LOG.warn('{} ignored, does not exist'.format(key)) + continue k = key + if o.debug: + LOG.dbg('removing {}'.format(key)) + # make sure is part of the profile if dotfile.key not in [d.key for d in o.dotfiles]: LOG.warn('{} ignored, not associated to this profile'.format(key)) diff --git a/tests/test_remove.py b/tests/test_remove.py new file mode 100644 index 0000000..df01dce --- /dev/null +++ b/tests/test_remove.py @@ -0,0 +1,136 @@ +""" +author: deadc0de6 (https://github.com/deadc0de6) +Copyright (c) 2019, deadc0de6 +basic unittest for the remove function +""" + +import yaml +import unittest +import os + +# local imports +from dotdrop.dotdrop import cmd_remove +from tests.helpers import (clean, create_dir, + create_random_file, load_options, + get_tempdir) + + +class TestRemove(unittest.TestCase): + + def load_yaml(self, path): + """Load yaml to dict""" + self.assertTrue(os.path.exists(path)) + content = '' + with open(path, 'r') as f: + content = yaml.safe_load(f) + return content + + def test_remove(self): + """test the remove command""" + + # dotfiles in dotpath + dotdrop_home = get_tempdir() + self.assertTrue(os.path.exists(dotdrop_home)) + self.addCleanup(clean, dotdrop_home) + + dotfilespath = os.path.join(dotdrop_home, 'dotfiles') + confpath = os.path.join(dotdrop_home, 'config.yaml') + create_dir(dotfilespath) + + df1, _ = create_random_file(dotfilespath) + df2, _ = create_random_file(dotfilespath) + df3, _ = create_random_file(dotfilespath) + configdic = { + 'config': { + 'dotpath': 'dotfiles', + }, + 'dotfiles': { + 'f_test1': { + 'src': df1, + 'dst': '/dev/null' + }, + 'f_test2': { + 'src': df2, + 'dst': '/dev/null' + }, + 'f_test3': { + 'src': df3, + 'dst': '/tmp/some-fake-path' + }, + }, + 'profiles': { + 'host1': { + 'dotfiles': ['f_test1', 'f_test2', 'f_test3'], + }, + 'host2': { + 'dotfiles': ['f_test1'], + }, + 'host3': { + 'dotfiles': ['f_test2'], + }, + }, + } + + with open(confpath, 'w') as f: + yaml.safe_dump(configdic, f) + o = load_options(confpath, 'host1') + o.remove_path = ['f_test1'] + o.remove_iskey = True + o.debug = True + o.safe = False + # by key + cmd_remove(o) + + # ensure file is deleted + self.assertFalse(os.path.exists(df1)) + self.assertTrue(os.path.exists(df2)) + self.assertTrue(os.path.exists(df3)) + + # load dict + y = self.load_yaml(confpath) + + # ensure not present + self.assertTrue('f_test1' not in y['dotfiles']) + self.assertTrue('f_test1' not in y['profiles']['host1']['dotfiles']) + self.assertTrue('host2' not in y['profiles']) + + # assert rest is intact + self.assertTrue('f_test2' in y['dotfiles'].keys()) + self.assertTrue('f_test3' in y['dotfiles'].keys()) + self.assertTrue('f_test2' in y['profiles']['host1']['dotfiles']) + self.assertTrue('f_test3' in y['profiles']['host1']['dotfiles']) + self.assertTrue(y['profiles']['host3']['dotfiles'] == ['f_test2']) + + o = load_options(confpath, 'host1') + o.remove_path = ['/tmp/some-fake-path'] + o.remove_iskey = False + o.debug = True + o.safe = False + # by path + cmd_remove(o) + + # ensure file is deleted + self.assertTrue(os.path.exists(df2)) + self.assertFalse(os.path.exists(df3)) + + # load dict + y = self.load_yaml(confpath) + + # ensure not present + self.assertTrue('f_test3' not in y['dotfiles']) + self.assertTrue('f_test3' not in y['profiles']['host1']['dotfiles']) + + # assert rest is intact + self.assertTrue('host1' in y['profiles'].keys()) + self.assertFalse('host2' in y['profiles'].keys()) + self.assertTrue('host3' in y['profiles'].keys()) + self.assertTrue(y['profiles']['host1']['dotfiles'] == ['f_test2']) + self.assertTrue(y['profiles']['host3']['dotfiles'] == ['f_test2']) + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() From e01b84fa73afaa33ac9be8626654226703bf863c Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 5 Jun 2019 09:13:04 +0200 Subject: [PATCH 36/58] disable sort_keys on yaml only for version >= 5.1 --- dotdrop/cfg_yaml.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 630f291..8b63e70 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -733,6 +733,18 @@ class CfgYaml: new[k] = newv return new + def _yaml_version(self): + """returns True if version >= 5.1""" + minv = [5, 1] + try: + cur = list(map(int, yaml.__version__.split('.'))) + if cur.pop(0) >= minv.pop(0) and \ + cur.pop(1) >= minv.pop(1): + return True + except: + return False + return False + def save(self): """save this instance and return True if saved""" if not self.dirty: @@ -748,10 +760,12 @@ class CfgYaml: if self.key_profiles not in content: content[self.key_profiles] = None + opts = {'default_flow_style': False, 'indent': 2} + if self._yaml_version(): + opts['sort_keys'] = False + data = yaml.safe_dump(content, **opts) + # ensure no null are displayed - data = yaml.safe_dump(content, - default_flow_style=False, - indent=2, sort_keys=False) data = data.replace('null', '') # save to file From cd722e6eb1aeef1f828ec9eab5685e5ca6f947f0 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 5 Jun 2019 09:27:43 +0200 Subject: [PATCH 37/58] fix bare exception for pep8 --- dotdrop/cfg_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 8b63e70..a07d224 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -741,7 +741,7 @@ class CfgYaml: if cur.pop(0) >= minv.pop(0) and \ cur.pop(1) >= minv.pop(1): return True - except: + except Exception: return False return False From d008e6a895aa670cbbedc53362d03709a6a92d1b Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 6 Jun 2019 17:11:13 +0200 Subject: [PATCH 38/58] migrate from PyYAML to ruamel.yaml --- dotdrop/cfg_yaml.py | 57 ++++++++++++++++------------------ dotdrop/dotdrop.py | 5 ++- packages/arch-dotdrop/.SRCINFO | 2 +- packages/arch-dotdrop/PKGBUILD | 2 +- requirements.txt | 2 +- scripts/change-link.py | 4 +-- setup.py | 2 +- tests/helpers.py | 32 ++++++++++++------- tests/test_import.py | 9 ++---- tests/test_remove.py | 15 +++------ tests/test_yamlcfg.py | 17 +++------- 11 files changed, 71 insertions(+), 76 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index a07d224..48949bb 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -6,7 +6,7 @@ handle lower level of the config file """ import os -import yaml +from ruamel.yaml import YAML as yaml import glob from copy import deepcopy @@ -572,12 +572,11 @@ class CfgYaml: content = {} if not os.path.exists(path): raise YamlException('config path not found: {}'.format(path)) - with open(path, 'r') as f: - try: - content = yaml.safe_load(f) - except Exception as e: - self.log.err(e) - raise YamlException('invalid config: {}'.format(path)) + try: + content = self._yaml_load(path) + except Exception as e: + self.log.err(e) + raise YamlException('invalid config: {}'.format(path)) return content def _new_profile(self, key): @@ -733,18 +732,6 @@ class CfgYaml: new[k] = newv return new - def _yaml_version(self): - """returns True if version >= 5.1""" - minv = [5, 1] - try: - cur = list(map(int, yaml.__version__.split('.'))) - if cur.pop(0) >= minv.pop(0) and \ - cur.pop(1) >= minv.pop(1): - return True - except Exception: - return False - return False - def save(self): """save this instance and return True if saved""" if not self.dirty: @@ -760,19 +747,14 @@ class CfgYaml: if self.key_profiles not in content: content[self.key_profiles] = None - opts = {'default_flow_style': False, 'indent': 2} - if self._yaml_version(): - opts['sort_keys'] = False - data = yaml.safe_dump(content, **opts) - - # ensure no null are displayed - data = data.replace('null', '') - # save to file if self.debug: - self.log.dbg('saving: {}'.format(content)) - with open(self.path, 'w') as f: - f.write(data) + self.log.dbg('saving to {}'.format(self.path)) + try: + self._yaml_dump(content, self.path) + except Exception as e: + self.log.err(e) + raise YamlException('error saving config: {}'.format(self.path)) self.dirty = False return True @@ -780,3 +762,18 @@ class CfgYaml: def dump(self): """dump the config dictionary""" return self.yaml_dict + + def _yaml_load(self, path): + """load from yaml""" + with open(path, 'r') as f: + content = yaml(typ='safe').load(f) + return content + + def _yaml_dump(self, content, path): + """dump to yaml""" + with open(self.path, 'w') as f: + y = yaml() + y.default_flow_style = False + y.indent = 2 + y.typ = 'safe' + y.dump(content, f) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 31eb1cf..d60f656 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -472,7 +472,10 @@ def cmd_remove(o): LOG.raw(o.conf.dump()) else: o.conf.save() - LOG.log('\ndotfile(s) removed: {}'.format(','.join(removed))) + if removed: + LOG.log('\ndotfile(s) removed: {}'.format(','.join(removed))) + else: + LOG.log('\nno dotfile removed') return True diff --git a/packages/arch-dotdrop/.SRCINFO b/packages/arch-dotdrop/.SRCINFO index 26b449c..67b5eaf 100644 --- a/packages/arch-dotdrop/.SRCINFO +++ b/packages/arch-dotdrop/.SRCINFO @@ -10,7 +10,7 @@ pkgbase = dotdrop depends = python-setuptools depends = python-jinja depends = python-docopt - depends = python-pyaml + depends = python-ruamel-yaml source = git+https://github.com/deadc0de6/dotdrop.git#tag=v0.28.0 md5sums = SKIP diff --git a/packages/arch-dotdrop/PKGBUILD b/packages/arch-dotdrop/PKGBUILD index 5891d6a..9e4e779 100644 --- a/packages/arch-dotdrop/PKGBUILD +++ b/packages/arch-dotdrop/PKGBUILD @@ -8,7 +8,7 @@ arch=('any') url="https://github.com/deadc0de6/dotdrop" license=('GPL') groups=() -depends=('python' 'python-setuptools' 'python-jinja' 'python-docopt' 'python-pyaml') +depends=('python' 'python-setuptools' 'python-jinja' 'python-docopt' 'python-ruamel-yaml') makedepends=('git') source=("git+https://github.com/deadc0de6/dotdrop.git#tag=v${pkgver}") md5sums=('SKIP') diff --git a/requirements.txt b/requirements.txt index 4fbbcb1..2c6a2c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ Jinja2; python_version >= '3.0' docopt; python_version >= '3.0' -PyYAML; python_version >= '3.0' +ruamel.yaml; python_version >= '3.0' diff --git a/scripts/change-link.py b/scripts/change-link.py index f2ee974..7a0854b 100755 --- a/scripts/change-link.py +++ b/scripts/change-link.py @@ -13,7 +13,7 @@ usage example: from docopt import docopt import sys import os -import yaml +from ruamel.yaml import YAML as yaml USAGE = """ change-link.py @@ -42,7 +42,7 @@ def main(): ignores = args['--ignore'] with open(path, 'r') as f: - content = yaml.safe_load(f) + content = yaml(typ='safe').load(f) for k, v in content[key].items(): if k in ignores: continue diff --git a/setup.py b/setup.py index 73a3a78..455eb35 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ setup( keywords='dotfiles jinja2', packages=find_packages(exclude=['tests*']), - install_requires=['docopt', 'Jinja2', 'PyYAML'], + install_requires=['docopt', 'Jinja2', 'ruamel.yaml'], extras_require={ 'dev': ['check-manifest'], diff --git a/tests/helpers.py b/tests/helpers.py index 7bd57f6..836101b 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -11,7 +11,7 @@ import string import tempfile from unittest import TestCase -import yaml +from ruamel.yaml import YAML as yaml from dotdrop.options import Options from dotdrop.linktypes import LinkTypes @@ -221,9 +221,8 @@ def create_yaml_keyval(pairs, parent_dir=None, top_key=None): if not parent_dir: parent_dir = get_tempdir() - fd, file_name = tempfile.mkstemp(dir=parent_dir, suffix='.yaml', text=True) - with os.fdopen(fd, 'w') as f: - yaml.safe_dump(pairs, f) + _, file_name = tempfile.mkstemp(dir=parent_dir, suffix='.yaml', text=True) + yaml_dump(pairs, file_name) return file_name @@ -234,8 +233,7 @@ def populate_fake_config(config, dotfiles={}, profiles={}, actions={}, is_path = isinstance(config, str) if is_path: config_path = config - with open(config_path) as config_file: - config = yaml.safe_load(config_file) + config = yaml_load(config_path) config['dotfiles'] = dotfiles config['profiles'] = profiles @@ -246,9 +244,7 @@ def populate_fake_config(config, dotfiles={}, profiles={}, actions={}, config['dynvariables'] = dynvariables if is_path: - with open(config_path, 'w') as config_file: - yaml.safe_dump(config, config_file, default_flow_style=False, - indent=2) + yaml_dump(config, config_path) def file_in_yaml(yaml_file, path, link=False): @@ -256,8 +252,7 @@ def file_in_yaml(yaml_file, path, link=False): strip = get_path_strip_version(path) if isinstance(yaml_file, str): - with open(yaml_file) as f: - yaml_conf = yaml.safe_load(f) + yaml_conf = yaml_load(yaml_file) else: yaml_conf = yaml_file @@ -275,3 +270,18 @@ def file_in_yaml(yaml_file, path, link=False): return False return in_src and in_dst and has_link return in_src and in_dst + + +def yaml_load(path): + with open(path, 'r') as f: + content = yaml(typ='safe').load(f) + return content + + +def yaml_dump(content, path): + with open(path, 'w') as f: + y = yaml() + y.default_flow_style = False + y.indent = 2 + y.typ = 'safe' + y.dump(content, f) diff --git a/tests/test_import.py b/tests/test_import.py index 98efdea..801ff55 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -7,7 +7,6 @@ basic unittest for the import function import unittest import os -import yaml from dotdrop.dotdrop import cmd_importer from dotdrop.dotdrop import cmd_list_profiles @@ -18,7 +17,8 @@ from dotdrop.linktypes import LinkTypes from tests.helpers import (clean, create_dir, create_fake_config, create_random_file, edit_content, file_in_yaml, get_path_strip_version, get_string, get_tempdir, - load_options, populate_fake_config) + load_options, populate_fake_config, + yaml_load) class TestImport(unittest.TestCase): @@ -31,10 +31,7 @@ class TestImport(unittest.TestCase): def load_yaml(self, path): """Load yaml to dict""" self.assertTrue(os.path.exists(path)) - content = '' - with open(path, 'r') as f: - content = yaml.safe_load(f) - return content + return yaml_load(path) def assert_file(self, path, o, profile): """Make sure path has been inserted in conf for profile""" diff --git a/tests/test_remove.py b/tests/test_remove.py index df01dce..2c8d262 100644 --- a/tests/test_remove.py +++ b/tests/test_remove.py @@ -4,7 +4,6 @@ Copyright (c) 2019, deadc0de6 basic unittest for the remove function """ -import yaml import unittest import os @@ -12,7 +11,7 @@ import os from dotdrop.dotdrop import cmd_remove from tests.helpers import (clean, create_dir, create_random_file, load_options, - get_tempdir) + get_tempdir, yaml_load, yaml_dump) class TestRemove(unittest.TestCase): @@ -20,10 +19,7 @@ class TestRemove(unittest.TestCase): def load_yaml(self, path): """Load yaml to dict""" self.assertTrue(os.path.exists(path)) - content = '' - with open(path, 'r') as f: - content = yaml.safe_load(f) - return content + return yaml_load(path) def test_remove(self): """test the remove command""" @@ -71,8 +67,7 @@ class TestRemove(unittest.TestCase): }, } - with open(confpath, 'w') as f: - yaml.safe_dump(configdic, f) + yaml_dump(configdic, confpath) o = load_options(confpath, 'host1') o.remove_path = ['f_test1'] o.remove_iskey = True @@ -87,7 +82,7 @@ class TestRemove(unittest.TestCase): self.assertTrue(os.path.exists(df3)) # load dict - y = self.load_yaml(confpath) + y = yaml_load(confpath) # ensure not present self.assertTrue('f_test1' not in y['dotfiles']) @@ -114,7 +109,7 @@ class TestRemove(unittest.TestCase): self.assertFalse(os.path.exists(df3)) # load dict - y = self.load_yaml(confpath) + y = yaml_load(confpath) # ensure not present self.assertTrue('f_test3' not in y['dotfiles']) diff --git a/tests/test_yamlcfg.py b/tests/test_yamlcfg.py index 0a22420..33570b1 100644 --- a/tests/test_yamlcfg.py +++ b/tests/test_yamlcfg.py @@ -8,14 +8,13 @@ basic unittest for the config parser import unittest from unittest.mock import patch import os -import yaml from dotdrop.cfg_yaml import CfgYaml as Cfg from dotdrop.options import Options from dotdrop.linktypes import LinkTypes from tests.helpers import (SubsetTestCase, _fake_args, clean, create_fake_config, create_yaml_keyval, get_tempdir, - populate_fake_config) + populate_fake_config, yaml_load, yaml_dump) class TestConfig(SubsetTestCase): @@ -142,8 +141,7 @@ profiles: create=self.CONFIG_CREATE) # edit the config - with open(confpath, 'r') as f: - content = yaml.safe_load(f) + content = yaml_load(confpath) # adding dotfiles df1key = 'f_vimrc' @@ -162,9 +160,7 @@ profiles: } # save the new config - with open(confpath, 'w') as f: - yaml.safe_dump(content, f, default_flow_style=False, - indent=2) + yaml_dump(content, confpath) # do the tests conf = Cfg(confpath, debug=True) @@ -185,17 +181,14 @@ profiles: # test not existing included profile # edit the config - with open(confpath, 'r') as f: - content = yaml.safe_load(f) + content = yaml_load(confpath) content['profiles'] = { pf1key: {'dotfiles': [df2key], 'include': ['host2']}, pf2key: {'dotfiles': [df1key], 'include': ['host3']} } # save the new config - with open(confpath, 'w') as f: - yaml.safe_dump(content, f, default_flow_style=False, - indent=2) + yaml_dump(content, confpath) # do the tests conf = Cfg(confpath, debug=True) From df9b56214d18be7add3b7701071644d37c8c7cae Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 6 Jun 2019 17:22:40 +0200 Subject: [PATCH 39/58] python 3.4 no longer supported (EOL) --- dotdrop/cfg_yaml.py | 6 +----- packages/arch-dotdrop-git/PKGBUILD | 2 +- requirements.txt | 6 +++--- setup.py | 2 +- tests-requirements.txt | 10 +++++----- 5 files changed, 11 insertions(+), 15 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 48949bb..5320046 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -549,11 +549,7 @@ class CfgYaml: def _merge_dict(self, high, low): """merge low into high""" - # won't work in python3.4 - # return {**low, **high} - new = deepcopy(low) - new.update(high) - return new + return {**low, **high} def _get_entry(self, dic, key, mandatory=True): """return entry from yaml dictionary""" diff --git a/packages/arch-dotdrop-git/PKGBUILD b/packages/arch-dotdrop-git/PKGBUILD index 67ecb31..28d762f 100644 --- a/packages/arch-dotdrop-git/PKGBUILD +++ b/packages/arch-dotdrop-git/PKGBUILD @@ -9,7 +9,7 @@ arch=('any') url="https://github.com/deadc0de6/dotdrop" license=('GPL') groups=() -depends=('python' 'python-setuptools' 'python-jinja' 'python-docopt' 'python-pyaml') +depends=('python' 'python-setuptools' 'python-jinja' 'python-docopt' 'python-ruamel-yaml') makedepends=('git') provides=(dotdrop) conflicts=(dotdrop) diff --git a/requirements.txt b/requirements.txt index 2c6a2c7..212fe0e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -Jinja2; python_version >= '3.0' -docopt; python_version >= '3.0' -ruamel.yaml; python_version >= '3.0' +Jinja2; python_version > '3.4' +docopt; python_version > '3.4' +ruamel.yaml; python_version > '3.4' diff --git a/setup.py b/setup.py index 455eb35..7c6153d 100644 --- a/setup.py +++ b/setup.py @@ -32,9 +32,9 @@ setup( python_requires=REQUIRES_PYTHON, classifiers=[ 'Development Status :: 5 - Production/Stable', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', ], diff --git a/tests-requirements.txt b/tests-requirements.txt index 2172e29..c2325bf 100644 --- a/tests-requirements.txt +++ b/tests-requirements.txt @@ -1,5 +1,5 @@ -pycodestyle; python_version >= '3.0' -nose; python_version >= '3.0' -coverage; python_version >= '3.0' -coveralls; python_version >= '3.0' -pyflakes; python_version >= '3.0' +pycodestyle; python_version > '3.4' +nose; python_version > '3.4' +coverage; python_version > '3.4' +coveralls; python_version > '3.4' +pyflakes; python_version > '3.4' From 73696f1ee3be9dd24ecd3d4add069c1aa47c1932 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 6 Jun 2019 17:22:56 +0200 Subject: [PATCH 40/58] remove python 3.4 on travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index eb13f05..e60a940 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "3.4" - "3.5" - "3.6" - "3.7" From e1f1eadc1fec68154d89bc7fc60029e99a0d7898 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Jun 2019 08:53:02 +0200 Subject: [PATCH 41/58] fix naked actions parsed as pre action instead of post --- dotdrop/cfg_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 5320046..ecc6d0b 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -287,7 +287,7 @@ class CfgYaml: for key, action in v.items(): new[key] = (k, action) else: - new[k] = (self.action_pre, v) + new[k] = (self.action_post, v) return new def _norm_dotfiles(self, dotfiles): From 9df59fda0ba996879854208155e78a37eed6a1b3 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Jun 2019 08:53:19 +0200 Subject: [PATCH 42/58] improve profile action test --- tests-ng/profile-actions.sh | 24 ++++++++++++++++++++---- tests.sh | 1 + 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/tests-ng/profile-actions.sh b/tests-ng/profile-actions.sh index 88d124a..c2c6251 100755 --- a/tests-ng/profile-actions.sh +++ b/tests-ng/profile-actions.sh @@ -74,6 +74,12 @@ dotfiles: f_abc: dst: ${tmpd}/abc src: abc + f_def: + dst: ${tmpd}/def + src: def + f_ghi: + dst: ${tmpd}/ghi + src: ghi profiles: p0: actions: @@ -82,11 +88,15 @@ profiles: - nakedaction dotfiles: - f_abc + - f_def + - f_ghi _EOF #cat ${cfg} # create the dotfile echo "test" > ${tmps}/dotfiles/abc +echo "test" > ${tmps}/dotfiles/def +echo "test" > ${tmps}/dotfiles/ghi # install cd ${ddpath} | ${bin} install -f -c ${cfg} -p p0 -V @@ -97,19 +107,25 @@ cd ${ddpath} | ${bin} install -f -c ${cfg} -p p0 -V [ ! -e ${tmpa}/naked ] && echo 'action not executed' && exit 1 grep pre2 ${tmpa}/pre2 +nb=`wc -l ${tmpa}/pre2 | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "profile action executed multiple times" && exit 1 + grep post2 ${tmpa}/post2 +nb=`wc -l ${tmpa}/post2 | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "profile action executed multiple times" && exit 1 + grep naked ${tmpa}/naked +nb=`wc -l ${tmpa}/naked | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "profile action executed multiple times" && exit 1 # install again cd ${ddpath} | ${bin} install -f -c ${cfg} -p p0 -V # check actions not executed twice -nb=`wc -l ${tmpa}/pre2 | awk '{print $1}'` -[ "${nb}" -gt "1" ] && echo "action executed twice" && exit 1 nb=`wc -l ${tmpa}/post2 | awk '{print $1}'` -[ "${nb}" -gt "1" ] && echo "action executed twice" && exit 1 +[ "${nb}" -gt "1" ] && echo "action post2 executed twice" && exit 1 nb=`wc -l ${tmpa}/naked | awk '{print $1}'` -[ "${nb}" -gt "1" ] && echo "action executed twice" && exit 1 +[ "${nb}" -gt "1" ] && echo "action naked executed twice" && exit 1 ## CLEANING rm -rf ${tmps} ${tmpd} ${tmpa} diff --git a/tests.sh b/tests.sh index 3d2088b..b0049e1 100755 --- a/tests.sh +++ b/tests.sh @@ -49,3 +49,4 @@ PYTHONPATH=dotdrop ${nosebin} -s --with-coverage --cover-package=dotdrop rm -f ${log} } +echo "All test finished successfully" From 1015672a400abfd4f4db32be6490b883ce699b23 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Jun 2019 08:56:27 +0200 Subject: [PATCH 43/58] fix profile actions for #152 --- dotdrop/dotdrop.py | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index d60f656..668a1d7 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -28,7 +28,7 @@ TRANS_SUFFIX = 'trans' ########################################################### -def action_executor(o, dotfile, actions, defactions, templater, post=False): +def action_executor(o, actions, defactions, templater, post=False): """closure for action execution""" def execute(): """ @@ -71,6 +71,10 @@ def action_executor(o, dotfile, actions, defactions, templater, post=False): def cmd_install(o): """install dotfiles for this profile""" dotfiles = o.dotfiles + prof = o.conf.get_profile(o.profile) + pro_pre_actions = prof.get_pre_actions() + pro_post_actions = prof.get_post_actions() + if o.install_keys: # filtered dotfiles to install uniq = uniq_list(o.install_keys) @@ -94,6 +98,15 @@ def cmd_install(o): backup_suffix=o.install_backup_suffix) installed = 0 tvars = t.add_tmp_vars() + + # execute profile pre-action + if o.debug: + LOG.dbg('execute profile pre actions') + ret, err = action_executor(o, pro_pre_actions, [], t, post=False)() + if not ret: + return False + + # install each dotfile for dotfile in dotfiles: # add dotfile variables t.restore_vars(tvars) @@ -103,11 +116,9 @@ def cmd_install(o): preactions = [] if not o.install_temporary: preactions.extend(dotfile.get_pre_actions()) - prof = o.conf.get_profile(o.profile) - preactions.extend(prof.get_pre_actions()) defactions = o.install_default_actions_pre - pre_actions_exec = action_executor(o, dotfile, preactions, - defactions, t, post=False) + pre_actions_exec = action_executor(o, preactions, defactions, + t, post=False) if o.debug: LOG.dbg('installing {}'.format(dotfile)) @@ -138,29 +149,35 @@ def cmd_install(o): if not o.install_temporary: defactions = o.install_default_actions_post postactions = dotfile.get_post_actions() - prof = o.conf.get_profile(o.profile) - postactions.extend(prof.get_post_actions()) - post_actions_exec = action_executor(o, dotfile, postactions, - defactions, t, post=True) + post_actions_exec = action_executor(o, postactions, defactions, + t, post=True) post_actions_exec() installed += 1 elif not r: # dotfile was NOT installed if o.install_force_action: # pre-actions - LOG.dbg('force pre action execution ...') + if o.debug: + LOG.dbg('force pre action execution ...') pre_actions_exec() # post-actions LOG.dbg('force post action execution ...') postactions = dotfile.get_post_actions() - prof = o.conf.get_profile(o.profile) - postactions.extend(prof.get_post_actions()) - post_actions_exec = action_executor(o, dotfile, postactions, - defactions, t, post=True) + post_actions_exec = action_executor(o, postactions, defactions, + t, post=True) post_actions_exec() if err: LOG.err('installing \"{}\" failed: {}'.format(dotfile.key, err)) + + # execute profile post-action + if installed > 0 or o.install_force_action: + if o.debug: + LOG.dbg('execute profile post actions') + ret, err = action_executor(o, pro_post_actions, [], t, post=False)() + if not ret: + return False + if o.install_temporary: LOG.log('\ninstalled to tmp \"{}\".'.format(tmpdir)) LOG.log('\n{} dotfile(s) installed.'.format(installed)) From a89fae29cf5bd629d7d6f9927bb85e09f5646557 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Jun 2019 09:55:08 +0200 Subject: [PATCH 44/58] inherit actions from profile include (for #152) --- dotdrop/cfg_yaml.py | 83 ++++++++++------ tests-ng/include-actions.sh | 183 ++++++++++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 27 deletions(-) create mode 100755 tests-ng/include-actions.sh diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index ecc6d0b..58aa046 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -36,7 +36,6 @@ class CfgYaml: action_post = 'post' # profiles/dotfiles entries - key_profiles_dotfiles = 'dotfiles' key_dotfile_src = 'src' key_dotfile_dst = 'dst' key_dotfile_link = 'link' @@ -44,9 +43,11 @@ class CfgYaml: key_dotfile_link_children = 'link_children' # profile + key_profile_dotfiles = 'dotfiles' key_profile_include = 'include' key_profile_variables = 'variables' key_profile_dvariables = 'dynvariables' + key_profile_actions = 'actions' key_all = 'ALL' # import entries @@ -470,44 +471,72 @@ class CfgYaml: """resolve some other parts of the config""" # profile -> ALL for k, v in self.profiles.items(): - dfs = v.get(self.key_profiles_dotfiles, None) + dfs = v.get(self.key_profile_dotfiles, None) if not dfs: continue if self.debug: self.log.dbg('add ALL to profile {}'.format(k)) if self.key_all in dfs: - v[self.key_profiles_dotfiles] = self.dotfiles.keys() + v[self.key_profile_dotfiles] = self.dotfiles.keys() # profiles -> include other profile for k, v in self.profiles.items(): self._rec_resolve_profile_include(k) def _rec_resolve_profile_include(self, profile): - """recursively resolve include of other profiles's dotfiles""" - values = self.profiles[profile] - current = values.get(self.key_profiles_dotfiles, []) - inc = values.get(self.key_profile_include, None) - if not inc: - return current + """ + recursively resolve include of other profiles's: + * dotfiles + * actions + """ + this_profile = self.profiles[profile] + + # include + dotfiles = this_profile.get(self.key_profile_dotfiles, []) + actions = this_profile.get(self.key_profile_actions, []) + includes = this_profile.get(self.key_profile_include, None) + if not includes: + # nothing to include + return dotfiles, actions + if self.debug: + self.log.dbg('{} includes: {}'.format(profile, ','.join(includes))) + self.log.dbg('{} dotfiles before include: {}'.format(profile, + dotfiles)) + self.log.dbg('{} actions before include: {}'.format(profile, + actions)) + seen = [] - for i in inc: + for i in uniq_list(includes): + # ensure no include loop occurs if i in seen: raise YamlException('\"include loop\"') seen.append(i) + # included profile even exists if i not in self.profiles.keys(): self.log.warn('include unknown profile: {}'.format(i)) continue - p = self.profiles[i] - others = p.get(self.key_profiles_dotfiles, []) - if self.key_profile_include in p.keys(): - others.extend(self._rec_resolve_profile_include(i)) - current.extend(others) - # unique them - values[self.key_profiles_dotfiles] = uniq_list(current) + # recursive resolve + o_dfs, o_actions = self._rec_resolve_profile_include(i) + # merge dotfile keys + dotfiles.extend(o_dfs) + this_profile[self.key_profile_dotfiles] = uniq_list(dotfiles) + # merge actions keys + actions.extend(o_actions) + this_profile[self.key_profile_actions] = uniq_list(actions) + + dotfiles = this_profile.get(self.key_profile_dotfiles, []) + actions = this_profile.get(self.key_profile_actions, []) if self.debug: - dfs = values[self.key_profiles_dotfiles] - self.log.dbg('{} dfs after include: {}'.format(profile, dfs)) - return values.get(self.key_profiles_dotfiles, []) + self.log.dbg('{} dotfiles after include: {}'.format(profile, + dotfiles)) + self.log.dbg('{} actions after include: {}'.format(profile, + actions)) + + # since dotfiles and actions are resolved here + # and variables have been already done at the beginning + # of the parsing, we can clear these include + self.profiles[profile][self.key_profile_include] = None + return dotfiles, actions def _resolve_path(self, path): """resolve a path either absolute or relative to config path""" @@ -580,7 +609,7 @@ class CfgYaml: if key not in self.profiles.keys(): # update yaml_dict self.yaml_dict[self.key_profiles][key] = { - self.key_profiles_dotfiles: [] + self.key_profile_dotfiles: [] } if self.debug: self.log.dbg('adding new profile: {}'.format(key)) @@ -590,8 +619,8 @@ class CfgYaml: """add an existing dotfile key to a profile_key""" self._new_profile(profile_key) profile = self.yaml_dict[self.key_profiles][profile_key] - if dotfile_key not in profile[self.key_profiles_dotfiles]: - profile[self.key_profiles_dotfiles].append(dotfile_key) + if dotfile_key not in profile[self.key_profile_dotfiles]: + profile[self.key_profile_dotfiles].append(dotfile_key) if self.debug: msg = 'add \"{}\" to profile \"{}\"'.format(dotfile_key, profile_key) @@ -641,15 +670,15 @@ class CfgYaml: return False # get the profile dictionary profile = self.yaml_dict[self.key_profiles][pro_key] - if df_key not in profile[self.key_profiles_dotfiles]: + if df_key not in profile[self.key_profile_dotfiles]: return True if self.debug: - dfs = profile[self.key_profiles_dotfiles] + dfs = profile[self.key_profile_dotfiles] self.log.dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) self.log.dbg('remove {} from profile {}'.format(df_key, pro_key)) - profile[self.key_profiles_dotfiles].remove(df_key) + profile[self.key_profile_dotfiles].remove(df_key) if self.debug: - dfs = profile[self.key_profiles_dotfiles] + dfs = profile[self.key_profile_dotfiles] self.log.dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) self.dirty = True return True diff --git a/tests-ng/include-actions.sh b/tests-ng/include-actions.sh new file mode 100755 index 0000000..6aff90d --- /dev/null +++ b/tests-ng/include-actions.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2019, deadc0de6 +# +# test the use of the keyword "include" +# with action inheritance +# returns 1 in case of error +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "\e[96m\e[1m==> RUNNING $(basename $BASH_SOURCE) <==\e[0m" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests'` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests'` +# the action temp +tmpa=`mktemp -d --suffix='-dotdrop-tests'` + +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +actions: + pre: + preaction: echo 'pre' >> ${tmpa}/pre + preaction2: echo 'pre2' >> ${tmpa}/pre2 + post: + postaction: echo 'post' >> ${tmpa}/post + postaction2: echo 'post2' >> ${tmpa}/post2 + nakedaction: echo 'naked' >> ${tmpa}/naked +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: + f_abc: + dst: ${tmpd}/abc + src: abc +profiles: + p0: + include: + - p3 + p1: + dotfiles: + - f_abc + actions: + - preaction + - postaction + p2: + include: + - p1 + actions: + - preaction2 + - postaction2 + p3: + include: + - p2 + actions: + - nakedaction +_EOF + +# create the source +mkdir -p ${tmps}/dotfiles/ +echo "test" > ${tmps}/dotfiles/abc + +# install +echo "PROFILE p2" +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p2 -V + +# checks +[ ! -e ${tmpa}/pre ] && echo "pre not found" && exit 1 +nb=`wc -l ${tmpa}/pre | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "pre executed multiple times" && exit 1 + +[ ! -e ${tmpa}/pre2 ] && echo "pre2 not found" && exit 1 +nb=`wc -l ${tmpa}/pre2 | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "pre2 executed multiple times" && exit 1 + +[ ! -e ${tmpa}/post ] && echo "post not found" && exit 1 +nb=`wc -l ${tmpa}/post | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "post executed multiple times" && exit 1 + +[ ! -e ${tmpa}/post2 ] && echo "post2 not found" && exit 1 +nb=`wc -l ${tmpa}/post2 | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "post2 executed multiple times" && exit 1 + +# install +rm -f ${tmpa}/pre ${tmpa}/pre2 ${tmpa}/post ${tmpa}/post2 ${tmpa}/naked +rm -f ${tmpd}/abc +echo "PROFILE p3" +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p3 -V + +# checks +[ ! -e ${tmpa}/pre ] && echo "pre not found" && exit 1 +nb=`wc -l ${tmpa}/pre | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "pre executed multiple times" && exit 1 + +[ ! -e ${tmpa}/pre2 ] && echo "pre2 not found" && exit 1 +nb=`wc -l ${tmpa}/pre2 | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "pre2 executed multiple times" && exit 1 + +[ ! -e ${tmpa}/post ] && echo "post not found" && exit 1 +nb=`wc -l ${tmpa}/post | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "post executed multiple times" && exit 1 + +[ ! -e ${tmpa}/post2 ] && echo "post2 not found" && exit 1 +nb=`wc -l ${tmpa}/post2 | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "post2 executed multiple times" && exit 1 + +[ ! -e ${tmpa}/naked ] && echo "naked not found" && exit 1 +nb=`wc -l ${tmpa}/naked | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "naked executed multiple times" && exit 1 + +# install +rm -f ${tmpa}/pre ${tmpa}/pre2 ${tmpa}/post ${tmpa}/post2 ${tmpa}/naked +rm -f ${tmpd}/abc +echo "PROFILE p0" +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p0 -V + +# checks +[ ! -e ${tmpa}/pre ] && echo "pre not found" && exit 1 +nb=`wc -l ${tmpa}/pre | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "pre executed multiple times" && exit 1 + +[ ! -e ${tmpa}/pre2 ] && echo "pre2 not found" && exit 1 +nb=`wc -l ${tmpa}/pre2 | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "pre2 executed multiple times" && exit 1 + +[ ! -e ${tmpa}/post ] && echo "post not found" && exit 1 +nb=`wc -l ${tmpa}/post | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "post executed multiple times" && exit 1 + +[ ! -e ${tmpa}/post2 ] && echo "post2 not found" && exit 1 +nb=`wc -l ${tmpa}/post2 | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "post2 executed multiple times" && exit 1 + +[ ! -e ${tmpa}/naked ] && echo "naked not found" && exit 1 +nb=`wc -l ${tmpa}/naked | awk '{print $1}'` +[ "${nb}" != "1" ] && echo "naked executed multiple times" && exit 1 + +## CLEANING +rm -rf ${tmps} ${tmpd} ${tmpa} + +echo "OK" +exit 0 From 469a827f7d3be3373de65c0230ae28d9c7be68c7 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 8 Jun 2019 11:47:30 +0200 Subject: [PATCH 45/58] fix relative cmpignore for #149 --- dotdrop/comparator.py | 8 +++--- dotdrop/dotdrop.py | 2 +- tests-ng/compare-ignore-relative.sh | 42 +++++++++-------------------- 3 files changed, 18 insertions(+), 34 deletions(-) diff --git a/dotdrop/comparator.py b/dotdrop/comparator.py index eda8387..bbf964e 100644 --- a/dotdrop/comparator.py +++ b/dotdrop/comparator.py @@ -115,9 +115,9 @@ class Comparator: def _diff(self, left, right, header=False): """diff using the unix tool diff""" - d = diff(left, right, raw=False, - opts=self.diffopts, debug=self.debug) + out = diff(left, right, raw=False, + opts=self.diffopts, debug=self.debug) if header: lshort = os.path.basename(left) - d = '=> diff \"{}\":\n{}'.format(lshort, diff) - return d + out = '=> diff \"{}\":\n{}'.format(lshort, out) + return out diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 668a1d7..ceeef8d 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -245,7 +245,7 @@ def cmd_compare(o, tmp): same = False continue ignores = list(set(o.compare_ignore + dotfile.cmpignore)) - ignores = patch_ignores(ignores, dotfile.src) + ignores = patch_ignores(ignores, dotfile.dst) diff = comp.compare(insttmp, dotfile.dst, ignore=ignores) if tmpsrc: # clean tmp transformed dotfile if any diff --git a/tests-ng/compare-ignore-relative.sh b/tests-ng/compare-ignore-relative.sh index 519d8bf..46d6779 100755 --- a/tests-ng/compare-ignore-relative.sh +++ b/tests-ng/compare-ignore-relative.sh @@ -54,10 +54,9 @@ echo "[+] dotpath dir: ${basedir}/dotfiles" tmpd=`mktemp -d --suffix='-dotdrop-tests'` # some files -mkdir -p ${tmpd}/{program,config} +mkdir -p ${tmpd}/{program,config,vscode} touch ${tmpd}/program/a touch ${tmpd}/config/a -mkdir ${tmpd}/vscode touch ${tmpd}/vscode/extensions.txt touch ${tmpd}/vscode/keybindings.json @@ -71,51 +70,35 @@ cd ${ddpath} | ${bin} import -c ${cfg} ${tmpd}/program cd ${ddpath} | ${bin} import -c ${cfg} ${tmpd}/config cd ${ddpath} | ${bin} import -c ${cfg} ${tmpd}/vscode -# add files +# add files on filesystem echo "[+] add files" touch ${tmpd}/program/b touch ${tmpd}/config/b # expects diff -echo "[+] comparing normal - 2 diffs" +echo "[+] comparing normal - diffs expected" set +e cd ${ddpath} | ${bin} compare -c ${cfg} --verbose -[ "$?" = "0" ] && exit 1 +ret="$?" +echo ${ret} +[ "${ret}" = "0" ] && exit 1 set -e # expects one diff -patt="${tmpd}/config/b" -echo "[+] comparing with ignore (pattern: ${patt}) - 1 diff" -set +e -cd ${ddpath} | ${bin} compare -c ${cfg} --verbose --ignore=${patt} -[ "$?" = "0" ] && exit 1 -set -e - -# expects no diff -patt="*b" -echo "[+] comparing with ignore (pattern: ${patt}) - 0 diff" +patt="b" +echo "[+] comparing with ignore (pattern: ${patt}) - no diff expected" set +e cd ${ddpath} | ${bin} compare -c ${cfg} --verbose --ignore=${patt} [ "$?" != "0" ] && exit 1 set -e -# expects one diff -patt="*/config/*b" -echo "[+] comparing with ignore (pattern: ${patt}) - 1 diff" -set +e -cd ${ddpath} | ${bin} compare -c ${cfg} --verbose --ignore=${patt} -[ "$?" = "0" ] && exit 1 -set -e - -#cat ${cfg} - # adding ignore in dotfile cfg2="${basedir}/config2.yaml" -sed '/d_config:/a \ \ \ \ cmpignore:\n\ \ \ \ - "config/b"' ${cfg} > ${cfg2} +sed '/d_config:/a \ \ \ \ cmpignore:\n\ \ \ \ - "b"' ${cfg} > ${cfg2} #cat ${cfg2} # expects one diff -echo "[+] comparing with ignore in dotfile - 1 diff" +echo "[+] comparing with ignore in dotfile - diff expected" set +e cd ${ddpath} | ${bin} compare -c ${cfg2} --verbose [ "$?" = "0" ] && exit 1 @@ -128,8 +111,7 @@ sed -i '/d_program:/a \ \ \ \ cmpignore:\n\ \ \ \ - "b"' ${cfg2} #cat ${cfg2} # expects no diff -patt="*b" -echo "[+] comparing with ignore in dotfile - 0 diff" +echo "[+] comparing with ignore in dotfile - no diff expected" set +e cd ${ddpath} | ${bin} compare -c ${cfg2} --verbose [ "$?" != "0" ] && exit 1 @@ -140,12 +122,14 @@ echo touched > ${tmpd}/vscode/extensions.txt echo touched > ${tmpd}/vscode/keybindings.json # expect two diffs +echo "[+] comparing - diff expected" set +e cd ${ddpath} | ${bin} compare -c ${cfg} --verbose -C ${tmpd}/vscode [ "$?" = "0" ] && exit 1 set -e # expects no diff +echo "[+] comparing with ignore in dotfile - no diff expected" sed '/d_vscode:/a \ \ \ \ cmpignore:\n\ \ \ \ - "extensions.txt"\n\ \ \ \ - "keybindings.json"' ${cfg} > ${cfg2} set +e cd ${ddpath} | ${bin} compare -c ${cfg2} --verbose -C ${tmpd}/vscode From 41f0e018fcb236c1bcfeb8ef23551eaf70663f18 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 9 Jun 2019 12:54:56 +0200 Subject: [PATCH 46/58] adding symlink test --- tests-ng/symlink.sh | 221 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100755 tests-ng/symlink.sh diff --git a/tests-ng/symlink.sh b/tests-ng/symlink.sh new file mode 100755 index 0000000..f68dbe8 --- /dev/null +++ b/tests-ng/symlink.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2019, deadc0de6 +# +# test symlinking dotfiles +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "\e[96m\e[1m==> RUNNING $(basename $BASH_SOURCE) <==\e[0m" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests'` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests'` +#echo "dotfile destination: ${tmpd}" + +################################################## +# test symlink directory +################################################## +# create the dotfile +mkdir -p ${tmps}/dotfiles/abc +echo "file1" > ${tmps}/dotfiles/abc/file1 +echo "file2" > ${tmps}/dotfiles/abc/file2 + +# create a shell script +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + link_dotfile_default: nolink +dotfiles: + d_abc: + dst: ${tmpd}/abc + src: abc + link: link +profiles: + p1: + dotfiles: + - d_abc +_EOF +#cat ${cfg} + +# install +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V +#cat ${cfg} + +# ensure exists and is link +[ ! -h ${tmpd}/abc ] && echo "not a symlink" && exit 1 +[ ! -e ${tmpd}/abc/file1 ] && echo "does not exist" && exit 1 +[ ! -e ${tmpd}/abc/file2 ] && echo "does not exist" && exit 1 + +################################################## +# test symlink files +################################################## +# clean +rm -rf ${tmps}/dotfiles ${tmpd}/abc + +# create the dotfiles +mkdir -p ${tmps}/dotfiles/ +echo "abc" > ${tmps}/dotfiles/abc + +# create a shell script +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + link_dotfile_default: nolink +dotfiles: + f_abc: + dst: ${tmpd}/abc + src: abc + link: link +profiles: + p1: + dotfiles: + - f_abc +_EOF +#cat ${cfg} + +# install +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V +#cat ${cfg} + +# ensure exists and is link +[ ! -h ${tmpd}/abc ] && echo "not a symlink" && exit 1 + +################################################## +# test link_children +################################################## +# clean +rm -rf ${tmps}/dotfiles ${tmpd}/abc + +# create the dotfile +mkdir -p ${tmps}/dotfiles/abc +echo "file1" > ${tmps}/dotfiles/abc/file1 +echo "file2" > ${tmps}/dotfiles/abc/file2 + +# create a shell script +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + link_dotfile_default: nolink +dotfiles: + d_abc: + dst: ${tmpd}/abc + src: abc + link: link_children +profiles: + p1: + dotfiles: + - d_abc +_EOF +#cat ${cfg} + +# install +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V +#cat ${cfg} + +# ensure exists and is link +[ ! -d ${tmpd}/abc ] && echo "not a symlink" && exit 1 +[ ! -h ${tmpd}/abc/file1 ] && echo "does not exist" && exit 1 +[ ! -h ${tmpd}/abc/file2 ] && echo "does not exist" && exit 1 + +################################################## +# test link_children with templates +################################################## +# clean +rm -rf ${tmps}/dotfiles ${tmpd}/abc + +# create the dotfile +mkdir -p ${tmps}/dotfiles/abc +echo "{{@@ profile @@}}" > ${tmps}/dotfiles/abc/file1 +echo "file2" > ${tmps}/dotfiles/abc/file2 + +# create a shell script +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + link_dotfile_default: nolink +dotfiles: + d_abc: + dst: ${tmpd}/abc + src: abc + link: link_children +profiles: + p1: + dotfiles: + - d_abc +_EOF +#cat ${cfg} + +# install +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V +#cat ${cfg} + +# ensure exists and is link +[ ! -d ${tmpd}/abc ] && echo "not a symlink" && exit 1 +[ ! -h ${tmpd}/abc/file1 ] && echo "does not exist" && exit 1 +[ ! -h ${tmpd}/abc/file2 ] && echo "does not exist" && exit 1 +grep '^p1$' ${tmpd}/abc/file1 + +## CLEANING +rm -rf ${tmps} ${tmpd} ${scr} + +echo "OK" +exit 0 From bbe6bc76fe747e59120f437e9f28a39572d5cac3 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 9 Jun 2019 13:02:01 +0200 Subject: [PATCH 47/58] backport bug fix --- dotdrop/dotdrop.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index ceeef8d..ff91b31 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -123,12 +123,12 @@ def cmd_install(o): if o.debug: LOG.dbg('installing {}'.format(dotfile)) if hasattr(dotfile, 'link') and dotfile.link == LinkTypes.LINK: - r = inst.link(t, dotfile.src, dotfile.dst, - actionexec=pre_actions_exec) + r, err = inst.link(t, dotfile.src, dotfile.dst, + actionexec=pre_actions_exec) elif hasattr(dotfile, 'link') and \ dotfile.link == LinkTypes.LINK_CHILDREN: - r = inst.link_children(t, dotfile.src, dotfile.dst, - actionexec=pre_actions_exec) + r, err = inst.link_children(t, dotfile.src, dotfile.dst, + actionexec=pre_actions_exec) else: src = dotfile.src tmp = None From acc2c83e226f886a2de74bb902fdac5bd7e5604a Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 9 Jun 2019 13:44:18 +0200 Subject: [PATCH 48/58] fix relative cmpignore for #149 --- dotdrop/dotdrop.py | 7 ++++- dotdrop/updater.py | 2 +- dotdrop/utils.py | 17 +++++++---- tests-ng/compare-ignore-relative.sh | 45 +++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 7 deletions(-) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index ff91b31..98d442c 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -221,6 +221,8 @@ def cmd_compare(o, tmp): tmpsrc = None if dotfile.trans_r: # apply transformation + if o.debug: + LOG.dbg('applying transformation before comparing') tmpsrc = apply_trans(o.dotpath, dotfile, debug=o.debug) if not tmpsrc: # could not apply trans @@ -245,7 +247,7 @@ def cmd_compare(o, tmp): same = False continue ignores = list(set(o.compare_ignore + dotfile.cmpignore)) - ignores = patch_ignores(ignores, dotfile.dst) + ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) diff = comp.compare(insttmp, dotfile.dst, ignore=ignores) if tmpsrc: # clean tmp transformed dotfile if any @@ -570,6 +572,9 @@ def main(): LOG.err('config file error: {}'.format(str(e))) return False + if o.debug: + LOG.dbg('\n\n') + ret = True try: diff --git a/dotdrop/updater.py b/dotdrop/updater.py index 48df214..b19fa3f 100644 --- a/dotdrop/updater.py +++ b/dotdrop/updater.py @@ -78,7 +78,7 @@ class Updater: ret = False new_path = None ignores = list(set(self.ignore + dotfile.upignore)) - self.ignores = patch_ignores(ignores, dotfile.dst) + self.ignores = patch_ignores(ignores, dotfile.dst, debug=self.debug) if self.debug: self.log.dbg('ignore pattern(s): {}'.format(self.ignores)) diff --git a/dotdrop/utils.py b/dotdrop/utils.py index ac36a74..9300d9b 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -124,6 +124,8 @@ def must_ignore(paths, ignores, debug=False): """return true if any paths in list matches any ignore patterns""" if not ignores: return False + if debug: + LOG.dbg('must ignore? {} against {}'.format(paths, ignores)) for p in paths: for i in ignores: if fnmatch.fnmatch(p, i): @@ -142,19 +144,24 @@ def uniq_list(a_list): return new -def patch_ignores(ignores, prefix): +def patch_ignores(ignores, prefix, debug=False): """allow relative ignore pattern""" new = [] + if debug: + LOG.dbg('ignores before patching: {}'.format(ignores)) for ignore in ignores: - if STAR in ignore: - # is glob - new.append(ignore) - continue if os.path.isabs(ignore): # is absolute new.append(ignore) continue + if STAR in ignore: + if ignore.startswith(STAR) or ignore.startswith(os.sep): + # is glob + new.append(ignore) + continue # patch upignore path = os.path.join(prefix, ignore) new.append(path) + if debug: + LOG.dbg('ignores after patching: {}'.format(new)) return new diff --git a/tests-ng/compare-ignore-relative.sh b/tests-ng/compare-ignore-relative.sh index 46d6779..8d18c22 100755 --- a/tests-ng/compare-ignore-relative.sh +++ b/tests-ng/compare-ignore-relative.sh @@ -136,6 +136,51 @@ cd ${ddpath} | ${bin} compare -c ${cfg2} --verbose -C ${tmpd}/vscode [ "$?" != "0" ] && exit 1 set -e +#################### +# test for #149 +#################### +mkdir -p ${tmpd}/.zsh +touch ${tmpd}/.zsh/somefile +mkdir -p ${tmpd}/.zsh/plugins +touch ${tmpd}/.zsh/plugins/someplugin + +echo "[+] import .zsh" +cd ${ddpath} | ${bin} import -c ${cfg} ${tmpd}/.zsh + +# no diff expected +echo "[+] comparing .zsh" +cd ${ddpath} | ${bin} compare -c ${cfg} --verbose -C ${tmpd}/.zsh --ignore=${patt} +[ "$?" != "0" ] && exit 1 + +# add some files +touch ${tmpd}/.zsh/plugins/ignore-1.zsh +touch ${tmpd}/.zsh/plugins/ignore-2.zsh + +# expects diff +echo "[+] comparing .zsh with new files" +set +e +cd ${ddpath} | ${bin} compare -c ${cfg} --verbose -C ${tmpd}/.zsh +ret="$?" +echo ${ret} +[ "${ret}" = "0" ] && exit 1 +set -e + +# expects no diff +patt="plugins/ignore-*.zsh" +echo "[+] comparing with ignore (pattern: ${patt}) - no diff expected" +set +e +cd ${ddpath} | ${bin} compare -c ${cfg} --verbose -C ${tmpd}/.zsh --ignore=${patt} +[ "$?" != "0" ] && exit 1 +set -e + +# expects no diff +echo "[+] comparing with ignore in dotfile - no diff expected" +sed '/d_zsh:/a \ \ \ \ cmpignore:\n\ \ \ \ - "plugins/ignore-*.zsh"' ${cfg} > ${cfg2} +set +e +cd ${ddpath} | ${bin} compare -c ${cfg2} --verbose -C ${tmpd}/.zsh +[ "$?" != "0" ] && exit 1 +set -e + ## CLEANING rm -rf ${basedir} ${tmpd} From 9df3522a173d44db194c22dfa2357e04a0a8545d Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 9 Jun 2019 17:50:06 +0200 Subject: [PATCH 49/58] trans gets replaced with trans_read in dotfiles --- dotdrop/cfg_aggregator.py | 3 ++- dotdrop/cfg_yaml.py | 10 +++++++++- dotdrop/dotfile.py | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dotdrop/cfg_aggregator.py b/dotdrop/cfg_aggregator.py index fc751d6..4074fa8 100644 --- a/dotdrop/cfg_aggregator.py +++ b/dotdrop/cfg_aggregator.py @@ -124,7 +124,8 @@ class CfgAggregator: for k in okeys: o = get_by_key(k) if not o: - err = 'bad key for \"{}\": {}'.format(c.key, k) + err = 'bad {} key for \"{}\": {}'.format(keys, c.key, k) + self.log.err(err) raise Exception(err) objects.append(o) if self.debug: diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 58aa046..4dfc8db 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -292,16 +292,24 @@ class CfgYaml: return new def _norm_dotfiles(self, dotfiles): - """add 'src' as 'key if not present""" + """normalize dotfiles entries""" if not dotfiles: return dotfiles new = {} for k, v in dotfiles.items(): + # add 'src' as key' if not present if self.key_dotfile_src not in v: v[self.key_dotfile_src] = k new[k] = v else: new[k] = v + # fix deprecated trans key + if self.old_key_trans_r in k: + msg = '\"trans\" is deprecated, please use \"trans_read\"' + self.log.warn(msg) + v[self.key_trans_r] = v[self.old_key_trans_r].copy() + del v[self.old_key_trans_r] + new[k] = v return new def _get_variables_dict(self, profile, seen, sub=False): diff --git a/dotdrop/dotfile.py b/dotdrop/dotfile.py index ea6a384..333ce74 100644 --- a/dotdrop/dotfile.py +++ b/dotdrop/dotfile.py @@ -14,7 +14,7 @@ class Dotfile(DictParser): """Represent a dotfile.""" # dotfile keys key_noempty = 'ignoreempty' - key_trans_r = 'trans' + key_trans_r = 'trans_read' key_trans_w = 'trans_write' def __init__(self, key, dst, src, From 9d0c30e63319a47672e45830c2e111de028bf99d Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 9 Jun 2019 18:04:34 +0200 Subject: [PATCH 50/58] migrate trans to trans_read --- dotdrop/cfg_yaml.py | 11 +++++++---- tests/helpers.py | 2 +- tests/test_import.py | 4 ++-- tests/test_install.py | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 4dfc8db..41e7369 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -84,7 +84,7 @@ class CfgYaml: self._fix_deprecated(self.yaml_dict) self._parse_main_yaml(self.yaml_dict) if self.debug: - self.log.dbg('current dict: {}'.format(self.yaml_dict)) + self.log.dbg('before normalization: {}'.format(self.yaml_dict)) # resolve variables allvars = self._merge_and_apply_variables() @@ -97,6 +97,8 @@ class CfgYaml: self._resolve_rest() # patch dotfiles paths self._resolve_dotfile_paths() + if self.debug: + self.log.dbg('after normalization: {}'.format(self.yaml_dict)) def _parse_main_yaml(self, dic): """parse the different blocks""" @@ -142,7 +144,8 @@ class CfgYaml: key = self.key_trans_r if self.old_key_trans_r in dic: self.log.warn('\"trans\" is deprecated, please use \"trans_read\"') - key = self.old_key_trans_r + dic[self.key_trans_r] = dic[self.old_key_trans_r] + del dic[self.old_key_trans_r] self.ori_trans_r = self._get_entry(dic, key, mandatory=False) self.trans_r = deepcopy(self.ori_trans_r) if self.debug: @@ -304,10 +307,10 @@ class CfgYaml: else: new[k] = v # fix deprecated trans key - if self.old_key_trans_r in k: + if self.old_key_trans_r in v: msg = '\"trans\" is deprecated, please use \"trans_read\"' self.log.warn(msg) - v[self.key_trans_r] = v[self.old_key_trans_r].copy() + v[self.key_trans_r] = v[self.old_key_trans_r] del v[self.old_key_trans_r] new[k] = v return new diff --git a/tests/helpers.py b/tests/helpers.py index 836101b..80cc8d0 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -238,7 +238,7 @@ def populate_fake_config(config, dotfiles={}, profiles={}, actions={}, config['dotfiles'] = dotfiles config['profiles'] = profiles config['actions'] = actions - config['trans'] = trans + config['trans_read'] = trans config['trans_write'] = trans_write config['variables'] = variables config['dynvariables'] = dynvariables diff --git a/tests/test_import.py b/tests/test_import.py index 801ff55..08df58c 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -349,7 +349,7 @@ class TestImport(unittest.TestCase): self.assertFalse(any(a.endswith('ing') for a in actions)) # testing transformations - transformations = y['trans'].keys() + transformations = y['trans_read'].keys() self.assertTrue(all(t.endswith('ed') for t in transformations)) self.assertFalse(any(t.endswith('ing') for t in transformations)) transformations = y['trans_write'].keys() @@ -391,7 +391,7 @@ class TestImport(unittest.TestCase): self.assertFalse(any(action.endswith('ed') for action in actions)) # testing transformations - transformations = y['trans'].keys() + transformations = y['trans_read'].keys() self.assertTrue(all(t.endswith('ing') for t in transformations)) self.assertFalse(any(t.endswith('ed') for t in transformations)) transformations = y['trans_write'].keys() diff --git a/tests/test_install.py b/tests/test_install.py index af68d97..5ba35cb 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -65,7 +65,7 @@ exec bspwm f.write(' - {}\n'.format(action.key)) if d.trans_r: for tr in d.trans_r: - f.write(' trans: {}\n'.format(tr.key)) + f.write(' trans_read: {}\n'.format(tr.key)) f.write('profiles:\n') f.write(' {}:\n'.format(profile)) f.write(' dotfiles:\n') From 7bd5d7fd45e0192e063d98f78cc85a8bc5d18dab Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 10 Jun 2019 20:36:58 +0200 Subject: [PATCH 51/58] fix bug for #157 --- dotdrop/updater.py | 9 ++++----- tests-ng/transformations.sh | 7 +++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/dotdrop/updater.py b/dotdrop/updater.py index b19fa3f..78c8634 100644 --- a/dotdrop/updater.py +++ b/dotdrop/updater.py @@ -93,13 +93,12 @@ class Updater: new_path = self._apply_trans_w(path, dotfile) if not new_path: return False - path = new_path - if os.path.isdir(path): - ret = self._handle_dir(path, dtpath) + if os.path.isdir(new_path): + ret = self._handle_dir(new_path, dtpath) else: - ret = self._handle_file(path, dtpath) + ret = self._handle_file(new_path, dtpath) # clean temporary files - if new_path and os.path.exists(new_path): + if new_path != path and os.path.exists(new_path): remove(new_path) return ret diff --git a/tests-ng/transformations.sh b/tests-ng/transformations.sh index 2f1861c..1b62b9b 100755 --- a/tests-ng/transformations.sh +++ b/tests-ng/transformations.sh @@ -164,6 +164,13 @@ set -e # test update ########################### +# update single file +echo 'update' > ${tmpd}/def +cd ${ddpath} | ${bin} update -f -k -c ${cfg} -p p1 -b -V f_def +[ "$?" != "0" ] && exit 1 +[ ! -e ${tmpd}/def ] && echo 'dotfile in FS removed' && exit 1 +[ ! -e ${tmps}/dotfiles/def ] && echo 'dotfile in dotpath removed' && exit 1 + # update single file cd ${ddpath} | ${bin} update -f -k -c ${cfg} -p p1 -b -V f_abc [ "$?" != "0" ] && exit 1 From 9519d3e9908448c8e58597f9cc91c0dc230a9366 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 10 Jun 2019 21:02:12 +0200 Subject: [PATCH 52/58] adding showdiff for symlink for #156 --- dotdrop/installer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dotdrop/installer.py b/dotdrop/installer.py index c53e350..da620ec 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -216,6 +216,10 @@ class Installer: if self.dry: self.log.dry('would remove {} and link to {}'.format(dst, src)) return True, None + if self.showdiff: + with open(src, 'rb') as f: + content = f.read() + self._diff_before_write(src, dst, content) msg = 'Remove "{}" for link creation?'.format(dst) if self.safe and not self.log.ask(msg): err = 'ignoring "{}", link was not created'.format(dst) From e04a0b86e5fb37bf442c674fc3e9d88e2fdd3628 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 10 Jun 2019 21:06:23 +0200 Subject: [PATCH 53/58] adding version in debug logs --- dotdrop/options.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dotdrop/options.py b/dotdrop/options.py index 6ba7d6c..5126ff3 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -118,6 +118,7 @@ class Options(AttrMonitor): self.profile = self.args['--profile'] self.confpath = self._get_config_path() if self.debug: + self.log.dbg('version: {}'.format(VERSION)) self.log.dbg('config file: {}'.format(self.confpath)) self._read_config() From d6d5ea2ccff6f5f1dc83d34cf19f77ec66a2f7d6 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 11 Jun 2019 09:18:07 +0200 Subject: [PATCH 54/58] link already exists is not an error (for #154) --- dotdrop/installer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dotdrop/installer.py b/dotdrop/installer.py index da620ec..632cd7c 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -211,8 +211,10 @@ class Installer: overwrite = not self.safe if os.path.lexists(dst): if os.path.realpath(dst) == os.path.realpath(src): - err = 'ignoring "{}", link exists'.format(dst) - return False, err + msg = 'ignoring "{}", link already exists'.format(dst) + if self.debug: + self.log.dbg(msg) + return True, None if self.dry: self.log.dry('would remove {} and link to {}'.format(dst, src)) return True, None From e0bbef6fb2ef0fffda7da2e018ba807d10137f10 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 11 Jun 2019 12:34:31 +0200 Subject: [PATCH 55/58] trans_r and trans_w are not list anymore --- dotdrop/cfg_aggregator.py | 17 ++++++++++------- dotdrop/dotdrop.py | 22 +++++++++++----------- dotdrop/dotfile.py | 20 +++----------------- tests/test_update.py | 2 +- 4 files changed, 25 insertions(+), 36 deletions(-) diff --git a/dotdrop/cfg_aggregator.py b/dotdrop/cfg_aggregator.py index 4074fa8..747a2fb 100644 --- a/dotdrop/cfg_aggregator.py +++ b/dotdrop/cfg_aggregator.py @@ -102,15 +102,14 @@ class CfgAggregator: # patch trans_w/trans_r in dotfiles self._patch_keys_to_objs(self.dotfiles, - "trans_r", self._get_trans_r) + "trans_r", self._get_trans_r, islist=False) self._patch_keys_to_objs(self.dotfiles, - "trans_w", self._get_trans_w) + "trans_w", self._get_trans_w, islist=False) - def _patch_keys_to_objs(self, containers, keys, get_by_key): + def _patch_keys_to_objs(self, containers, keys, get_by_key, islist=True): """ - patch each object in "containers" containing - a list of keys in the attribute "keys" with - the returned object of the function "get_by_key" + map for each key in the attribute 'keys' in 'containers' + the returned object from the method 'get_by_key' """ if not containers: return @@ -121,13 +120,17 @@ class CfgAggregator: okeys = getattr(c, keys) if not okeys: continue + if not islist: + okeys = [okeys] for k in okeys: o = get_by_key(k) if not o: - err = 'bad {} key for \"{}\": {}'.format(keys, c.key, k) + err = 'bad {} key for \"{}\": {}'.format(keys, c, k) self.log.err(err) raise Exception(err) objects.append(o) + if not islist: + objects = objects[0] if self.debug: self.log.dbg('patching {}.{} with {}'.format(c, keys, objects)) setattr(c, keys, objects) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 98d442c..77ff1df 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -545,17 +545,17 @@ def apply_trans(dotpath, dotfile, debug=False): """ src = dotfile.src new_src = '{}.{}'.format(src, TRANS_SUFFIX) - for trans in dotfile.trans_r: - if debug: - LOG.dbg('executing transformation {}'.format(trans)) - s = os.path.join(dotpath, src) - temp = os.path.join(dotpath, new_src) - if not trans.transform(s, temp): - msg = 'transformation \"{}\" failed for {}' - LOG.err(msg.format(trans.key, dotfile.key)) - if new_src and os.path.exists(new_src): - remove(new_src) - return None + trans = dotfile.trans_r + if debug: + LOG.dbg('executing transformation {}'.format(trans)) + s = os.path.join(dotpath, src) + temp = os.path.join(dotpath, new_src) + if not trans.transform(s, temp): + msg = 'transformation \"{}\" failed for {}' + LOG.err(msg.format(trans.key, dotfile.key)) + if new_src and os.path.exists(new_src): + remove(new_src) + return None return new_src diff --git a/dotdrop/dotfile.py b/dotdrop/dotfile.py index 333ce74..0f3d5e8 100644 --- a/dotdrop/dotfile.py +++ b/dotdrop/dotfile.py @@ -18,7 +18,7 @@ class Dotfile(DictParser): key_trans_w = 'trans_write' def __init__(self, key, dst, src, - actions=[], trans_r=[], trans_w=[], + actions=[], trans_r=None, trans_w=None, link=LinkTypes.NOLINK, cmpignore=[], noempty=False, upignore=[]): """ @@ -42,11 +42,7 @@ class Dotfile(DictParser): self.noempty = noempty self.src = src self.trans_r = trans_r - if trans_r and len(self.trans_r) > 1: - raise Exception('only one trans_read allowed') self.trans_w = trans_w - if trans_w and len(self.trans_w) > 1: - raise Exception('only one trans_write allowed') self.upignore = upignore if link != LinkTypes.NOLINK and \ @@ -80,28 +76,18 @@ class Dotfile(DictParser): def get_trans_r(self): """return trans_r object""" - if self.trans_r: - return self.trans_r[0] - return None + return self.trans_r def get_trans_w(self): """return trans_w object""" - if self.trans_w: - return self.trans_w[0] - return None + return self.trans_w @classmethod def _adjust_yaml_keys(cls, value): """patch dict""" value['noempty'] = value.get(cls.key_noempty, False) value['trans_r'] = value.get(cls.key_trans_r) - if value['trans_r']: - # ensure is a list - value['trans_r'] = [value['trans_r']] value['trans_w'] = value.get(cls.key_trans_w) - if value['trans_w']: - # ensure is a list - value['trans_w'] = [value['trans_w']] # remove old entries value.pop(cls.key_noempty, None) value.pop(cls.key_trans_r, None) diff --git a/tests/test_update.py b/tests/test_update.py index 885fbd6..81eb98f 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -118,7 +118,7 @@ class TestUpdate(unittest.TestCase): # retrieve the path of the sub in the dotpath d1indotpath = os.path.join(o.dotpath, dotfile.src) d1indotpath = os.path.expanduser(d1indotpath) - dotfile.trans_w = [trans] + dotfile.trans_w = trans # update template o.update_path = [d3t] From fd67adf380cfb230688e33a383a17d5b60174a2b Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 11 Jun 2019 13:22:33 +0200 Subject: [PATCH 56/58] cleaning --- dotdrop/logger.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dotdrop/logger.py b/dotdrop/logger.py index 51c6e8a..a784ac0 100644 --- a/dotdrop/logger.py +++ b/dotdrop/logger.py @@ -39,15 +39,12 @@ class Logger: ce = self._color(self.RESET) sys.stderr.write('{}{}{}'.format(cs, string, ce)) - def err(self, string, end='\n', *, throw=None): + def err(self, string, end='\n'): cs = self._color(self.RED) ce = self._color(self.RESET) msg = '{} {}'.format(string, end) sys.stderr.write('{}[ERR] {}{}'.format(cs, msg, ce)) - if throw is not None: - raise throw(msg) - def warn(self, string, end='\n'): cs = self._color(self.YELLOW) ce = self._color(self.RESET) From 42f195d7c64736f5544eaf5e693c00d3626fd500 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 12 Jun 2019 10:04:21 +0200 Subject: [PATCH 57/58] allow comments in yaml --- dotdrop/cfg_yaml.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 41e7369..c106387 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -802,7 +802,9 @@ class CfgYaml: def _yaml_load(self, path): """load from yaml""" with open(path, 'r') as f: - content = yaml(typ='safe').load(f) + y = yaml() + y.typ = 'rt' + content = y.load(f) return content def _yaml_dump(self, content, path): @@ -811,5 +813,5 @@ class CfgYaml: y = yaml() y.default_flow_style = False y.indent = 2 - y.typ = 'safe' + y.typ = 'rt' y.dump(content, f) From 6384516cdc2b8831042d53039afd33fbb77dff2c Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 12 Jun 2019 10:10:38 +0200 Subject: [PATCH 58/58] fail if dynvariables cannot be executed successfully --- dotdrop/cfg_yaml.py | 7 ++++++- dotdrop/utils.py | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index c106387..0fa855b 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -209,7 +209,12 @@ class CfgYaml: # exec dynvariables for k in dvar.keys(): - allvars[k] = shell(allvars[k]) + ret, out = shell(allvars[k]) + if not ret: + err = 'command \"{}\" failed: {}'.format(allvars[k], out) + self.log.error(err) + raise YamlException(err) + allvars[k] = out if self.debug: self.log.dbg('variables:') diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 9300d9b..ce14d2b 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -49,8 +49,12 @@ def write_to_tmpfile(content): def shell(cmd): - """run a command in the shell (expects a string)""" - return subprocess.getoutput(cmd) + """ + run a command in the shell (expects a string) + returns True|False, output + """ + ret, out = subprocess.getstatusoutput(cmd) + return ret == 0, out def diff(src, dst, raw=True, opts='', debug=False):