From 262f0ba0db4c3c33609668d08acfb888af29b04e Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 13 Jul 2020 20:56:25 +0200 Subject: [PATCH 01/47] adding test for bug #250 --- tests-ng/profile-dyninclude.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests-ng/profile-dyninclude.sh b/tests-ng/profile-dyninclude.sh index b21022e..4cb09b1 100755 --- a/tests-ng/profile-dyninclude.sh +++ b/tests-ng/profile-dyninclude.sh @@ -90,6 +90,9 @@ dotfiles: f_def: dst: ${tmpd}/def src: def + f_ghi: + dst: '${tmpd}/{{@@ ghi @@}}' + src: ghi variables: mainvar: 'bad0' subvar: 'bad1' @@ -100,8 +103,10 @@ profiles: subprofile: dotfiles: - f_abc + - f_ghi dynvariables: subdyn: 'echo subdyncontent' + ghi: 'echo ghi' variables: subvar: 'subcontent' subignore: @@ -118,6 +123,7 @@ echo "{{@@ subdyn @@}}" >> ${tmps}/dotfiles/abc echo "{{@@ subvar @@}}" >> ${tmps}/dotfiles/abc echo "end" >> ${tmps}/dotfiles/abc #cat ${tmps}/dotfiles/abc +echo "ghi content" > ${tmps}/dotfiles/ghi # install cd ${ddpath} | ${bin} install -f -c ${cfg} -p profile_1 --verbose @@ -129,6 +135,7 @@ grep 'maindyncontent' ${tmpd}/abc >/dev/null || (echo "dynvariables 1 not resolv grep 'subcontent' ${tmpd}/abc >/dev/null || (echo "variables 2 not resolved" && exit 1) grep 'subdyncontent' ${tmpd}/abc >/dev/null || (echo "dynvariables 2 not resolved" && exit 1) #cat ${tmpd}/abc +[ ! -e ${tmpd}/ghi] && exit 1 ## CLEANING rm -rf ${tmps} ${tmpd} From e3b7359b95b5febba3def1483b2b34cd8719331c Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 13 Jul 2020 21:07:13 +0200 Subject: [PATCH 02/47] better handle variables and raise exception when an used variable is undefined --- dotdrop/templategen.py | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/dotdrop/templategen.py b/dotdrop/templategen.py index 54aa90c..9473b6e 100644 --- a/dotdrop/templategen.py +++ b/dotdrop/templategen.py @@ -7,7 +7,8 @@ jinja2 template generator import os from jinja2 import Environment, FileSystemLoader, \ - ChoiceLoader, FunctionLoader, TemplateNotFound + ChoiceLoader, FunctionLoader, TemplateNotFound, \ + StrictUndefined # local imports import dotdrop.utils as utils @@ -36,6 +37,7 @@ class Templategen: self.base = base.rstrip(os.sep) self.debug = debug self.log = Logger() + self.variables = {} loader1 = FileSystemLoader(self.base) loader2 = FunctionLoader(self._template_loader) loader = ChoiceLoader([loader1, loader2]) @@ -47,11 +49,14 @@ class Templategen: variable_start_string=VAR_START, variable_end_string=VAR_END, comment_start_string=COMMENT_START, - comment_end_string=COMMENT_END) + comment_end_string=COMMENT_END, + undefined=StrictUndefined) + # adding variables - self.env.globals['env'] = os.environ + self.variables['env'] = os.environ if variables: - self.env.globals.update(variables) + self.variables.update(variables) + # adding header method self.env.globals['header'] = self._header # adding helper methods @@ -72,32 +77,40 @@ class Templategen: self._debug_dict('template additional variables', variables) def generate(self, src): - """render template from path""" + """ + render template from path + may raise a jinja2.exceptions.UndefinedError + in case a variable is undefined + """ if not os.path.exists(src): return '' return self._handle_file(src) def generate_string(self, string): - """render template from string""" + """ + render template from string + may raise a jinja2.exceptions.UndefinedError + in case a variable is undefined + """ if not string: return '' - return self.env.from_string(string).render() + return self.env.from_string(string).render(self.variables) def add_tmp_vars(self, newvars={}): """add vars to the globals, make sure to call restore_vars""" - saved_globals = self.env.globals.copy() + saved_variables = self.variables.copy() if not newvars: - return saved_globals - self.env.globals.update(newvars) - return saved_globals + return saved_variables + self.variables.update(newvars) + return saved_variables def restore_vars(self, saved_globals): """restore globals from add_tmp_vars""" - self.env.globals = saved_globals.copy() + self.variables = saved_globals.copy() def update_variables(self, variables): """update variables""" - self.env.globals.update(variables) + self.variables.update(variables) def _load_path_to_dic(self, path, dic): mod = utils.get_module_from_path(path) @@ -160,7 +173,7 @@ class Templategen: template_rel_path = os.path.relpath(src, self.base) try: template = self.env.get_template(template_rel_path) - content = template.render() + content = template.render(self.variables) except UnicodeDecodeError: data = self._read_bad_encoded_text(src) content = self.generate_string(data) From 7775184d06a17d099581e850f02c94d0a16f9f3c Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 13 Jul 2020 21:37:53 +0200 Subject: [PATCH 03/47] fix missing variables --- dotdrop/options.py | 4 +++- tests-ng/ignore-empty.sh | 2 +- tests/helpers.py | 1 - tests/test_install.py | 1 - 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dotdrop/options.py b/dotdrop/options.py index d36025f..cb41914 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -109,9 +109,11 @@ class Options(AttrMonitor): """constructor @args: argument dictionary (if None use sys) """ - self.args = args + self.args = {} if not args: self.args = docopt(USAGE, version=VERSION) + if args: + self.args = args.copy() self.log = Logger() self.debug = self.args['--verbose'] or ENV_DEBUG in os.environ self.dry = self.args['--dry'] diff --git a/tests-ng/ignore-empty.sh b/tests-ng/ignore-empty.sh index fe8dbf1..df80274 100755 --- a/tests-ng/ignore-empty.sh +++ b/tests-ng/ignore-empty.sh @@ -76,7 +76,7 @@ _EOF # create the dotfile mkdir -p ${tmps}/dotfiles/d1 -echo "{{@@ var1 @@}}" > ${tmps}/dotfiles/d1/empty +echo "{#@@ should be stripped @@#}" > ${tmps}/dotfiles/d1/empty echo "not empty" > ${tmps}/dotfiles/d1/notempty # install diff --git a/tests/helpers.py b/tests/helpers.py index d8cef2a..ed2ae0e 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -158,7 +158,6 @@ def load_options(confpath, profile): o.import_link = LinkTypes.NOLINK o.install_showdiff = True o.debug = True - o.variables = {} return o diff --git a/tests/test_install.py b/tests/test_install.py index 5ba35cb..0f0ce66 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -186,7 +186,6 @@ exec bspwm o = load_options(confpath, profile) o.safe = False o.install_showdiff = True - o.variables = {} cmd_install(o) # now compare the generated files From c36854f250b8574a287f9077800b0f2309ada7b3 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 14 Jul 2020 20:32:05 +0200 Subject: [PATCH 04/47] propagate undefined variable exception --- dotdrop/action.py | 15 +++++++++++++-- dotdrop/cfg_aggregator.py | 8 +++++++- dotdrop/dotdrop.py | 7 +++++-- dotdrop/exceptions.py | 5 +++++ dotdrop/installer.py | 9 +++++++-- dotdrop/templategen.py | 17 +++++++++++++---- dotdrop/updater.py | 7 ++++++- 7 files changed, 56 insertions(+), 12 deletions(-) diff --git a/dotdrop/action.py b/dotdrop/action.py index ccc7bf5..c31254e 100644 --- a/dotdrop/action.py +++ b/dotdrop/action.py @@ -11,6 +11,7 @@ import os # local imports from dotdrop.dictparser import DictParser +from dotdrop.exceptions import UndefinedException class Cmd(DictParser): @@ -32,7 +33,12 @@ class Cmd(DictParser): ret = 1 action = self.action if templater: - action = templater.generate_string(self.action) + try: + action = templater.generate_string(self.action) + except UndefinedException as e: + err = 'bad {}: {}'.format(self.descr, e) + self.log.warn(err) + return False if debug: self.log.dbg('{}:'.format(self.descr)) self.log.dbg(' - raw \"{}\"'.format(self.action)) @@ -42,7 +48,12 @@ class Cmd(DictParser): if self.args: args = self.args if templater: - args = [templater.generate_string(a) for a in args] + try: + args = [templater.generate_string(a) for a in args] + except UndefinedException as e: + err = 'bad arguments for {}: {}'.format(self.descr, e) + self.log.warn(err) + return False if debug and args: self.log.dbg('action args:') for cnt, arg in enumerate(args): diff --git a/dotdrop/cfg_aggregator.py b/dotdrop/cfg_aggregator.py index 8360ff4..958af8a 100644 --- a/dotdrop/cfg_aggregator.py +++ b/dotdrop/cfg_aggregator.py @@ -17,6 +17,7 @@ from dotdrop.profile import Profile from dotdrop.action import Action, Transform from dotdrop.logger import Logger from dotdrop.utils import strip_home +from dotdrop.exceptions import UndefinedException TILD = '~' @@ -281,7 +282,12 @@ class CfgAggregator: @src: dotfile src (in dotpath) @dst: dotfile dst (on filesystem) """ - src = self.cfgyaml.resolve_dotfile_src(src) + try: + src = self.cfgyaml.resolve_dotfile_src(src) + except UndefinedException as e: + err = 'unable to resolve {}: {}' + self.log.err(err.format(src, e)) + return None dotfiles = self.get_dotfile_by_dst(dst) for d in dotfiles: if d.src == src: diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 417c095..efe618d 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -20,7 +20,7 @@ from dotdrop.comparator import Comparator from dotdrop.utils import get_tmpdir, remove, strip_home, \ run, uniq_list, patch_ignores, dependencies_met from dotdrop.linktypes import LinkTypes -from dotdrop.exceptions import YamlException +from dotdrop.exceptions import YamlException, UndefinedException LOG = Logger() TRANS_SUFFIX = 'trans' @@ -655,7 +655,10 @@ def main(): try: o = Options() except YamlException as e: - LOG.err('config file error: {}'.format(str(e))) + LOG.err('config error: {}'.format(str(e))) + return False + except UndefinedException as e: + LOG.err('config error: {}'.format(str(e))) return False if o.debug: diff --git a/dotdrop/exceptions.py b/dotdrop/exceptions.py index a327696..9c2df22 100644 --- a/dotdrop/exceptions.py +++ b/dotdrop/exceptions.py @@ -9,3 +9,8 @@ diverse exceptions class YamlException(Exception): """exception in CfgYaml""" pass + + +class UndefinedException(Exception): + """exception in templating""" + pass diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 0cc6856..0d16049 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -12,6 +12,7 @@ import errno from dotdrop.logger import Logger from dotdrop.templategen import Templategen import dotdrop.utils as utils +from dotdrop.exceptions import UndefinedException class Installer: @@ -325,8 +326,12 @@ class Installer: err = 'dotfile points to itself: {}'.format(dst) return False, err saved = templater.add_tmp_vars(self._get_tmp_file_vars(src, dst)) - content = templater.generate(src) - templater.restore_vars(saved) + try: + content = templater.generate(src) + except UndefinedException as e: + return False, e.message + finally: + templater.restore_vars(saved) if noempty and utils.content_empty(content): if self.debug: self.log.dbg('ignoring empty template: {}'.format(src)) diff --git a/dotdrop/templategen.py b/dotdrop/templategen.py index 9473b6e..ff2f06b 100644 --- a/dotdrop/templategen.py +++ b/dotdrop/templategen.py @@ -9,11 +9,14 @@ import os from jinja2 import Environment, FileSystemLoader, \ ChoiceLoader, FunctionLoader, TemplateNotFound, \ StrictUndefined +from jinja2.exceptions import UndefinedError + # local imports import dotdrop.utils as utils from dotdrop.logger import Logger import dotdrop.jhelpers as jhelpers +from dotdrop.exceptions import UndefinedException BLOCK_START = '{%@@' BLOCK_END = '@@%}' @@ -79,22 +82,28 @@ class Templategen: def generate(self, src): """ render template from path - may raise a jinja2.exceptions.UndefinedError + may raise a UndefinedException in case a variable is undefined """ if not os.path.exists(src): return '' - return self._handle_file(src) + try: + return self._handle_file(src) + except UndefinedError as e: + raise UndefinedException(e.message) def generate_string(self, string): """ render template from string - may raise a jinja2.exceptions.UndefinedError + may raise a UndefinedException in case a variable is undefined """ if not string: return '' - return self.env.from_string(string).render(self.variables) + try: + return self.env.from_string(string).render(self.variables) + except UndefinedError as e: + raise UndefinedException(e.message) def add_tmp_vars(self, newvars={}): """add vars to the globals, make sure to call restore_vars""" diff --git a/dotdrop/updater.py b/dotdrop/updater.py index 67b9f6a..c5cef2f 100644 --- a/dotdrop/updater.py +++ b/dotdrop/updater.py @@ -14,6 +14,7 @@ from dotdrop.logger import Logger from dotdrop.templategen import Templategen from dotdrop.utils import patch_ignores, remove, get_unique_tmp_name, \ write_to_tmpfile, must_ignore, mirror_file_rights +from dotdrop.exceptions import UndefinedException TILD = '~' @@ -186,7 +187,11 @@ class Updater: if self.debug: self.log.dbg('{} is a template'.format(dtpath)) if self.showpatch: - self._show_patch(path, dtpath) + try: + self._show_patch(path, dtpath) + except UndefinedException as e: + msg = 'unable to show patch for {}: {}'.format(path, e) + self.log.warn(msg) return False if compare and filecmp.cmp(path, dtpath, shallow=False) and \ self._same_rights(path, dtpath): From 8202cf6106c44f0bcce81aae5974dbc279c0c6ea Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 26 Jul 2020 14:59:27 +0200 Subject: [PATCH 05/47] refactoring cfg_yaml --- dotdrop/cfg_aggregator.py | 2 +- dotdrop/cfg_yaml.py | 1260 +++++++++++++++++++++---------------- 2 files changed, 712 insertions(+), 550 deletions(-) diff --git a/dotdrop/cfg_aggregator.py b/dotdrop/cfg_aggregator.py index 958af8a..2be9297 100644 --- a/dotdrop/cfg_aggregator.py +++ b/dotdrop/cfg_aggregator.py @@ -78,7 +78,7 @@ class CfgAggregator: self._debug_list('trans_w', self.trans_w) # variables - self.variables = self.cfgyaml.get_variables() + self.variables = self.cfgyaml.variables if self.debug: self._debug_dict('variables', self.variables) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 5dfa50a..c2bf973 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -3,6 +3,19 @@ author: deadc0de6 (https://github.com/deadc0de6) Copyright (c) 2019, deadc0de6 handle lower level of the config file + +will provide the following dictionaries to +the upper layer: + +* self.settings +* self.dotfiles +* self.profiles +* self.actions +* self.trans_r +* self.trans_w +* self.variables + +Additionally a few methods are exported. """ import os @@ -19,7 +32,7 @@ from dotdrop.logger import Logger from dotdrop.templategen import Templategen from dotdrop.linktypes import LinkTypes from dotdrop.utils import shell, uniq_list -from dotdrop.exceptions import YamlException +from dotdrop.exceptions import YamlException, UndefinedException class CfgYaml: @@ -76,6 +89,44 @@ class CfgYaml: lnk_link = LinkTypes.LINK.name.lower() lnk_children = LinkTypes.LINK_CHILDREN.name.lower() + # TODO + # + # "include" entries: + # - import_actions + # - import_configs + # - import_variables + # - profile's import + # - profile's include + # + # variable precedence: + # 1) profile variable + # 2) "import_variables" variables + # 3) "import_configs" variables + # 4) other variables + # + # parse a config file + # - parse settings + # - parse variables + # - interprete dynvariables + # - template the include entries + # - parse and integrate included elements (see below) + # - parse profiles + # - parse dotfiles + # + # parse "include" entry + # - same as parse config file + # - add new entry to the top dict + # + # TODO + # - properly handle the included profile + # - document precedence in wiki + # - document parsing in CONTRIBUTING.md + # - document dvars are executed in their own config file + # - remove unused functions/methods + # - coverage + # - remove ori_* + # + def __init__(self, path, profile=None, debug=False): """ config parser @@ -83,92 +134,302 @@ class CfgYaml: @profile: the selected profile @debug: debug flag """ - self.path = os.path.abspath(path) - self.profile = profile - self.debug = debug - self.log = Logger() + self._path = os.path.abspath(path) + self._profile = profile + self._debug = debug + self._log = Logger() # config needs to be written - self.dirty = False + self._dirty = False # indicates the config has been updated - self.dirty_deprecated = False + self._dirty_deprecated = False - if not os.path.exists(path): + # init the dictionaries + self.settings = {} + self.dotfiles = {} + self.profiles = {} + self.actions = {} + self.trans_r = {} + self.trans_w = {} + self.variables = {} + + if not os.path.exists(self._path): err = 'invalid config path: \"{}\"'.format(path) - if self.debug: - self.log.dbg(err) + if self._debug: + self._log.dbg(err) raise YamlException(err) - self.yaml_dict = self._load_yaml(self.path) - # live patch deprecated entries - self._fix_deprecated(self.yaml_dict) - # parse to self variables - self._parse_main_yaml(self.yaml_dict) - if self.debug: - self.log.dbg('BEFORE normalization: {}'.format(self.yaml_dict)) + self._yaml_dict = self._load_yaml(self._path) - # resolve variables - self.variables, self.prokeys = self._merge_variables() + # parse the "config" block + self.settings = self._parse_blk_settings(self._yaml_dict) - # apply variables - self._apply_variables() + # parse the "variables" block + self.variables = self._parse_blk_variables(self._yaml_dict) + self._redefine_templater() + + # parse the "dynvariables" block + dvariables = self._parse_blk_dynvariables(self._yaml_dict) + self._add_variables(dvariables) + + # interprete dynvariables + dvariables = self._template_dict(dvariables) + dvariables = self._shell_exec_dvars(dvariables) + + # merge variables and dynvariables + self.variables = self._merge_dict(dvariables, self.variables) + self._redefine_templater() + + # TODO template variables + self.variables = self._template_dict(self.variables) + if self._debug: + self._debug_dict('variables', self.variables) + + # template the "include" entries + self._template_include_entry() # process imported variables (import_variables) - self._import_variables() - # process imported actions (import_actions) + newvars = self._import_variables() + self._add_variables(newvars) + + # TODO process imported actions (import_actions) self._import_actions() - # process imported profile dotfiles (import) + # TODO process imported profile dotfiles (import) self._import_profiles_dotfiles() - # process imported configs (import_configs) + # TODO process imported configs (import_configs) self._import_configs() # process profile include self._resolve_profile_includes() + + # ===== + # TODO below + # ==== + if self._debug: + self._log.dbg('BEFORE normalization: {}'.format(self._yaml_dict)) + + # resolve variables + # TODO + # self.variables, self.prokeys = self._merge_variables() + + # apply variables + # self._apply_variables() + # process profile ALL self._resolve_profile_all() # patch dotfiles paths self._resolve_dotfile_paths() - if self.debug: - self.log.dbg('AFTER normalization: {}'.format(self.yaml_dict)) + # TODO ensure no element is left un-templated at the end - def get_variables(self): - """retrieve all variables""" - return self.variables + if self._debug: + self._log.dbg('AFTER normalization: {}'.format(self._yaml_dict)) + + def _add_variables(self, ext_variables): + """add new variables from external file""" + # TODO move me + # merge + self.variables = self._merge_dict(self.variables, ext_variables) + # ensure enriched variables are relative to this config + self.variables = self._enrich_vars(self.variables, self._profile) + # re-create the templater + self._redefine_templater() + # rec resolve variables with new ones + self.variables = self._template_dict(self.variables) + + def _redefine_templater(self): + """create templater based on current variables""" + fufile = self.settings[Settings.key_func_file] + fifile = self.settings[Settings.key_filter_file] + self._tmpl = Templategen(variables=self.variables, + func_file=fufile, + filter_file=fifile) ######################################################## - # parsing + # outside available methods ######################################################## - def _parse_main_yaml(self, dic): - """parse the different blocks""" - self.ori_settings = self._get_entry(dic, self.key_settings) - self.settings = Settings(None).serialize().get(self.key_settings) - self.settings.update(self.ori_settings) + def resolve_dotfile_src(self, src, templater=None): + """resolve dotfile src path""" + newsrc = '' + if src: + new = src + if templater: + new = templater.generate_string(src) + if new != src and self._debug: + msg = 'dotfile src: \"{}\" -> \"{}\"'.format(src, new) + self._log.dbg(msg) + src = new + src = os.path.join(self.settings[self.key_settings_dotpath], + src) + newsrc = self._norm_path(src) + return newsrc + + def resolve_dotfile_dst(self, dst, templater=None): + """resolve dotfile dst path""" + newdst = '' + if dst: + new = dst + if templater: + new = templater.generate_string(dst) + if new != dst and self._debug: + msg = 'dotfile dst: \"{}\" -> \"{}\"'.format(dst, new) + self._log.dbg(msg) + dst = new + newdst = self._norm_path(dst) + return newdst + + def add_dotfile_to_profile(self, dotfile_key, profile_key): + """add an existing dotfile key to a profile_key""" + self._new_profile(profile_key) + profile = self._yaml_dict[self.key_profiles][profile_key] + if self.key_profile_dotfiles not in profile or \ + profile[self.key_profile_dotfiles] is None: + profile[self.key_profile_dotfiles] = [] + pdfs = profile[self.key_profile_dotfiles] + if self.key_all not in pdfs and \ + dotfile_key not in pdfs: + profile[self.key_profile_dotfiles].append(dotfile_key) + if self._debug: + msg = 'add \"{}\" to profile \"{}\"'.format(dotfile_key, + profile_key) + msg.format(dotfile_key, profile_key) + self._log.dbg(msg) + self._dirty = True + return self._dirty + + def get_all_dotfile_keys(self): + """return all existing dotfile keys""" + return self.dotfiles.keys() + + def add_dotfile(self, key, src, dst, link): + """add a new dotfile""" + if key in self.dotfiles.keys(): + return False + if self._debug: + self._log.dbg('adding new dotfile: {}'.format(key)) + + df_dict = { + self.key_dotfile_src: src, + self.key_dotfile_dst: dst, + } + dfl = self.settings[self.key_settings_link_dotfile_default] + if str(link) != dfl: + df_dict[self.key_dotfile_link] = str(link) + self._yaml_dict[self.key_dotfiles][key] = df_dict + self._dirty = True + + def del_dotfile(self, key): + """remove this dotfile from config""" + if key not in self._yaml_dict[self.key_dotfiles]: + self._log.err('key not in dotfiles: {}'.format(key)) + return False + if self._debug: + self._log.dbg('remove dotfile: {}'.format(key)) + del self._yaml_dict[self.key_dotfiles][key] + if self._debug: + dfs = self._yaml_dict[self.key_dotfiles] + self._log.dbg('new dotfiles: {}'.format(dfs)) + self._dirty = True + return True + + def del_dotfile_from_profile(self, df_key, pro_key): + """remove this dotfile from that profile""" + if df_key not in self.dotfiles.keys(): + self._log.err('key not in dotfiles: {}'.format(df_key)) + return False + if pro_key not in self.profiles.keys(): + self._log.err('key not in profile: {}'.format(pro_key)) + return False + # get the profile dictionary + profile = self._yaml_dict[self.key_profiles][pro_key] + if df_key not in profile[self.key_profile_dotfiles]: + return True + if self._debug: + dfs = profile[self.key_profile_dotfiles] + self._log.dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) + self._log.dbg('remove {} from profile {}'.format(df_key, pro_key)) + profile[self.key_profile_dotfiles].remove(df_key) + if self._debug: + dfs = profile[self.key_profile_dotfiles] + self._log.dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) + self._dirty = True + return True + + def save(self): + """save this instance and return True if saved""" + if not self._dirty: + return False + + content = self._prepare_to_save(self._yaml_dict) + + if self._dirty_deprecated: + # add minversion + settings = content[self.key_settings] + settings[self.key_settings_minversion] = VERSION + + # save to file + if self._debug: + self._log.dbg('saving to {}'.format(self._path)) + try: + with open(self._path, 'w') as f: + self._yaml_dump(content, f) + except Exception as e: + self._log.err(e) + raise YamlException('error saving config: {}'.format(self._path)) + + if self._dirty_deprecated: + warn = 'your config contained deprecated entries' + warn += ' and was updated' + self._log.warn(warn) + + self._dirty = False + self.cfg_updated = False + return True + + def dump(self): + """dump the config dictionary""" + output = io.StringIO() + content = self._prepare_to_save(self._yaml_dict.copy()) + self._yaml_dump(content, output) + return output.getvalue() + + ######################################################## + # block parsing + ######################################################## + + def _parse_blk_settings(self, dic): + """parse the "config" block""" + block = self._get_entry(dic, self.key_settings) + # set defaults + settings = Settings(None).serialize().get(self.key_settings) + settings.update(block) # resolve minimum version - if self.key_settings_minversion in self.settings: - minversion = self.settings[self.key_settings_minversion] + if self.key_settings_minversion in settings: + minversion = settings[self.key_settings_minversion] self._check_minversion(minversion) - # resolve settings paths - p = self._norm_path(self.settings[self.key_settings_dotpath]) - self.settings[self.key_settings_dotpath] = p - p = self._norm_path(self.settings[self.key_settings_workdir]) - self.settings[self.key_settings_workdir] = p + # normalize paths + p = self._norm_path(settings[self.key_settings_dotpath]) + settings[self.key_settings_dotpath] = p + p = self._norm_path(settings[self.key_settings_workdir]) + settings[self.key_settings_workdir] = p p = [ self._norm_path(p) - for p in self.settings[Settings.key_filter_file] + for p in settings[Settings.key_filter_file] ] - self.settings[Settings.key_filter_file] = p + settings[Settings.key_filter_file] = p p = [ self._norm_path(p) - for p in self.settings[Settings.key_func_file] + for p in settings[Settings.key_func_file] ] - self.settings[Settings.key_func_file] = p - if self.debug: - self._debug_dict('settings', self.settings) + settings[Settings.key_func_file] = p + if self._debug: + self._debug_dict('settings', settings) + return settings - # dotfiles + def _parse_blk_dotfiles(self, dic): + """parse the "dotfiles" block""" self.ori_dotfiles = self._get_entry(dic, self.key_dotfiles) self.dotfiles = deepcopy(self.ori_dotfiles) keys = self.dotfiles.keys() @@ -177,58 +438,72 @@ class CfgYaml: err = 'duplicate dotfile keys found: {}'.format(dups) raise YamlException(err) self.dotfiles = self._norm_dotfiles(self.dotfiles) - if self.debug: + if self._debug: self._debug_dict('dotfiles', self.dotfiles) - # profiles + def _parse_blk_profiles(self, dic): + """parse the "profiles" block""" self.ori_profiles = self._get_entry(dic, self.key_profiles) self.profiles = deepcopy(self.ori_profiles) self.profiles = self._norm_profiles(self.profiles) - if self.debug: + if self._debug: self._debug_dict('profiles', self.profiles) - # actions + def _parse_blk_actions(self, dic): + """parse the "actions" block""" self.ori_actions = self._get_entry(dic, self.key_actions, mandatory=False) self.actions = deepcopy(self.ori_actions) self.actions = self._norm_actions(self.actions) - if self.debug: + if self._debug: self._debug_dict('actions', self.actions) - # trans_r + def _parse_blk_trans_r(self, dic): + """parse the "trans_r" block""" key = self.key_trans_r if self.old_key_trans_r in dic: - self.log.warn('\"trans\" is deprecated, please use \"trans_read\"') + msg = '\"trans\" is deprecated, please use \"trans_read\"' + self._log.warn(msg) dic[self.key_trans_r] = dic[self.old_key_trans_r] del dic[self.old_key_trans_r] self.ori_trans_r = self._get_entry(dic, key, mandatory=False) self.trans_r = deepcopy(self.ori_trans_r) - if self.debug: + if self._debug: self._debug_dict('trans_r', self.trans_r) - # trans_w + def _parse_blk_trans_w(self, dic): + """parse the "trans_w" block""" self.ori_trans_w = self._get_entry(dic, self.key_trans_w, mandatory=False) self.trans_w = deepcopy(self.ori_trans_w) - if self.debug: + if self._debug: self._debug_dict('trans_w', self.trans_w) - # variables - self.ori_variables = self._get_entry(dic, - self.key_variables, - mandatory=False) - if self.debug: - self._debug_dict('variables', self.ori_variables) + def _parse_blk_variables(self, dic): + """parse the "variables" block""" + variables = self._get_entry(dic, + self.key_variables, + mandatory=False) + if self._debug: + self._debug_dict('variables', variables) + return variables - # dynvariables - self.ori_dvariables = self._get_entry(dic, - self.key_dvariables, - mandatory=False) - if self.debug: - self._debug_dict('dynvariables', self.ori_dvariables) + def _parse_blk_dynvariables(self, dic): + """parse the "dynvariables" block""" + dvariables = self._get_entry(dic, + self.key_dvariables, + mandatory=False) + if self._debug: + self._debug_dict('dynvariables', dvariables) + return dvariables + + ######################################################## + # parsing helpers + ######################################################## def _resolve_dotfile_paths(self): """resolve dotfiles paths""" + # TODO remove t = Templategen(variables=self.variables, func_file=self.settings[Settings.key_func_file], filter_file=self.settings[Settings.key_filter_file]) @@ -243,39 +518,10 @@ class CfgYaml: newdst = self.resolve_dotfile_dst(dst, templater=t) dotfile[self.key_dotfile_dst] = newdst - def resolve_dotfile_src(self, src, templater=None): - """resolve dotfile src path""" - newsrc = '' - if src: - new = src - if templater: - new = templater.generate_string(src) - if new != src and self.debug: - msg = 'dotfile src: \"{}\" -> \"{}\"'.format(src, new) - self.log.dbg(msg) - src = new - src = os.path.join(self.settings[self.key_settings_dotpath], - src) - newsrc = self._norm_path(src) - return newsrc - - def resolve_dotfile_dst(self, dst, templater=None): - """resolve dotfile dst path""" - newdst = '' - if dst: - new = dst - if templater: - new = templater.generate_string(dst) - if new != dst and self.debug: - msg = 'dotfile dst: \"{}\" -> \"{}\"'.format(dst, new) - self.log.dbg(msg) - dst = new - newdst = self._norm_path(dst) - return newdst - def _rec_resolve_vars(self, variables): """recursive resolve variables""" - default = self._get_variables_dict(self.profile) + # TODO remove this and any call + default = self._get_variables_dict(self._profile) t = Templategen(variables=self._merge_dict(default, variables), func_file=self.settings[Settings.key_func_file], filter_file=self.settings[Settings.key_filter_file]) @@ -289,6 +535,7 @@ class CfgYaml: def _get_profile_included_vars(self, tvars): """resolve profile included variables/dynvariables""" + # TODO remove t = Templategen(variables=tvars, func_file=self.settings[Settings.key_func_file], filter_file=self.settings[Settings.key_filter_file]) @@ -301,12 +548,12 @@ class CfgYaml: v[self.key_profile_include] = new # now get the included ones - pro_var = self._get_profile_included_item(self.profile, + pro_var = self._get_profile_included_item(self._profile, self.key_profile_variables, - seen=[self.profile]) - pro_dvar = self._get_profile_included_item(self.profile, + seen=[self._profile]) + pro_dvar = self._get_profile_included_item(self._profile, self.key_profile_dvariables, - seen=[self.profile]) + seen=[self._profile]) # exec incl dynvariables self._shell_exec_dvars(pro_dvar.keys(), pro_dvar) @@ -318,11 +565,11 @@ class CfgYaml: apply them to any needed entries and return the full list of variables """ - if self.debug: - self.log.dbg('get local variables') + if self._debug: + self._log.dbg('get local variables') # get all variables from local and resolve - var = self._get_variables_dict(self.profile) + var = self._get_variables_dict(self._profile) # get all dynvariables from local and resolve dvar = self._get_dvariables_dict() @@ -330,13 +577,13 @@ class CfgYaml: # temporarly resolve all variables for "include" merged = self._merge_dict(dvar, var) merged = self._rec_resolve_vars(merged) - if self.debug: + if self._debug: self._debug_dict('variables', merged) # exec dynvariables self._shell_exec_dvars(dvar.keys(), merged) - if self.debug: - self.log.dbg('local variables resolved') + if self._debug: + self._log.dbg('local variables resolved') self._debug_dict('variables', merged) # resolve profile included variables/dynvariables @@ -347,42 +594,38 @@ class CfgYaml: merged = self._merge_dict(pro_dvar, merged) merged = self._rec_resolve_vars(merged) - if self.debug: - self.log.dbg('resolve all uses of variables in config') + if self._debug: + self._log.dbg('resolve all uses of variables in config') self._debug_dict('variables', merged) prokeys = list(pro_var.keys()) + list(pro_dvar.keys()) return merged, prokeys - def _apply_variables(self): - """template any needed parts of the config""" - t = Templategen(variables=self.variables, - func_file=self.settings[Settings.key_func_file], - filter_file=self.settings[Settings.key_filter_file]) - + def _template_include_entry(self): + """template all "include" entries""" # import_actions new = [] entries = self.settings.get(self.key_import_actions, []) - new = self._template_list(t, entries) + new = self._template_list(entries) if new: self.settings[self.key_import_actions] = new # import_configs entries = self.settings.get(self.key_import_configs, []) - new = self._template_list(t, entries) + new = self._template_list(entries) if new: self.settings[self.key_import_configs] = new # import_variables entries = self.settings.get(self.key_import_variables, []) - new = self._template_list(t, entries) + new = self._template_list(entries) if new: self.settings[self.key_import_variables] = new # profile's import for k, v in self.profiles.items(): entries = v.get(self.key_import_profile_dfs, []) - new = self._template_list(t, entries) + new = self._template_list(entries) if new: v[self.key_import_profile_dfs] = new @@ -432,7 +675,7 @@ class CfgYaml: # fix deprecated trans key if self.old_key_trans_r in v: msg = '\"trans\" is deprecated, please use \"trans_read\"' - self.log.warn(msg) + self._log.warn(msg) v[self.key_trans_r] = v[self.old_key_trans_r] del v[self.old_key_trans_r] new[k] = v @@ -446,9 +689,8 @@ class CfgYaml: v[self.key_dotfile_noempty] = val return new - def _get_variables_dict(self, profile): + def _enrich_vars(self, variables, profile): """return enriched variables""" - variables = deepcopy(self.ori_variables) # add profile variable if profile: variables['profile'] = profile @@ -456,7 +698,7 @@ class CfgYaml: p = self.settings.get(self.key_settings_dotpath) p = self._norm_path(p) variables['_dotdrop_dotpath'] = p - variables['_dotdrop_cfgpath'] = self._norm_path(self.path) + variables['_dotdrop_cfgpath'] = self._norm_path(self._path) p = self.settings.get(self.key_settings_workdir) p = self._norm_path(p) variables['_dotdrop_workdir'] = p @@ -483,9 +725,9 @@ class CfgYaml: seen.append(inherited_profile) new = self._get_profile_included_item(inherited_profile, item, seen) - if self.debug: + if self._debug: msg = 'included {} from {}: {}' - self.log.dbg(msg.format(item, inherited_profile, new)) + self._log.dbg(msg.format(item, inherited_profile, new)) items.update(new) cur = pentry.get(item, {}) @@ -499,8 +741,8 @@ class CfgYaml: if not dfs: continue if self.key_all in dfs: - if self.debug: - self.log.dbg('add ALL to profile {}'.format(k)) + if self._debug: + self._log.dbg('add ALL to profile {}'.format(k)) v[self.key_profile_dotfiles] = self.dotfiles.keys() def _resolve_profile_includes(self): @@ -532,22 +774,22 @@ class CfgYaml: # nothing to include return dotfiles, actions, pvars, pdvars - if self.debug: - self.log.dbg('{} includes {}'.format(profile, ','.join(includes))) - self.log.dbg('{} dotfiles before include: {}'.format(profile, - dotfiles)) - self.log.dbg('{} actions before include: {}'.format(profile, - actions)) - self.log.dbg('{} variables before include: {}'.format(profile, - pvars)) - self.log.dbg('{} dynvariables before include: {}'.format(profile, - pdvars)) + if self._debug: + self._log.dbg('{} includes {}'.format(profile, ','.join(includes))) + self._log.dbg('{} dotfiles before include: {}'.format(profile, + dotfiles)) + self._log.dbg('{} actions before include: {}'.format(profile, + actions)) + self._log.dbg('{} variables before include: {}'.format(profile, + pvars)) + self._log.dbg('{} dynvariables before include: {}'.format(profile, + pdvars)) seen = [] for i in uniq_list(includes): - if self.debug: - self.log.dbg('resolving includes "{}" <- "{}"' - .format(profile, i)) + if self._debug: + self._log.dbg('resolving includes "{}" <- "{}"' + .format(profile, i)) # ensure no include loop occurs if i in seen: @@ -555,41 +797,41 @@ class CfgYaml: seen.append(i) # included profile even exists if i not in self.profiles.keys(): - self.log.warn('include unknown profile: {}'.format(i)) + self._log.warn('include unknown profile: {}'.format(i)) continue # recursive resolve - if self.debug: - self.log.dbg('recursively resolving includes for profile "{}"' - .format(i)) + if self._debug: + self._log.dbg('recursively resolving includes for profile "{}"' + .format(i)) o_dfs, o_actions, o_v, o_dv = self._rec_resolve_profile_include(i) # merge dotfile keys - if self.debug: - self.log.dbg('Merging dotfiles {} <- {}: {} <- {}' - .format(profile, i, dotfiles, o_dfs)) + if self._debug: + self._log.dbg('Merging dotfiles {} <- {}: {} <- {}' + .format(profile, i, dotfiles, o_dfs)) dotfiles.extend(o_dfs) this_profile[self.key_profile_dotfiles] = uniq_list(dotfiles) # merge actions keys - if self.debug: - self.log.dbg('Merging actions {} <- {}: {} <- {}' - .format(profile, i, actions, o_actions)) + if self._debug: + self._log.dbg('Merging actions {} <- {}: {} <- {}' + .format(profile, i, actions, o_actions)) actions.extend(o_actions) this_profile[self.key_profile_actions] = uniq_list(actions) # merge variables - if self.debug: - self.log.dbg('Merging variables {} <- {}: {} <- {}' - .format(profile, i, dict(pvars), dict(o_v))) + if self._debug: + self._log.dbg('Merging variables {} <- {}: {} <- {}' + .format(profile, i, dict(pvars), dict(o_v))) pvars = self._merge_dict(o_v, pvars) this_profile[self.key_profile_variables] = pvars # merge dynvariables - if self.debug: - self.log.dbg('Merging dynamic variables {} <- {}: {} <- {}' - .format(profile, i, dict(pdvars), - dict(o_dv))) + if self._debug: + self._log.dbg('Merging dynamic variables {} <- {}: {} <- {}' + .format(profile, i, dict(pdvars), + dict(o_dv))) pdvars = self._merge_dict(o_dv, pdvars) this_profile[self.key_profile_dvariables] = pdvars @@ -598,17 +840,17 @@ class CfgYaml: pvars = this_profile.get(self.key_profile_variables, {}) or {} pdvars = this_profile.get(self.key_profile_dvariables, {}) or {} - if self.debug: - self.log.dbg('{} dotfiles after include: {}'.format(profile, - dotfiles)) - self.log.dbg('{} actions after include: {}'.format(profile, - actions)) - self.log.dbg('{} variables after include: {}'.format(profile, - pvars)) - self.log.dbg('{} dynvariables after include: {}'.format(profile, - pdvars)) + if self._debug: + self._log.dbg('{} dotfiles after include: {}'.format(profile, + dotfiles)) + self._log.dbg('{} actions after include: {}'.format(profile, + actions)) + self._log.dbg('{} variables after include: {}'.format(profile, + pvars)) + self._log.dbg('{} dynvariables after include: {}'.format(profile, + pdvars)) - if profile == self.profile: + if profile == self._profile: # Only for the selected profile, we execute dynamic variables and # we merge variables/dynvariables into the global variables self._shell_exec_dvars(pdvars.keys(), pdvars) @@ -630,30 +872,254 @@ class CfgYaml: if not paths: return paths = self._resolve_paths(paths) + newvars = {} for path in paths: - if self.debug: - self.log.dbg('import variables from {}'.format(path)) + if self._debug: + self._log.dbg('import variables from {}'.format(path)) var = self._import_sub(path, self.key_variables, mandatory=False) - if self.debug: - self.log.dbg('import dynvariables from {}'.format(path)) + if self._debug: + self._log.dbg('import dynvariables from {}'.format(path)) dvar = self._import_sub(path, self.key_dvariables, mandatory=False) merged = self._merge_dict(dvar, var) - merged = self._rec_resolve_vars(merged) - # execute dvar - self._shell_exec_dvars(dvar.keys(), merged) - self._clear_profile_vars(merged) - self.variables = self._merge_dict(merged, self.variables) + merged = self._template_dict(merged) + newvars = self._merge_dict(newvars, merged) + # TODO needed? + # self._clear_profile_vars(merged) + return newvars + + def _import_actions(self): + """import external actions from paths""" + paths = self.settings.get(self.key_import_actions, None) + if not paths: + return + paths = self._resolve_paths(paths) + for path in paths: + if self._debug: + self._log.dbg('import actions from {}'.format(path)) + new = self._import_sub(path, self.key_actions, + mandatory=False, + patch_func=self._norm_actions) + self.actions = self._merge_dict(new, self.actions) + + def _import_profiles_dotfiles(self): + """import profile dotfiles""" + for k, v in self.profiles.items(): + imp = v.get(self.key_import_profile_dfs, None) + if not imp: + continue + if self._debug: + self._log.dbg('import dotfiles for profile {}'.format(k)) + paths = self._resolve_paths(imp) + for path in paths: + current = v.get(self.key_dotfiles, []) + new = self._import_sub(path, self.key_dotfiles, + mandatory=False) + v[self.key_dotfiles] = new + current + + def _import_config(self, path): + """import config from path""" + if self._debug: + self._log.dbg('import config from {}'.format(path)) + sub = CfgYaml(path, profile=self._profile, debug=self._debug) + + # settings are ignored from external file + # except for filter_file and func_file + self.settings[Settings.key_func_file] += [ + self._norm_path(func_file) + for func_file in sub.settings[Settings.key_func_file] + ] + self.settings[Settings.key_filter_file] += [ + self._norm_path(func_file) + for func_file in sub.settings[Settings.key_filter_file] + ] + + # merge top entries + self.dotfiles = self._merge_dict(self.dotfiles, sub.dotfiles) + self.profiles = self._merge_dict(self.profiles, sub.profiles) + self.actions = self._merge_dict(self.actions, sub.actions) + self.trans_r = self._merge_dict(self.trans_r, sub.trans_r) + self.trans_w = self._merge_dict(self.trans_w, sub.trans_w) + self._clear_profile_vars(sub.variables) + + if self._debug: + self._debug_dict('add import_configs var', sub.variables) + self.variables = self._merge_dict(sub.variables, self.variables) + + def _import_configs(self): + """import configs from external files""" + # settings -> import_configs + imp = self.settings.get(self.key_import_configs, None) + if not imp: + return + paths = self._resolve_paths(imp) + for path in paths: + self._import_config(path) + + def _import_sub(self, path, key, mandatory=False, patch_func=None): + """ + import the block "key" from "path" + patch_func is applied to each element if defined + """ + if self._debug: + self._log.dbg('import \"{}\" from \"{}\"'.format(key, path)) + extdict = self._load_yaml(path) + new = self._get_entry(extdict, key, mandatory=mandatory) + if patch_func: + if self._debug: + self._log.dbg('calling patch: {}'.format(patch_func)) + new = patch_func(new) + if not new and mandatory: + err = 'no \"{}\" imported from \"{}\"'.format(key, path) + self._log.warn(err) + raise YamlException(err) + if self._debug: + self._log.dbg('imported \"{}\": {}'.format(key, new)) + return new + + ######################################################## + # add/remove entries + ######################################################## + + def _new_profile(self, key): + """add a new profile if it doesn't exist""" + if key not in self.profiles.keys(): + # update yaml_dict + self._yaml_dict[self.key_profiles][key] = { + self.key_profile_dotfiles: [] + } + if self._debug: + self._log.dbg('adding new profile: {}'.format(key)) + self._dirty = True + + ######################################################## + # handle deprecated entries + ######################################################## + + def _fix_deprecated(self, yamldict): + """fix deprecated entries""" + self._fix_deprecated_link_by_default(yamldict) + self._fix_deprecated_dotfile_link(yamldict) + return yamldict + + def _fix_deprecated_link_by_default(self, yamldict): + """fix deprecated link_by_default""" + key = 'link_by_default' + newkey = self.key_imp_link + if self.key_settings not in yamldict: + return + if not yamldict[self.key_settings]: + return + config = yamldict[self.key_settings] + if key not in config: + return + if config[key]: + config[newkey] = self.lnk_link + else: + config[newkey] = self.lnk_nolink + del config[key] + self._log.warn('deprecated \"link_by_default\"') + self._dirty = True + self._dirty_deprecated = True + + def _fix_deprecated_dotfile_link(self, yamldict): + """fix deprecated link in dotfiles""" + if self.key_dotfiles not in yamldict: + return + if not yamldict[self.key_dotfiles]: + return + for k, dotfile in yamldict[self.key_dotfiles].items(): + new = self.lnk_nolink + if self.key_dotfile_link in dotfile and \ + type(dotfile[self.key_dotfile_link]) is bool: + # patch link: + cur = dotfile[self.key_dotfile_link] + new = self.lnk_nolink + if cur: + new = self.lnk_link + dotfile[self.key_dotfile_link] = new + self._dirty = True + self._dirty_deprecated = True + self._log.warn('deprecated \"link\" value') + + elif self.key_dotfile_link_children in dotfile and \ + type(dotfile[self.key_dotfile_link_children]) is bool: + # patch link_children: + cur = dotfile[self.key_dotfile_link_children] + new = self.lnk_nolink + if cur: + new = self.lnk_children + del dotfile[self.key_dotfile_link_children] + dotfile[self.key_dotfile_link] = new + self._dirty = True + self._dirty_deprecated = True + self._log.warn('deprecated \"link_children\" value') + + ######################################################## + # yaml utils + ######################################################## + + def _prepare_to_save(self, content): + content = self._clear_none(content) + + # make sure we have the base entries + if self.key_settings not in content: + content[self.key_settings] = None + if self.key_dotfiles not in content: + content[self.key_dotfiles] = None + if self.key_profiles not in content: + content[self.key_profiles] = None + return content + + def _load_yaml(self, path): + """load a yaml file to a dict""" + content = {} + if self._debug: + self._log.dbg('----------start:{}----------'.format(path)) + cfg = '\n' + with open(path, 'r') as f: + for line in f: + cfg += line + self._log.dbg(cfg.rstrip()) + self._log.dbg('----------end:{}----------'.format(path)) + try: + content = self._yaml_load(path) + except Exception as e: + self._log.err(e) + raise YamlException('invalid config: {}'.format(path)) + # live patch deprecated entries + return self._fix_deprecated(content) + + def _yaml_load(self, path): + """load from yaml""" + with open(path, 'r') as f: + y = yaml() + y.typ = 'rt' + content = y.load(f) + return content + + def _yaml_dump(self, content, where): + """dump to yaml""" + y = yaml() + y.default_flow_style = False + y.indent = 2 + y.typ = 'rt' + y.dump(content, where) + + ######################################################## + # helpers + ######################################################## def _clear_profile_vars(self, dic): """remove profile variables from dic if found""" + # TODO check why this is used at all [dic.pop(k, None) for k in self.prokeys] def _parse_extended_import_path(self, path_entry): """Parse an import path in a tuple (path, fatal_not_found).""" - if self.debug: - self.log.dbg('parsing path entry {}'.format(path_entry)) + if self._debug: + self._log.dbg('parsing path entry {}'.format(path_entry)) path, _, attribute = path_entry.rpartition(self.key_import_sep) fatal_not_found = attribute != self.key_import_ignore_key @@ -666,14 +1132,14 @@ class CfgYaml: # attribute is set to whatever comes after the separator by # str.rpartition # In both cases, path_entry is the path we're looking for. - if self.debug: - self.log.dbg('using attribute default values for path {}' - .format(path_entry)) + if self._debug: + self._log.dbg('using attribute default values for path {}' + .format(path_entry)) path = path_entry fatal_not_found = self.key_import_fatal_not_found - elif self.debug: - self.log.dbg('path entry {} has fatal_not_found flag set to {}' - .format(path_entry, fatal_not_found)) + elif self._debug: + self._log.dbg('path entry {} has fatal_not_found flag set to {}' + .format(path_entry, fatal_not_found)) return path, fatal_not_found def _handle_non_existing_path(self, path, fatal_not_found=True): @@ -681,13 +1147,13 @@ class CfgYaml: error = 'bad path {}'.format(path) if fatal_not_found: raise YamlException(error) - self.log.warn(error) + self._log.warn(error) def _check_path_existence(self, path, fatal_not_found=True): """Check if a path exists, raising if necessary.""" if os.path.exists(path): - if self.debug: - self.log.dbg('path {} exists'.format(path)) + if self._debug: + self._log.dbg('path {} exists'.format(path)) return path self._handle_non_existing_path(path, fatal_not_found) @@ -714,8 +1180,8 @@ class CfgYaml: path = self._norm_path(path) paths = self._glob_path(path) if self._is_glob(path) else [path] if not paths: - if self.debug: - self.log.dbg("glob path {} didn't expand".format(path)) + if self._debug: + self._log.dbg("glob path {} didn't expand".format(path)) self._handle_non_existing_path(path, fatal_not_found) return [] @@ -738,341 +1204,22 @@ class CfgYaml: processed_paths = (self._process_path(p) for p in paths) return list(chain.from_iterable(processed_paths)) - def _import_actions(self): - """import external actions from paths""" - paths = self.settings.get(self.key_import_actions, None) - if not paths: - return - paths = self._resolve_paths(paths) - for path in paths: - if self.debug: - self.log.dbg('import actions from {}'.format(path)) - new = self._import_sub(path, self.key_actions, - mandatory=False, - patch_func=self._norm_actions) - self.actions = self._merge_dict(new, self.actions) - - def _import_profiles_dotfiles(self): - """import profile dotfiles""" - for k, v in self.profiles.items(): - imp = v.get(self.key_import_profile_dfs, None) - if not imp: - continue - if self.debug: - self.log.dbg('import dotfiles for profile {}'.format(k)) - paths = self._resolve_paths(imp) - for path in paths: - current = v.get(self.key_dotfiles, []) - new = self._import_sub(path, self.key_dotfiles, - mandatory=False) - v[self.key_dotfiles] = new + current - - def _import_config(self, path): - """import config from path""" - if self.debug: - self.log.dbg('import config from {}'.format(path)) - sub = CfgYaml(path, profile=self.profile, debug=self.debug) - - # settings are ignored from external file - # except for filter_file and func_file - self.settings[Settings.key_func_file] += [ - self._norm_path(func_file) - for func_file in sub.settings[Settings.key_func_file] - ] - self.settings[Settings.key_filter_file] += [ - self._norm_path(func_file) - for func_file in sub.settings[Settings.key_filter_file] - ] - - # merge top entries - self.dotfiles = self._merge_dict(self.dotfiles, sub.dotfiles) - self.profiles = self._merge_dict(self.profiles, sub.profiles) - self.actions = self._merge_dict(self.actions, sub.actions) - self.trans_r = self._merge_dict(self.trans_r, sub.trans_r) - self.trans_w = self._merge_dict(self.trans_w, sub.trans_w) - self._clear_profile_vars(sub.variables) - - if self.debug: - self._debug_dict('add import_configs var', sub.variables) - self.variables = self._merge_dict(sub.variables, self.variables) - - def _import_configs(self): - """import configs from external files""" - # settings -> import_configs - imp = self.settings.get(self.key_import_configs, None) - if not imp: - return - paths = self._resolve_paths(imp) - for path in paths: - self._import_config(path) - - def _import_sub(self, path, key, mandatory=False, patch_func=None): + def _template_item(self, item, exc_if_fail=True): """ - import the block "key" from "path" - patch_func is applied to each element if defined + template an item using the templategen + will raise an exception if template failed and exc_if_fail """ - if self.debug: - self.log.dbg('import \"{}\" from \"{}\"'.format(key, path)) - extdict = self._load_yaml(path) - new = self._get_entry(extdict, key, mandatory=mandatory) - if patch_func: - if self.debug: - self.log.dbg('calling patch: {}'.format(patch_func)) - new = patch_func(new) - if not new and mandatory: - err = 'no \"{}\" imported from \"{}\"'.format(key, path) - self.log.warn(err) - raise YamlException(err) - if self.debug: - self.log.dbg('imported \"{}\": {}'.format(key, new)) - return new - - ######################################################## - # add/remove entries - ######################################################## - - def _new_profile(self, key): - """add a new profile if it doesn't exist""" - if key not in self.profiles.keys(): - # update yaml_dict - self.yaml_dict[self.key_profiles][key] = { - self.key_profile_dotfiles: [] - } - if self.debug: - self.log.dbg('adding new profile: {}'.format(key)) - self.dirty = True - - def add_dotfile_to_profile(self, dotfile_key, profile_key): - """add an existing dotfile key to a profile_key""" - self._new_profile(profile_key) - profile = self.yaml_dict[self.key_profiles][profile_key] - if self.key_profile_dotfiles not in profile or \ - profile[self.key_profile_dotfiles] is None: - profile[self.key_profile_dotfiles] = [] - pdfs = profile[self.key_profile_dotfiles] - if self.key_all not in pdfs and \ - dotfile_key not in pdfs: - profile[self.key_profile_dotfiles].append(dotfile_key) - if self.debug: - msg = 'add \"{}\" to profile \"{}\"'.format(dotfile_key, - profile_key) - msg.format(dotfile_key, profile_key) - self.log.dbg(msg) - self.dirty = True - return self.dirty - - def get_all_dotfile_keys(self): - """return all existing dotfile keys""" - return self.dotfiles.keys() - - def add_dotfile(self, key, src, dst, link): - """add a new dotfile""" - if key in self.dotfiles.keys(): - return False - if self.debug: - self.log.dbg('adding new dotfile: {}'.format(key)) - - df_dict = { - self.key_dotfile_src: src, - self.key_dotfile_dst: dst, - } - dfl = self.settings[self.key_settings_link_dotfile_default] - if str(link) != dfl: - df_dict[self.key_dotfile_link] = str(link) - self.yaml_dict[self.key_dotfiles][key] = df_dict - self.dirty = True - - def del_dotfile(self, key): - """remove this dotfile from config""" - if key not in self.yaml_dict[self.key_dotfiles]: - self.log.err('key not in dotfiles: {}'.format(key)) - return False - if self.debug: - self.log.dbg('remove dotfile: {}'.format(key)) - del self.yaml_dict[self.key_dotfiles][key] - if self.debug: - dfs = self.yaml_dict[self.key_dotfiles] - self.log.dbg('new dotfiles: {}'.format(dfs)) - self.dirty = True - return True - - def del_dotfile_from_profile(self, df_key, pro_key): - """remove this dotfile from that profile""" - if df_key not in self.dotfiles.keys(): - self.log.err('key not in dotfiles: {}'.format(df_key)) - return False - if pro_key not in self.profiles.keys(): - self.log.err('key not in profile: {}'.format(pro_key)) - return False - # get the profile dictionary - profile = self.yaml_dict[self.key_profiles][pro_key] - if df_key not in profile[self.key_profile_dotfiles]: - return True - if self.debug: - dfs = profile[self.key_profile_dotfiles] - self.log.dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) - self.log.dbg('remove {} from profile {}'.format(df_key, pro_key)) - profile[self.key_profile_dotfiles].remove(df_key) - if self.debug: - dfs = profile[self.key_profile_dotfiles] - self.log.dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) - self.dirty = True - return True - - ######################################################## - # handle deprecated entries - ######################################################## - - def _fix_deprecated(self, yamldict): - """fix deprecated entries""" - self._fix_deprecated_link_by_default(yamldict) - self._fix_deprecated_dotfile_link(yamldict) - - def _fix_deprecated_link_by_default(self, yamldict): - """fix deprecated link_by_default""" - key = 'link_by_default' - newkey = self.key_imp_link - if self.key_settings not in yamldict: - return - if not yamldict[self.key_settings]: - return - config = yamldict[self.key_settings] - if key not in config: - return - if config[key]: - config[newkey] = self.lnk_link - else: - config[newkey] = self.lnk_nolink - del config[key] - self.log.warn('deprecated \"link_by_default\"') - self.dirty = True - self.dirty_deprecated = True - - def _fix_deprecated_dotfile_link(self, yamldict): - """fix deprecated link in dotfiles""" - if self.key_dotfiles not in yamldict: - return - if not yamldict[self.key_dotfiles]: - return - for k, dotfile in yamldict[self.key_dotfiles].items(): - new = self.lnk_nolink - if self.key_dotfile_link in dotfile and \ - type(dotfile[self.key_dotfile_link]) is bool: - # patch link: - cur = dotfile[self.key_dotfile_link] - new = self.lnk_nolink - if cur: - new = self.lnk_link - dotfile[self.key_dotfile_link] = new - self.dirty = True - self.dirty_deprecated = True - self.log.warn('deprecated \"link\" value') - - elif self.key_dotfile_link_children in dotfile and \ - type(dotfile[self.key_dotfile_link_children]) is bool: - # patch link_children: - cur = dotfile[self.key_dotfile_link_children] - new = self.lnk_nolink - if cur: - new = self.lnk_children - del dotfile[self.key_dotfile_link_children] - dotfile[self.key_dotfile_link] = new - self.dirty = True - self.dirty_deprecated = True - self.log.warn('deprecated \"link_children\" value') - - ######################################################## - # yaml utils - ######################################################## - - def _prepare_to_save(self, content): - content = self._clear_none(content) - - # make sure we have the base entries - if self.key_settings not in content: - content[self.key_settings] = None - if self.key_dotfiles not in content: - content[self.key_dotfiles] = None - if self.key_profiles not in content: - content[self.key_profiles] = None - - return content - - def save(self): - """save this instance and return True if saved""" - if not self.dirty: - return False - - content = self._prepare_to_save(self.yaml_dict) - - if self.dirty_deprecated: - # add minversion - settings = content[self.key_settings] - settings[self.key_settings_minversion] = VERSION - - # save to file - if self.debug: - self.log.dbg('saving to {}'.format(self.path)) + # TODO use this across the entire file + if not Templategen.var_is_template(item): + return item try: - with open(self.path, 'w') as f: - self._yaml_dump(content, f) - except Exception as e: - self.log.err(e) - raise YamlException('error saving config: {}'.format(self.path)) - - if self.dirty_deprecated: - warn = 'your config contained deprecated entries' - warn += ' and was updated' - self.log.warn(warn) - - self.dirty = False - self.cfg_updated = False - return True - - def dump(self): - """dump the config dictionary""" - output = io.StringIO() - content = self._prepare_to_save(self.yaml_dict.copy()) - self._yaml_dump(content, output) - return output.getvalue() - - def _load_yaml(self, path): - """load a yaml file to a dict""" - content = {} - if self.debug: - self.log.dbg('----------start:{}----------'.format(path)) - cfg = '\n' - with open(path, 'r') as f: - for line in f: - cfg += line - self.log.dbg(cfg.rstrip()) - self.log.dbg('----------end:{}----------'.format(path)) - try: - content = self._yaml_load(path) - except Exception as e: - self.log.err(e) - raise YamlException('invalid config: {}'.format(path)) - return content - - def _yaml_load(self, path): - """load from yaml""" - with open(path, 'r') as f: - y = yaml() - y.typ = 'rt' - content = y.load(f) - return content - - def _yaml_dump(self, content, where): - """dump to yaml""" - y = yaml() - y.default_flow_style = False - y.indent = 2 - y.typ = 'rt' - y.dump(content, where) - - ######################################################## - # helpers - ######################################################## + val = item + while Templategen.var_is_template(val): + val = self._tmpl.generate_string(val) + except UndefinedException as e: + if exc_if_fail: + raise e + return val def _merge_dict(self, high, low): """merge high and low dict""" @@ -1119,8 +1266,8 @@ class CfgYaml: def _glob_path(self, path): """Expand a glob.""" - if self.debug: - self.log.dbg('expanding glob {}'.format(path)) + if self._debug: + self._log.dbg('expanding glob {}'.format(path)) expanded_path = os.path.expanduser(path) return glob.glob(expanded_path, recursive=True) @@ -1130,41 +1277,56 @@ class CfgYaml: return path path = os.path.expanduser(path) if not os.path.isabs(path): - d = os.path.dirname(self.path) + d = os.path.dirname(self._path) ret = os.path.join(d, path) - if self.debug: + if self._debug: msg = 'normalizing relative to cfg: {} -> {}' - self.log.dbg(msg.format(path, ret)) + self._log.dbg(msg.format(path, ret)) return ret ret = os.path.normpath(path) - if self.debug and path != ret: - self.log.dbg('normalizing: {} -> {}'.format(path, ret)) + if self._debug and path != ret: + self._log.dbg('normalizing: {} -> {}'.format(path, ret)) return ret - def _shell_exec_dvars(self, keys, variables): + def _shell_exec_dvars(self, dic): """shell execute dynvariables""" - for k in list(keys): - ret, out = shell(variables[k], debug=self.debug) + # TODO remove other calls outside initial setup of dvars + executed = {} + for k, v in dic.items(): + ret, out = shell(v, debug=self._debug) if not ret: - err = 'var \"{}: {}\" failed: {}'.format(k, variables[k], out) - self.log.err(err) + err = 'var \"{}: {}\" failed: {}'.format(k, v, out) + self._log.err(err) raise YamlException(err) - if self.debug: - self.log.dbg('\"{}\": {} -> {}'.format(k, variables[k], out)) - variables[k] = out + if self._debug: + self._log.dbg('\"{}\": {} -> {}'.format(k, v, out)) + executed[k] = out + return executed - def _template_list(self, t, entries): + def _template_list(self, entries): """template a list of entries""" new = [] if not entries: return new for e in entries: - et = t.generate_string(e) - if self.debug and e != et: - self.log.dbg('resolved: {} -> {}'.format(e, et)) + et = self._template_item(e) + if self._debug and e != et: + self._log.dbg('resolved: {} -> {}'.format(e, et)) new.append(et) return new + def _template_dict(self, entries): + """template a dictionary of entries""" + new = {} + if not entries: + return new + for k, v in entries.items(): + vt = self._template_item(v) + if self._debug and v != vt: + self._log.dbg('resolved: {} -> {}'.format(v, vt)) + new[k] = vt + return new + def _check_minversion(self, minversion): if not minversion: return @@ -1181,10 +1343,10 @@ class CfgYaml: def _debug_dict(self, title, elems): """pretty print dict""" - if not self.debug: + if not self._debug: return - self.log.dbg('{}:'.format(title)) + self._log.dbg('{}:'.format(title)) if not elems: return for k, v in elems.items(): - self.log.dbg('\t- \"{}\": {}'.format(k, v)) + self._log.dbg('\t- \"{}\": {}'.format(k, v)) From bca2cada0da877d54b746c60263491e082ae69df Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 26 Jul 2020 16:19:42 +0200 Subject: [PATCH 06/47] make sure parsing fails early --- tests-ng/compare-ignore-relative.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests-ng/compare-ignore-relative.sh b/tests-ng/compare-ignore-relative.sh index 57c27a3..741aacf 100755 --- a/tests-ng/compare-ignore-relative.sh +++ b/tests-ng/compare-ignore-relative.sh @@ -66,9 +66,9 @@ create_conf ${cfg} # sets token # import echo "[+] import" -cd ${ddpath} | ${bin} import -c ${cfg} ${tmpd}/program -cd ${ddpath} | ${bin} import -c ${cfg} ${tmpd}/config -cd ${ddpath} | ${bin} import -c ${cfg} ${tmpd}/vscode +cd ${ddpath} | ${bin} import --verbose -c ${cfg} ${tmpd}/program || exit 1 +cd ${ddpath} | ${bin} import --verbose -c ${cfg} ${tmpd}/config || exit 1 +cd ${ddpath} | ${bin} import --verbose -c ${cfg} ${tmpd}/vscode || exit 1 # add files on filesystem echo "[+] add files" From 0708e2376ee069597bbbde0f5f02609ad46608da Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 26 Jul 2020 16:25:07 +0200 Subject: [PATCH 07/47] adding command line to debug logs --- dotdrop/options.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dotdrop/options.py b/dotdrop/options.py index cb41914..6f64593 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -124,6 +124,7 @@ class Options(AttrMonitor): self.confpath = self._get_config_path() if self.debug: self.log.dbg('version: {}'.format(VERSION)) + self.log.dbg('command: {}'.format(' '.join(sys.argv))) self.log.dbg('config file: {}'.format(self.confpath)) self._read_config() From 3f6a36ec452a9eb7f1cb83de690db04c816e4714 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 26 Jul 2020 16:26:18 +0200 Subject: [PATCH 08/47] adding debug logs --- tests-ng/duplicate-key.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests-ng/duplicate-key.sh b/tests-ng/duplicate-key.sh index b40451f..9177323 100755 --- a/tests-ng/duplicate-key.sh +++ b/tests-ng/duplicate-key.sh @@ -86,7 +86,7 @@ mkdir -p ${tmpd}/sub/sub echo "test2" > ${tmpd}/sub/sub/abc # import -cd ${ddpath} | ${bin} import -c ${cfg} -p p2 \ +cd ${ddpath} | ${bin} import --verbose -c ${cfg} -p p2 \ ${tmpd}/abc \ ${tmpd}/sub/abc \ ${tmpd}/sub/abc \ @@ -94,7 +94,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p2 \ ${tmpd}/sub/sub2/abc # count dotfiles for p2 -cnt=`cd ${ddpath} | ${bin} files -c ${cfg} -p p2 -b | grep '^f_' | wc -l` +cnt=`cd ${ddpath} | ${bin} files --verbose -c ${cfg} -p p2 -b | grep '^f_' | wc -l` [ "${cnt}" != "4" ] && exit 1 ## CLEANING From 0752c7e4eaac737ba539df992b53aac41e4f7e52 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 26 Jul 2020 17:58:50 +0200 Subject: [PATCH 09/47] fix exception handling --- dotdrop/installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 0d16049..3313eeb 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -329,7 +329,7 @@ class Installer: try: content = templater.generate(src) except UndefinedException as e: - return False, e.message + return False, str(e) finally: templater.restore_vars(saved) if noempty and utils.content_empty(content): From 4bf00d70997faa0deead879d205152fb14949ca8 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 26 Jul 2020 19:47:29 +0200 Subject: [PATCH 10/47] refactoring cfg_yaml --- dotdrop/cfg_yaml.py | 444 +++++++++++++++++++++++++------------------- 1 file changed, 253 insertions(+), 191 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index c2bf973..a6aa8b6 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -104,14 +104,18 @@ class CfgYaml: # 3) "import_configs" variables # 4) other variables # + # variables / dynvariables hints + # - dynvariables are executed in their own config file + # # parse a config file # - parse settings # - parse variables # - interprete dynvariables # - template the include entries - # - parse and integrate included elements (see below) # - parse profiles # - parse dotfiles + # - parse other elements (actions, trans_r, trans_w) + # - parse and integrate included elements (see below) # # parse "include" entry # - same as parse config file @@ -124,7 +128,6 @@ class CfgYaml: # - document dvars are executed in their own config file # - remove unused functions/methods # - coverage - # - remove ori_* # def __init__(self, path, profile=None, debug=False): @@ -142,6 +145,8 @@ class CfgYaml: self._dirty = False # indicates the config has been updated self._dirty_deprecated = False + # profile variables + self._profilevarskeys = [] # init the dictionaries self.settings = {} @@ -159,54 +164,83 @@ class CfgYaml: raise YamlException(err) self._yaml_dict = self._load_yaml(self._path) + # live patch deprecated entries + self._fix_deprecated(self._yaml_dict) + + ################################################## + # parse the config and variables + ################################################## # parse the "config" block self.settings = self._parse_blk_settings(self._yaml_dict) - # parse the "variables" block - self.variables = self._parse_blk_variables(self._yaml_dict) + # base templater (when no vars/dvars exist) + self.variables = self._enrich_vars(self.variables, self._profile) self._redefine_templater() + # parse the "variables" block + var = self._parse_blk_variables(self._yaml_dict) + self._add_variables(var, template=False) + # parse the "dynvariables" block dvariables = self._parse_blk_dynvariables(self._yaml_dict) - self._add_variables(dvariables) + self._add_variables(dvariables, shell=True) - # interprete dynvariables - dvariables = self._template_dict(dvariables) - dvariables = self._shell_exec_dvars(dvariables) + # parse the "profiles" block + self.profiles = self._parse_blk_profiles(self._yaml_dict) - # merge variables and dynvariables - self.variables = self._merge_dict(dvariables, self.variables) - self._redefine_templater() + # include the profile's variables/dynvariables + # last as it overwrites existing ones + pv, pvd = self._get_profile_included_vars() + self._add_variables(pv, prio=True) + self._add_variables(pvd, shell=True, prio=True) + self._profilevarskeys.extend(pv.keys()) + self._profilevarskeys.extend(pvd.keys()) + # TODO handle prio when importing included profile from somewhere else - # TODO template variables + # template variables self.variables = self._template_dict(self.variables) if self._debug: self._debug_dict('variables', self.variables) + ################################################## # template the "include" entries + ################################################## + self._template_include_entry() + ################################################## + # parse the other blocks + ################################################## + + # parse the "dotfiles" block + self.dotfiles = self._parse_blk_dotfiles(self._yaml_dict) + # parse the "actions" block + self.actions = self._parse_blk_actions(self._yaml_dict) + # parse the "trans_r" block + self.trans_r = self._parse_blk_trans_r(self._yaml_dict) + # parse the "trans_w" block + self.trans_w = self._parse_blk_trans_w(self._yaml_dict) + + ################################################## + # import elements + ################################################## + # process imported variables (import_variables) newvars = self._import_variables() + self._clear_profile_vars(newvars) self._add_variables(newvars) - # TODO process imported actions (import_actions) + # process imported actions (import_actions) self._import_actions() - # TODO process imported profile dotfiles (import) + # process imported profile dotfiles (import) self._import_profiles_dotfiles() - # TODO process imported configs (import_configs) + # process imported configs (import_configs) self._import_configs() # process profile include self._resolve_profile_includes() - # ===== - # TODO below - # ==== - if self._debug: - self._log.dbg('BEFORE normalization: {}'.format(self._yaml_dict)) - # resolve variables # TODO # self.variables, self.prokeys = self._merge_variables() @@ -217,32 +251,43 @@ class CfgYaml: # process profile ALL self._resolve_profile_all() # patch dotfiles paths - self._resolve_dotfile_paths() + self._template_dotfiles_paths() # TODO ensure no element is left un-templated at the end if self._debug: - self._log.dbg('AFTER normalization: {}'.format(self._yaml_dict)) + self._log.dbg('########### {} ###########'.format('final config')) + self._debug_entries() - def _add_variables(self, ext_variables): - """add new variables from external file""" + def _add_variables(self, new, shell=False, template=True, prio=False): + """ + add new variables + @shell: execute the variable through the shell + @template: template the variable + @prio: new takes priority over existing variables + """ # TODO move me + if not new: + return # merge - self.variables = self._merge_dict(self.variables, ext_variables) + if prio: + self.variables = self._merge_dict(new, self.variables) + else: + self.variables = self._merge_dict(self.variables, new) # ensure enriched variables are relative to this config self.variables = self._enrich_vars(self.variables, self._profile) # re-create the templater self._redefine_templater() - # rec resolve variables with new ones - self.variables = self._template_dict(self.variables) - - def _redefine_templater(self): - """create templater based on current variables""" - fufile = self.settings[Settings.key_func_file] - fifile = self.settings[Settings.key_filter_file] - self._tmpl = Templategen(variables=self.variables, - func_file=fufile, - filter_file=fifile) + if template: + # rec resolve variables with new ones + self._rec_resolve_variables(self.variables) + if self._debug: + self._debug_dict('variables', self.variables) + if shell: + # shell exec + self._shell_exec_dvars(self.variables, keys=new.keys()) + # re-create the templater + self._redefine_templater() ######################################################## # outside available methods @@ -307,6 +352,8 @@ class CfgYaml: return False if self._debug: self._log.dbg('adding new dotfile: {}'.format(key)) + self._log.dbg('new dotfile src: {}'.format(src)) + self._log.dbg('new dotfile dst: {}'.format(dst)) df_dict = { self.key_dotfile_src: src, @@ -430,33 +477,33 @@ class CfgYaml: def _parse_blk_dotfiles(self, dic): """parse the "dotfiles" block""" - self.ori_dotfiles = self._get_entry(dic, self.key_dotfiles) - self.dotfiles = deepcopy(self.ori_dotfiles) - keys = self.dotfiles.keys() + dotfiles = self._get_entry(dic, self.key_dotfiles) + keys = dotfiles.keys() if len(keys) != len(list(set(keys))): dups = [x for x in keys if x not in list(set(keys))] err = 'duplicate dotfile keys found: {}'.format(dups) raise YamlException(err) - self.dotfiles = self._norm_dotfiles(self.dotfiles) + dotfiles = self._norm_dotfiles(dotfiles) if self._debug: - self._debug_dict('dotfiles', self.dotfiles) + self._debug_dict('dotfiles', dotfiles) + return dotfiles def _parse_blk_profiles(self, dic): """parse the "profiles" block""" - self.ori_profiles = self._get_entry(dic, self.key_profiles) - self.profiles = deepcopy(self.ori_profiles) - self.profiles = self._norm_profiles(self.profiles) + profiles = self._get_entry(dic, self.key_profiles) + profiles = self._norm_profiles(profiles) if self._debug: - self._debug_dict('profiles', self.profiles) + self._debug_dict('profiles', profiles) + return profiles def _parse_blk_actions(self, dic): """parse the "actions" block""" - self.ori_actions = self._get_entry(dic, self.key_actions, - mandatory=False) - self.actions = deepcopy(self.ori_actions) - self.actions = self._norm_actions(self.actions) + actions = self._get_entry(dic, self.key_actions, + mandatory=False) + actions = self._norm_actions(actions) if self._debug: - self._debug_dict('actions', self.actions) + self._debug_dict('actions', actions) + return actions def _parse_blk_trans_r(self, dic): """parse the "trans_r" block""" @@ -466,18 +513,18 @@ class CfgYaml: self._log.warn(msg) dic[self.key_trans_r] = dic[self.old_key_trans_r] del dic[self.old_key_trans_r] - self.ori_trans_r = self._get_entry(dic, key, mandatory=False) - self.trans_r = deepcopy(self.ori_trans_r) + trans_r = self._get_entry(dic, key, mandatory=False) if self._debug: - self._debug_dict('trans_r', self.trans_r) + self._debug_dict('trans_r', trans_r) + return trans_r def _parse_blk_trans_w(self, dic): """parse the "trans_w" block""" - self.ori_trans_w = self._get_entry(dic, self.key_trans_w, - mandatory=False) - self.trans_w = deepcopy(self.ori_trans_w) + trans_w = self._get_entry(dic, self.key_trans_w, + mandatory=False) if self._debug: - self._debug_dict('trans_w', self.trans_w) + self._debug_dict('trans_w', trans_w) + return trans_w def _parse_blk_variables(self, dic): """parse the "variables" block""" @@ -500,71 +547,13 @@ class CfgYaml: ######################################################## # parsing helpers ######################################################## - - def _resolve_dotfile_paths(self): - """resolve dotfiles paths""" - # TODO remove - t = Templategen(variables=self.variables, - func_file=self.settings[Settings.key_func_file], - filter_file=self.settings[Settings.key_filter_file]) - - for dotfile in self.dotfiles.values(): - # src - src = dotfile[self.key_dotfile_src] - newsrc = self.resolve_dotfile_src(src, templater=t) - dotfile[self.key_dotfile_src] = newsrc - # dst - dst = dotfile[self.key_dotfile_dst] - newdst = self.resolve_dotfile_dst(dst, templater=t) - dotfile[self.key_dotfile_dst] = newdst - - def _rec_resolve_vars(self, variables): - """recursive resolve variables""" - # TODO remove this and any call - default = self._get_variables_dict(self._profile) - t = Templategen(variables=self._merge_dict(default, variables), - func_file=self.settings[Settings.key_func_file], - filter_file=self.settings[Settings.key_filter_file]) - for k in variables.keys(): - val = variables[k] - while Templategen.var_is_template(val): - val = t.generate_string(val) - variables[k] = val - t.update_variables(variables) - return variables - - def _get_profile_included_vars(self, tvars): - """resolve profile included variables/dynvariables""" - # TODO remove - t = Templategen(variables=tvars, - func_file=self.settings[Settings.key_func_file], - filter_file=self.settings[Settings.key_filter_file]) - - for k, v in self.profiles.items(): - if self.key_profile_include in v: - new = [] - for x in v[self.key_profile_include]: - new.append(t.generate_string(x)) - v[self.key_profile_include] = new - - # now get the included ones - pro_var = self._get_profile_included_item(self._profile, - self.key_profile_variables, - seen=[self._profile]) - pro_dvar = self._get_profile_included_item(self._profile, - self.key_profile_dvariables, - seen=[self._profile]) - - # exec incl dynvariables - self._shell_exec_dvars(pro_dvar.keys(), pro_dvar) - return pro_var, pro_dvar - def _merge_variables(self): """ resolve all variables across the config apply them to any needed entries and return the full list of variables """ + # TODO remove me ??? if self._debug: self._log.dbg('get local variables') @@ -587,7 +576,7 @@ class CfgYaml: self._debug_dict('variables', merged) # resolve profile included variables/dynvariables - pro_var, pro_dvar = self._get_profile_included_vars(merged) + pro_var, pro_dvar = self._get_profile_included_vars() # merge all and resolve merged = self._merge_dict(pro_var, merged) @@ -706,6 +695,7 @@ class CfgYaml: def _get_dvariables_dict(self): """return dynvariables""" + # TODO remove me ?? variables = deepcopy(self.ori_dvariables) return variables @@ -746,7 +736,7 @@ class CfgYaml: v[self.key_profile_dotfiles] = self.dotfiles.keys() def _resolve_profile_includes(self): - # profiles -> include other profile + """resolve profile(s) including other profiles""" for k, v in self.profiles.items(): self._rec_resolve_profile_include(k) @@ -768,6 +758,7 @@ class CfgYaml: dotfiles = this_profile.get(self.key_profile_dotfiles, []) or [] actions = this_profile.get(self.key_profile_actions, []) or [] includes = this_profile.get(self.key_profile_include, []) or [] + # TODO ignore those as already included through _get_profiles_vars pvars = this_profile.get(self.key_profile_variables, {}) or {} pdvars = this_profile.get(self.key_profile_dvariables, {}) or {} if not includes: @@ -853,7 +844,7 @@ class CfgYaml: if profile == self._profile: # Only for the selected profile, we execute dynamic variables and # we merge variables/dynvariables into the global variables - self._shell_exec_dvars(pdvars.keys(), pdvars) + self._shell_exec_dvars(pdvars) self.variables = self._merge_dict(pvars, self.variables) self.variables = self._merge_dict(pdvars, self.variables) @@ -882,11 +873,15 @@ class CfgYaml: self._log.dbg('import dynvariables from {}'.format(path)) dvar = self._import_sub(path, self.key_dvariables, mandatory=False) + merged = self._merge_dict(dvar, var) - merged = self._template_dict(merged) + self._rec_resolve_variables(merged) + if dvar.keys(): + self._shell_exec_dvars(merged, keys=dvar.keys()) + self._clear_profile_vars(merged) newvars = self._merge_dict(newvars, merged) - # TODO needed? - # self._clear_profile_vars(merged) + if self._debug: + self._debug_dict('imported variables', newvars) return newvars def _import_actions(self): @@ -999,6 +994,8 @@ class CfgYaml: def _fix_deprecated(self, yamldict): """fix deprecated entries""" + if not yamldict: + return self._fix_deprecated_link_by_default(yamldict) self._fix_deprecated_dotfile_link(yamldict) return yamldict @@ -1088,8 +1085,7 @@ class CfgYaml: except Exception as e: self._log.err(e) raise YamlException('invalid config: {}'.format(path)) - # live patch deprecated entries - return self._fix_deprecated(content) + return content def _yaml_load(self, path): """load from yaml""" @@ -1107,14 +1103,120 @@ class CfgYaml: y.typ = 'rt' y.dump(content, where) + ######################################################## + # templating + ######################################################## + + def _redefine_templater(self): + """create templater based on current variables""" + fufile = self.settings[Settings.key_func_file] + fifile = self.settings[Settings.key_filter_file] + self._tmpl = Templategen(variables=self.variables, + func_file=fufile, + filter_file=fifile) + + def _template_item(self, item, exc_if_fail=True): + """ + template an item using the templategen + will raise an exception if template failed and exc_if_fail + """ + # TODO use this across the entire file + if not Templategen.var_is_template(item): + return item + try: + val = item + while Templategen.var_is_template(val): + val = self._tmpl.generate_string(val) + except UndefinedException as e: + if exc_if_fail: + raise e + return val + + def _template_list(self, entries): + """template a list of entries""" + new = [] + if not entries: + return new + for e in entries: + et = self._template_item(e) + if self._debug and e != et: + self._log.dbg('resolved: {} -> {}'.format(e, et)) + new.append(et) + return new + + def _template_dict(self, entries): + """template a dictionary of entries""" + new = {} + if not entries: + return new + for k, v in entries.items(): + vt = self._template_item(v) + if self._debug and v != vt: + self._log.dbg('resolved: {} -> {}'.format(v, vt)) + new[k] = vt + return new + + def _template_dotfiles_paths(self): + """template dotfiles paths""" + for dotfile in self.dotfiles.values(): + # src + src = dotfile[self.key_dotfile_src] + newsrc = self.resolve_dotfile_src(src, templater=self._tmpl) + dotfile[self.key_dotfile_src] = newsrc + # dst + dst = dotfile[self.key_dotfile_dst] + newdst = self.resolve_dotfile_dst(dst, templater=self._tmpl) + dotfile[self.key_dotfile_dst] = newdst + + def _rec_resolve_variables(self, variables): + """recursive resolve variables""" + var = self._enrich_vars(variables, self._profile) + # use a separated templategen to handle variables + # resolved outside the main config + t = Templategen(variables=var, + func_file=self.settings[Settings.key_func_file], + filter_file=self.settings[Settings.key_filter_file]) + for k in variables.keys(): + val = variables[k] + while Templategen.var_is_template(val): + val = t.generate_string(val) + variables[k] = val + t.update_variables(variables) + + def _get_profile_included_vars(self): + """resolve profile included variables/dynvariables""" + for k, v in self.profiles.items(): + if self.key_profile_include in v: + new = [] + for x in v[self.key_profile_include]: + new.append(self._tmpl.generate_string(x)) + v[self.key_profile_include] = new + + # now get the included ones + pro_var = self._get_profile_included_item(self._profile, + self.key_profile_variables, + seen=[self._profile]) + pro_dvar = self._get_profile_included_item(self._profile, + self.key_profile_dvariables, + seen=[self._profile]) + + # exec incl dynvariables + return pro_var, pro_dvar + ######################################################## # helpers ######################################################## def _clear_profile_vars(self, dic): - """remove profile variables from dic if found""" - # TODO check why this is used at all - [dic.pop(k, None) for k in self.prokeys] + """ + remove profile variables from dic if found inplace + to avoid profile variables being overwriten + """ + # TODO + # [dic.pop(k, None) for k in self.prokeys] + if not dic: + return + [dic.pop(k, None) for k in self._profilevarskeys] def _parse_extended_import_path(self, path_entry): """Parse an import path in a tuple (path, fatal_not_found).""" @@ -1161,8 +1263,7 @@ class CfgYaml: return None def _process_path(self, path_entry): - """Process a path entry to a normalized form. - + """ This method processed a path entry. Namely it: - Normalizes the path. - Expands globs. @@ -1170,11 +1271,6 @@ class CfgYaml: This method always returns a list containing only absolute paths existing on the filesystem. If the input is not a glob, the list contains at most one element, otheriwse it could hold more. - - :param path_entry: A path with an optional attribute. - :type path_entry: str - :return: A list of normalized existing paths, obtained from the input. - :rtype: List of str """ path, fatal_not_found = self._parse_extended_import_path(path_entry) path = self._norm_path(path) @@ -1190,37 +1286,14 @@ class CfgYaml: return [p for p in checked_paths if p] def _resolve_paths(self, paths): - """Resolve a list of path to existing paths. - + """ This function resolves a list of paths. This means normalizing, expanding globs and checking for existence, taking in account fatal_not_found flags. - - :param paths: A list of paths. Might contain globs and options. - :type paths: List of str - :return: A list of processed paths. - :rtype: List of str """ processed_paths = (self._process_path(p) for p in paths) return list(chain.from_iterable(processed_paths)) - def _template_item(self, item, exc_if_fail=True): - """ - template an item using the templategen - will raise an exception if template failed and exc_if_fail - """ - # TODO use this across the entire file - if not Templategen.var_is_template(item): - return item - try: - val = item - while Templategen.var_is_template(val): - val = self._tmpl.generate_string(val) - except UndefinedException as e: - if exc_if_fail: - raise e - return val - def _merge_dict(self, high, low): """merge high and low dict""" if not high: @@ -1230,16 +1303,16 @@ class CfgYaml: return {**low, **high} def _get_entry(self, dic, key, mandatory=True): - """return entry from yaml dictionary""" + """return copy of entry from yaml dictionary""" if key not in dic: if mandatory: raise YamlException('invalid config: no {} found'.format(key)) dic[key] = {} - return dic[key] + return deepcopy(dic[key]) if mandatory and not dic[key]: # ensure is not none dic[key] = {} - return dic[key] + return deepcopy(dic[key]) def _clear_none(self, dic): """recursively delete all none/empty values in a dictionary.""" @@ -1288,11 +1361,13 @@ class CfgYaml: self._log.dbg('normalizing: {} -> {}'.format(path, ret)) return ret - def _shell_exec_dvars(self, dic): - """shell execute dynvariables""" + def _shell_exec_dvars(self, dic, keys=[]): + """shell execute dynvariables in-place""" # TODO remove other calls outside initial setup of dvars - executed = {} - for k, v in dic.items(): + if not keys: + keys = dic.keys() + for k in keys: + v = dic[k] ret, out = shell(v, debug=self._debug) if not ret: err = 'var \"{}: {}\" failed: {}'.format(k, v, out) @@ -1300,32 +1375,7 @@ class CfgYaml: raise YamlException(err) if self._debug: self._log.dbg('\"{}\": {} -> {}'.format(k, v, out)) - executed[k] = out - return executed - - def _template_list(self, entries): - """template a list of entries""" - new = [] - if not entries: - return new - for e in entries: - et = self._template_item(e) - if self._debug and e != et: - self._log.dbg('resolved: {} -> {}'.format(e, et)) - new.append(et) - return new - - def _template_dict(self, entries): - """template a dictionary of entries""" - new = {} - if not entries: - return new - for k, v in entries.items(): - vt = self._template_item(v) - if self._debug and v != vt: - self._log.dbg('resolved: {} -> {}'.format(v, vt)) - new[k] = vt - return new + dic[k] = out def _check_minversion(self, minversion): if not minversion: @@ -1341,6 +1391,18 @@ class CfgYaml: err += ' Please update.' raise YamlException(err) + def _debug_entries(self): + """debug print all interesting entries""" + if not self._debug: + return + self._debug_dict('settings', self.settings) + self._debug_dict('dotfiles', self.dotfiles) + self._debug_dict('profiles', self.profiles) + self._debug_dict('actions', self.actions) + self._debug_dict('trans_r', self.trans_r) + self._debug_dict('trans_w', self.trans_w) + self._debug_dict('variables', self.variables) + def _debug_dict(self, title, elems): """pretty print dict""" if not self._debug: From edb897deff4bf76d8821eeeb983ef9b6a99c5de2 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Aug 2020 14:46:40 +0200 Subject: [PATCH 11/47] fix test --- tests-ng/profile-dyninclude.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests-ng/profile-dyninclude.sh b/tests-ng/profile-dyninclude.sh index 4cb09b1..9040b33 100755 --- a/tests-ng/profile-dyninclude.sh +++ b/tests-ng/profile-dyninclude.sh @@ -135,7 +135,7 @@ grep 'maindyncontent' ${tmpd}/abc >/dev/null || (echo "dynvariables 1 not resolv grep 'subcontent' ${tmpd}/abc >/dev/null || (echo "variables 2 not resolved" && exit 1) grep 'subdyncontent' ${tmpd}/abc >/dev/null || (echo "dynvariables 2 not resolved" && exit 1) #cat ${tmpd}/abc -[ ! -e ${tmpd}/ghi] && exit 1 +[ ! -e ${tmpd}/ghi ] && exit 1 ## CLEANING rm -rf ${tmps} ${tmpd} From 45a1db6702f5ca3ff689c793fc35b271a98696df Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Aug 2020 14:49:17 +0200 Subject: [PATCH 12/47] refactoring cfg_yaml --- dotdrop/cfg_yaml.py | 297 ++++++++++++++++++++++---------------------- 1 file changed, 150 insertions(+), 147 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index a6aa8b6..1f179c0 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -104,8 +104,11 @@ class CfgYaml: # 3) "import_configs" variables # 4) other variables # - # variables / dynvariables hints + # rules: # - dynvariables are executed in their own config file + # - profile cannot include profiles defined above in the import tree + # - config files do not have access to variables + # defined above in the import tree # # parse a config file # - parse settings @@ -130,11 +133,12 @@ class CfgYaml: # - coverage # - def __init__(self, path, profile=None, debug=False): + def __init__(self, path, profile=None, addprofiles=[], debug=False): """ config parser @path: config file path @profile: the selected profile + @addprofiles: included profiles @debug: debug flag """ self._path = os.path.abspath(path) @@ -147,6 +151,8 @@ class CfgYaml: self._dirty_deprecated = False # profile variables self._profilevarskeys = [] + # included profiles + self._inc_profiles = addprofiles # init the dictionaries self.settings = {} @@ -160,7 +166,7 @@ class CfgYaml: if not os.path.exists(self._path): err = 'invalid config path: \"{}\"'.format(path) if self._debug: - self._log.dbg(err) + self._dbg(err) raise YamlException(err) self._yaml_dict = self._load_yaml(self._path) @@ -189,9 +195,9 @@ class CfgYaml: # parse the "profiles" block self.profiles = self._parse_blk_profiles(self._yaml_dict) - # include the profile's variables/dynvariables - # last as it overwrites existing ones - pv, pvd = self._get_profile_included_vars() + # include the profile's variables/dynvariables last + # as it overwrites existing ones + self._inc_profiles, pv, pvd = self._get_profile_included_vars() self._add_variables(pv, prio=True) self._add_variables(pvd, shell=True, prio=True) self._profilevarskeys.extend(pv.keys()) @@ -201,13 +207,15 @@ class CfgYaml: # template variables self.variables = self._template_dict(self.variables) if self._debug: - self._debug_dict('variables', self.variables) + self._debug_dict('current variables defined', self.variables) ################################################## # template the "include" entries ################################################## self._template_include_entry() + if self._debug: + self._debug_dict('current variables defined', self.variables) ################################################## # parse the other blocks @@ -241,8 +249,18 @@ class CfgYaml: # process profile include self._resolve_profile_includes() + # TODO TODO + # add the current profile variables + _, pv, pvd = self._get_profile_included_vars() + self._add_variables(pv, prio=True) + self._add_variables(pvd, shell=True, prio=True) + self._profilevarskeys.extend(pv.keys()) + self._profilevarskeys.extend(pvd.keys()) + # resolve variables # TODO + self._clear_profile_vars(newvars) + self._add_variables(newvars) # self.variables, self.prokeys = self._merge_variables() # apply variables @@ -256,7 +274,7 @@ class CfgYaml: # TODO ensure no element is left un-templated at the end if self._debug: - self._log.dbg('########### {} ###########'.format('final config')) + self._dbg('########### {} ###########'.format('final config')) self._debug_entries() def _add_variables(self, new, shell=False, template=True, prio=False): @@ -281,8 +299,6 @@ class CfgYaml: if template: # rec resolve variables with new ones self._rec_resolve_variables(self.variables) - if self._debug: - self._debug_dict('variables', self.variables) if shell: # shell exec self._shell_exec_dvars(self.variables, keys=new.keys()) @@ -302,7 +318,7 @@ class CfgYaml: new = templater.generate_string(src) if new != src and self._debug: msg = 'dotfile src: \"{}\" -> \"{}\"'.format(src, new) - self._log.dbg(msg) + self._dbg(msg) src = new src = os.path.join(self.settings[self.key_settings_dotpath], src) @@ -318,7 +334,7 @@ class CfgYaml: new = templater.generate_string(dst) if new != dst and self._debug: msg = 'dotfile dst: \"{}\" -> \"{}\"'.format(dst, new) - self._log.dbg(msg) + self._dbg(msg) dst = new newdst = self._norm_path(dst) return newdst @@ -338,7 +354,7 @@ class CfgYaml: msg = 'add \"{}\" to profile \"{}\"'.format(dotfile_key, profile_key) msg.format(dotfile_key, profile_key) - self._log.dbg(msg) + self._dbg(msg) self._dirty = True return self._dirty @@ -351,9 +367,9 @@ class CfgYaml: if key in self.dotfiles.keys(): return False if self._debug: - self._log.dbg('adding new dotfile: {}'.format(key)) - self._log.dbg('new dotfile src: {}'.format(src)) - self._log.dbg('new dotfile dst: {}'.format(dst)) + self._dbg('adding new dotfile: {}'.format(key)) + self._dbg('new dotfile src: {}'.format(src)) + self._dbg('new dotfile dst: {}'.format(dst)) df_dict = { self.key_dotfile_src: src, @@ -371,11 +387,11 @@ class CfgYaml: self._log.err('key not in dotfiles: {}'.format(key)) return False if self._debug: - self._log.dbg('remove dotfile: {}'.format(key)) + self._dbg('remove dotfile: {}'.format(key)) del self._yaml_dict[self.key_dotfiles][key] if self._debug: dfs = self._yaml_dict[self.key_dotfiles] - self._log.dbg('new dotfiles: {}'.format(dfs)) + self._dbg('new dotfiles: {}'.format(dfs)) self._dirty = True return True @@ -393,12 +409,12 @@ class CfgYaml: return True if self._debug: dfs = profile[self.key_profile_dotfiles] - self._log.dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) - self._log.dbg('remove {} from profile {}'.format(df_key, pro_key)) + self._dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) + self._dbg('remove {} from profile {}'.format(df_key, pro_key)) profile[self.key_profile_dotfiles].remove(df_key) if self._debug: dfs = profile[self.key_profile_dotfiles] - self._log.dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) + self._dbg('{} profile dotfiles: {}'.format(pro_key, dfs)) self._dirty = True return True @@ -416,7 +432,7 @@ class CfgYaml: # save to file if self._debug: - self._log.dbg('saving to {}'.format(self._path)) + self._dbg('saving to {}'.format(self._path)) try: with open(self._path, 'w') as f: self._yaml_dump(content, f) @@ -472,7 +488,7 @@ class CfgYaml: ] settings[Settings.key_func_file] = p if self._debug: - self._debug_dict('settings', settings) + self._debug_dict('settings block:', settings) return settings def _parse_blk_dotfiles(self, dic): @@ -485,7 +501,7 @@ class CfgYaml: raise YamlException(err) dotfiles = self._norm_dotfiles(dotfiles) if self._debug: - self._debug_dict('dotfiles', dotfiles) + self._debug_dict('dotfiles block', dotfiles) return dotfiles def _parse_blk_profiles(self, dic): @@ -493,7 +509,7 @@ class CfgYaml: profiles = self._get_entry(dic, self.key_profiles) profiles = self._norm_profiles(profiles) if self._debug: - self._debug_dict('profiles', profiles) + self._debug_dict('profiles block', profiles) return profiles def _parse_blk_actions(self, dic): @@ -502,7 +518,7 @@ class CfgYaml: mandatory=False) actions = self._norm_actions(actions) if self._debug: - self._debug_dict('actions', actions) + self._debug_dict('actions block', actions) return actions def _parse_blk_trans_r(self, dic): @@ -515,7 +531,7 @@ class CfgYaml: del dic[self.old_key_trans_r] trans_r = self._get_entry(dic, key, mandatory=False) if self._debug: - self._debug_dict('trans_r', trans_r) + self._debug_dict('trans_r block', trans_r) return trans_r def _parse_blk_trans_w(self, dic): @@ -523,7 +539,7 @@ class CfgYaml: trans_w = self._get_entry(dic, self.key_trans_w, mandatory=False) if self._debug: - self._debug_dict('trans_w', trans_w) + self._debug_dict('trans_w block', trans_w) return trans_w def _parse_blk_variables(self, dic): @@ -532,7 +548,7 @@ class CfgYaml: self.key_variables, mandatory=False) if self._debug: - self._debug_dict('variables', variables) + self._debug_dict('variables block', variables) return variables def _parse_blk_dynvariables(self, dic): @@ -541,7 +557,7 @@ class CfgYaml: self.key_dvariables, mandatory=False) if self._debug: - self._debug_dict('dynvariables', dvariables) + self._debug_dict('dynvariables block', dvariables) return dvariables ######################################################## @@ -555,7 +571,7 @@ class CfgYaml: """ # TODO remove me ??? if self._debug: - self._log.dbg('get local variables') + self._dbg('get local variables') # get all variables from local and resolve var = self._get_variables_dict(self._profile) @@ -572,11 +588,11 @@ class CfgYaml: self._shell_exec_dvars(dvar.keys(), merged) if self._debug: - self._log.dbg('local variables resolved') + self._dbg('local variables resolved') self._debug_dict('variables', merged) # resolve profile included variables/dynvariables - pro_var, pro_dvar = self._get_profile_included_vars() + _, pro_var, pro_dvar = self._get_profile_included_vars() # merge all and resolve merged = self._merge_dict(pro_var, merged) @@ -584,7 +600,7 @@ class CfgYaml: merged = self._rec_resolve_vars(merged) if self._debug: - self._log.dbg('resolve all uses of variables in config') + self._dbg('resolve all uses of variables in config') self._debug_dict('variables', merged) prokeys = list(pro_var.keys()) + list(pro_dvar.keys()) @@ -699,8 +715,18 @@ class CfgYaml: variables = deepcopy(self.ori_dvariables) return variables - def _get_profile_included_item(self, profile, item, seen): - """recursively get included from profile""" + def _get_profile_included_item(self, keyitem): + """recursively get included in profile""" + profiles = [self._profile] + self._inc_profiles + items = {} + for profile in profiles: + seen = [self._profile] + i = self.__get_profile_included_item(profile, keyitem, seen) + items = self._merge_dict(i, items) + return items + + def __get_profile_included_item(self, profile, keyitem, seen): + """recursively get included from profile""" items = {} if not profile or profile not in self.profiles.keys(): return items @@ -708,19 +734,19 @@ class CfgYaml: # considered profile entry pentry = self.profiles.get(profile) - # recursively get from inherited profile + # recursively get from inherited profile for inherited_profile in pentry.get(self.key_profile_include, []): if inherited_profile == profile or inherited_profile in seen: raise YamlException('\"include\" loop') seen.append(inherited_profile) - new = self._get_profile_included_item(inherited_profile, - item, seen) + new = self.__get_profile_included_item(inherited_profile, + keyitem, seen) if self._debug: msg = 'included {} from {}: {}' - self._log.dbg(msg.format(item, inherited_profile, new)) + self._dbg(msg.format(keyitem, inherited_profile, new)) items.update(new) - cur = pentry.get(item, {}) + cur = pentry.get(keyitem, {}) return self._merge_dict(cur, items) def _resolve_profile_all(self): @@ -732,7 +758,7 @@ class CfgYaml: continue if self.key_all in dfs: if self._debug: - self._log.dbg('add ALL to profile {}'.format(k)) + self._dbg('add ALL to profile {}'.format(k)) v[self.key_profile_dotfiles] = self.dotfiles.keys() def _resolve_profile_includes(self): @@ -745,12 +771,7 @@ class CfgYaml: recursively resolve include of other profiles's: * dotfiles * actions - * variables - * dynvariables - variables/dynvariables are directly merged with the - global variables (self.variables) if these are - included in the selected profile - returns dotfiles, actions, variables, dynvariables + returns dotfiles, actions """ this_profile = self.profiles[profile] @@ -758,29 +779,22 @@ class CfgYaml: dotfiles = this_profile.get(self.key_profile_dotfiles, []) or [] actions = this_profile.get(self.key_profile_actions, []) or [] includes = this_profile.get(self.key_profile_include, []) or [] - # TODO ignore those as already included through _get_profiles_vars - pvars = this_profile.get(self.key_profile_variables, {}) or {} - pdvars = this_profile.get(self.key_profile_dvariables, {}) or {} if not includes: # nothing to include - return dotfiles, actions, pvars, pdvars + return dotfiles, actions if self._debug: - self._log.dbg('{} includes {}'.format(profile, ','.join(includes))) - self._log.dbg('{} dotfiles before include: {}'.format(profile, - dotfiles)) - self._log.dbg('{} actions before include: {}'.format(profile, - actions)) - self._log.dbg('{} variables before include: {}'.format(profile, - pvars)) - self._log.dbg('{} dynvariables before include: {}'.format(profile, - pdvars)) + self._dbg('{} includes {}'.format(profile, ','.join(includes))) + self._dbg('{} dotfiles before include: {}'.format(profile, + dotfiles)) + self._dbg('{} actions before include: {}'.format(profile, + actions)) seen = [] for i in uniq_list(includes): if self._debug: - self._log.dbg('resolving includes "{}" <- "{}"' - .format(profile, i)) + self._dbg('resolving includes "{}" <- "{}"' + .format(profile, i)) # ensure no include loop occurs if i in seen: @@ -793,65 +807,37 @@ class CfgYaml: # recursive resolve if self._debug: - self._log.dbg('recursively resolving includes for profile "{}"' - .format(i)) - o_dfs, o_actions, o_v, o_dv = self._rec_resolve_profile_include(i) + self._dbg('recursively resolving includes for profile "{}"' + .format(i)) + o_dfs, o_actions = self._rec_resolve_profile_include(i) # merge dotfile keys if self._debug: - self._log.dbg('Merging dotfiles {} <- {}: {} <- {}' - .format(profile, i, dotfiles, o_dfs)) + self._dbg('Merging dotfiles {} <- {}: {} <- {}' + .format(profile, i, dotfiles, o_dfs)) dotfiles.extend(o_dfs) this_profile[self.key_profile_dotfiles] = uniq_list(dotfiles) # merge actions keys if self._debug: - self._log.dbg('Merging actions {} <- {}: {} <- {}' - .format(profile, i, actions, o_actions)) + self._dbg('Merging actions {} <- {}: {} <- {}' + .format(profile, i, actions, o_actions)) actions.extend(o_actions) this_profile[self.key_profile_actions] = uniq_list(actions) - # merge variables - if self._debug: - self._log.dbg('Merging variables {} <- {}: {} <- {}' - .format(profile, i, dict(pvars), dict(o_v))) - pvars = self._merge_dict(o_v, pvars) - this_profile[self.key_profile_variables] = pvars - - # merge dynvariables - if self._debug: - self._log.dbg('Merging dynamic variables {} <- {}: {} <- {}' - .format(profile, i, dict(pdvars), - dict(o_dv))) - pdvars = self._merge_dict(o_dv, pdvars) - this_profile[self.key_profile_dvariables] = pdvars - dotfiles = this_profile.get(self.key_profile_dotfiles, []) actions = this_profile.get(self.key_profile_actions, []) - pvars = this_profile.get(self.key_profile_variables, {}) or {} - pdvars = this_profile.get(self.key_profile_dvariables, {}) or {} if self._debug: - self._log.dbg('{} dotfiles after include: {}'.format(profile, - dotfiles)) - self._log.dbg('{} actions after include: {}'.format(profile, - actions)) - self._log.dbg('{} variables after include: {}'.format(profile, - pvars)) - self._log.dbg('{} dynvariables after include: {}'.format(profile, - pdvars)) - - if profile == self._profile: - # Only for the selected profile, we execute dynamic variables and - # we merge variables/dynvariables into the global variables - self._shell_exec_dvars(pdvars) - self.variables = self._merge_dict(pvars, self.variables) - self.variables = self._merge_dict(pdvars, self.variables) + self._dbg('{} dotfiles after include: {}'.format(profile, + dotfiles)) + self._dbg('{} actions after include: {}'.format(profile, + actions)) # since included items are resolved here # we can clear these include - self.profiles[profile][self.key_profile_include] = None - return dotfiles, actions, pvars, pdvars + self.profiles[profile][self.key_profile_include] = [] + return dotfiles, actions ######################################################## # handle imported entries @@ -866,11 +852,11 @@ class CfgYaml: newvars = {} for path in paths: if self._debug: - self._log.dbg('import variables from {}'.format(path)) + self._dbg('import variables from {}'.format(path)) var = self._import_sub(path, self.key_variables, mandatory=False) if self._debug: - self._log.dbg('import dynvariables from {}'.format(path)) + self._dbg('import dynvariables from {}'.format(path)) dvar = self._import_sub(path, self.key_dvariables, mandatory=False) @@ -892,7 +878,7 @@ class CfgYaml: paths = self._resolve_paths(paths) for path in paths: if self._debug: - self._log.dbg('import actions from {}'.format(path)) + self._dbg('import actions from {}'.format(path)) new = self._import_sub(path, self.key_actions, mandatory=False, patch_func=self._norm_actions) @@ -905,7 +891,7 @@ class CfgYaml: if not imp: continue if self._debug: - self._log.dbg('import dotfiles for profile {}'.format(k)) + self._dbg('import dotfiles for profile {}'.format(k)) paths = self._resolve_paths(imp) for path in paths: current = v.get(self.key_dotfiles, []) @@ -916,8 +902,10 @@ class CfgYaml: def _import_config(self, path): """import config from path""" if self._debug: - self._log.dbg('import config from {}'.format(path)) - sub = CfgYaml(path, profile=self._profile, debug=self._debug) + self._dbg('import config from {}'.format(path)) + sub = CfgYaml(path, profile=self._profile, + addprofiles=self._inc_profiles, + debug=self._debug) # settings are ignored from external file # except for filter_file and func_file @@ -958,19 +946,19 @@ class CfgYaml: patch_func is applied to each element if defined """ if self._debug: - self._log.dbg('import \"{}\" from \"{}\"'.format(key, path)) + self._dbg('import \"{}\" from \"{}\"'.format(key, path)) extdict = self._load_yaml(path) new = self._get_entry(extdict, key, mandatory=mandatory) if patch_func: if self._debug: - self._log.dbg('calling patch: {}'.format(patch_func)) + self._dbg('calling patch: {}'.format(patch_func)) new = patch_func(new) if not new and mandatory: err = 'no \"{}\" imported from \"{}\"'.format(key, path) self._log.warn(err) raise YamlException(err) if self._debug: - self._log.dbg('imported \"{}\": {}'.format(key, new)) + self._dbg('imported \"{}\": {}'.format(key, new)) return new ######################################################## @@ -985,7 +973,7 @@ class CfgYaml: self.key_profile_dotfiles: [] } if self._debug: - self._log.dbg('adding new profile: {}'.format(key)) + self._dbg('adding new profile: {}'.format(key)) self._dirty = True ######################################################## @@ -1073,13 +1061,13 @@ class CfgYaml: """load a yaml file to a dict""" content = {} if self._debug: - self._log.dbg('----------start:{}----------'.format(path)) + self._dbg('----------start:{}----------'.format(path)) cfg = '\n' with open(path, 'r') as f: for line in f: cfg += line - self._log.dbg(cfg.rstrip()) - self._log.dbg('----------end:{}----------'.format(path)) + self._dbg(cfg.rstrip()) + self._dbg('----------end:{}----------'.format(path)) try: content = self._yaml_load(path) except Exception as e: @@ -1140,7 +1128,7 @@ class CfgYaml: for e in entries: et = self._template_item(e) if self._debug and e != et: - self._log.dbg('resolved: {} -> {}'.format(e, et)) + self._dbg('resolved: {} -> {}'.format(e, et)) new.append(et) return new @@ -1152,7 +1140,7 @@ class CfgYaml: for k, v in entries.items(): vt = self._template_item(v) if self._debug and v != vt: - self._log.dbg('resolved: {} -> {}'.format(v, vt)) + self._dbg('resolved: {} -> {}'.format(v, vt)) new[k] = vt return new @@ -1186,22 +1174,24 @@ class CfgYaml: def _get_profile_included_vars(self): """resolve profile included variables/dynvariables""" for k, v in self.profiles.items(): - if self.key_profile_include in v: + if self.key_profile_include in v and v[self.key_profile_include]: new = [] for x in v[self.key_profile_include]: new.append(self._tmpl.generate_string(x)) v[self.key_profile_include] = new # now get the included ones - pro_var = self._get_profile_included_item(self._profile, - self.key_profile_variables, - seen=[self._profile]) - pro_dvar = self._get_profile_included_item(self._profile, - self.key_profile_dvariables, - seen=[self._profile]) + pro_var = self._get_profile_included_item(self.key_profile_variables) + pro_dvar = self._get_profile_included_item(self.key_profile_dvariables) + + # the included profiles + inc_profiles = [] + if self._profile and self._profile in self.profiles.keys(): + pentry = self.profiles.get(self._profile) + inc_profiles = pentry.get(self.key_profile_include, []) # exec incl dynvariables - return pro_var, pro_dvar + return inc_profiles, pro_var, pro_dvar ######################################################## # helpers @@ -1221,7 +1211,7 @@ class CfgYaml: def _parse_extended_import_path(self, path_entry): """Parse an import path in a tuple (path, fatal_not_found).""" if self._debug: - self._log.dbg('parsing path entry {}'.format(path_entry)) + self._dbg('parsing path entry {}'.format(path_entry)) path, _, attribute = path_entry.rpartition(self.key_import_sep) fatal_not_found = attribute != self.key_import_ignore_key @@ -1235,13 +1225,13 @@ class CfgYaml: # str.rpartition # In both cases, path_entry is the path we're looking for. if self._debug: - self._log.dbg('using attribute default values for path {}' - .format(path_entry)) + self._dbg('using attribute default values for path {}' + .format(path_entry)) path = path_entry fatal_not_found = self.key_import_fatal_not_found elif self._debug: - self._log.dbg('path entry {} has fatal_not_found flag set to {}' - .format(path_entry, fatal_not_found)) + self._dbg('path entry {} has fatal_not_found flag set to {}' + .format(path_entry, fatal_not_found)) return path, fatal_not_found def _handle_non_existing_path(self, path, fatal_not_found=True): @@ -1255,7 +1245,7 @@ class CfgYaml: """Check if a path exists, raising if necessary.""" if os.path.exists(path): if self._debug: - self._log.dbg('path {} exists'.format(path)) + self._dbg('path {} exists'.format(path)) return path self._handle_non_existing_path(path, fatal_not_found) @@ -1277,7 +1267,7 @@ class CfgYaml: paths = self._glob_path(path) if self._is_glob(path) else [path] if not paths: if self._debug: - self._log.dbg("glob path {} didn't expand".format(path)) + self._dbg("glob path {} didn't expand".format(path)) self._handle_non_existing_path(path, fatal_not_found) return [] @@ -1340,7 +1330,7 @@ class CfgYaml: def _glob_path(self, path): """Expand a glob.""" if self._debug: - self._log.dbg('expanding glob {}'.format(path)) + self._dbg('expanding glob {}'.format(path)) expanded_path = os.path.expanduser(path) return glob.glob(expanded_path, recursive=True) @@ -1354,11 +1344,11 @@ class CfgYaml: ret = os.path.join(d, path) if self._debug: msg = 'normalizing relative to cfg: {} -> {}' - self._log.dbg(msg.format(path, ret)) + self._dbg(msg.format(path, ret)) return ret ret = os.path.normpath(path) if self._debug and path != ret: - self._log.dbg('normalizing: {} -> {}'.format(path, ret)) + self._dbg('normalizing: {} -> {}'.format(path, ret)) return ret def _shell_exec_dvars(self, dic, keys=[]): @@ -1374,7 +1364,7 @@ class CfgYaml: self._log.err(err) raise YamlException(err) if self._debug: - self._log.dbg('\"{}\": {} -> {}'.format(k, v, out)) + self._dbg('\"{}\": {} -> {}'.format(k, v, out)) dic[k] = out def _check_minversion(self, minversion): @@ -1391,24 +1381,37 @@ class CfgYaml: err += ' Please update.' raise YamlException(err) + def _profile_yes(self, profile): + """returns True if the profile is to be considered""" + if profile == self._profile: + return True + if profile in self._inc_profiles: + return True + return False + def _debug_entries(self): """debug print all interesting entries""" if not self._debug: return - self._debug_dict('settings', self.settings) - self._debug_dict('dotfiles', self.dotfiles) - self._debug_dict('profiles', self.profiles) - self._debug_dict('actions', self.actions) - self._debug_dict('trans_r', self.trans_r) - self._debug_dict('trans_w', self.trans_w) - self._debug_dict('variables', self.variables) + self._dbg('Current entries') + self._debug_dict('entry settings', self.settings) + self._debug_dict('entry dotfiles', self.dotfiles) + self._debug_dict('entry profiles', self.profiles) + self._debug_dict('entry actions', self.actions) + self._debug_dict('entry trans_r', self.trans_r) + self._debug_dict('entry trans_w', self.trans_w) + self._debug_dict('entry variables', self.variables) def _debug_dict(self, title, elems): """pretty print dict""" if not self._debug: return - self._log.dbg('{}:'.format(title)) + self._dbg('{}:'.format(title)) if not elems: return for k, v in elems.items(): - self._log.dbg('\t- \"{}\": {}'.format(k, v)) + self._dbg('\t- \"{}\": {}'.format(k, v)) + + def _dbg(self, content): + pre = os.path.basename(self._path) + self._log.dbg('[{}] {}'.format(pre, content)) From ff6fad8d84aab264d8cb3934ca05492f1794b7a2 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Aug 2020 14:59:26 +0200 Subject: [PATCH 13/47] adding a test for init --- tests/dummy.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 tests/dummy.py diff --git a/tests/dummy.py b/tests/dummy.py new file mode 100644 index 0000000..b85de83 --- /dev/null +++ b/tests/dummy.py @@ -0,0 +1,22 @@ +""" +author: deadc0de6 (https://github.com/deadc0de6) +Copyright (c) 2017, deadc0de6 +basic unittest for the import function +""" + + +import unittest +import dotdrop + + +class TestDummy(unittest.TestCase): + + dotdrop.main() + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() From abb93ead6592385a895cdba14a990364502cdbaa Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Aug 2020 15:05:44 +0200 Subject: [PATCH 14/47] enable debug tests --- tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests.sh b/tests.sh index ce49ee0..c8f1a46 100755 --- a/tests.sh +++ b/tests.sh @@ -25,13 +25,13 @@ which ${nosebin} 2>/dev/null [ "$?" != "0" ] && echo "Install nosetests" && exit 1 set -e -# do not print debugs when running tests (faster) -export DOTDROP_FORCE_NODEBUG=yes - # execute tests with coverage PYTHONPATH=dotdrop ${nosebin} -s --with-coverage --cover-package=dotdrop #PYTHONPATH=dotdrop python3 -m pytest tests +# do not print debugs when running tests (faster) +#export DOTDROP_FORCE_NODEBUG=yes + ## execute bash script tests [ "$1" = '--python-only' ] || { log=`mktemp` From ce4b515e16fd739279f367f5d1d9a7ef278c68b2 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Aug 2020 16:59:01 +0200 Subject: [PATCH 15/47] enable coverage for tests-ng --- tests-ng/actions-args-template.sh | 1 + tests-ng/actions-args.sh | 1 + tests-ng/actions-default.sh | 1 + tests-ng/actions-pre.sh | 1 + tests-ng/actions-template.sh | 1 + tests-ng/actions.sh | 1 + tests-ng/compare-ignore-relative.sh | 1 + tests-ng/compare-ignore.sh | 1 + tests-ng/compare.sh | 1 + tests-ng/deprecated-link.sh | 1 + tests-ng/diff-cmd.sh | 9 ++++++--- tests-ng/dir-import-update.sh | 1 + tests-ng/dotdrop-variables.sh | 1 + tests-ng/dotfile-no-src.sh | 1 + tests-ng/dotfile-sub-variables.sh | 1 + tests-ng/dotfile-variables.sh | 1 + tests-ng/dotfiles-all.sh | 1 + tests-ng/dotfiles-dyn-paths.sh | 1 + tests-ng/duplicate-key.sh | 1 + tests-ng/dynactions.sh | 1 + tests-ng/dyndotfilepaths.sh | 1 + tests-ng/dynextvariables.sh | 1 + tests-ng/dyninclude.sh | 1 + tests-ng/dynvariables.sh | 1 + tests-ng/ext-actions.sh | 1 + tests-ng/extvariables.sh | 1 + tests-ng/filter_file.sh | 1 + tests-ng/force-actions.sh | 1 + tests-ng/func_file.sh | 1 + tests-ng/global-compare-ignore.sh | 1 + tests-ng/global-update-ignore.sh | 1 + tests-ng/globs.sh | 1 + tests-ng/header.sh | 1 + tests-ng/ignore-empty.sh | 1 + tests-ng/import-as.sh | 1 + tests-ng/import-configs.sh | 1 + tests-ng/import-duplicate.sh | 1 + tests-ng/import-link-children.sh | 1 + tests-ng/import-non-existing.sh | 1 + tests-ng/import-profile-dotfiles.sh | 1 + tests-ng/import-subfile.sh | 1 + tests-ng/import.sh | 1 + tests-ng/imported-configs-variables.sh | 1 + tests-ng/include-actions.sh | 1 + tests-ng/include-order.sh | 1 + tests-ng/include.sh | 1 + tests-ng/inst-link-default.sh | 1 + tests-ng/install-empty.sh | 1 + tests-ng/install-ignore.sh | 1 + tests-ng/jhelpers.sh | 1 + tests-ng/link-import-default.sh | 1 + tests-ng/link-templates-dir-home.sh | 1 + tests-ng/link-templates-dir.sh | 1 + tests-ng/link-templates.sh | 1 + tests-ng/link-value-tests.sh | 1 + tests-ng/minversion.sh | 1 + tests-ng/profile-actions.sh | 1 + tests-ng/profile-dyninclude.sh | 1 + tests-ng/profile-dynvariables.sh | 1 + tests-ng/re-import.sh | 1 + tests-ng/recinclude.sh | 1 + tests-ng/recvariables.sh | 1 + tests-ng/remove.sh | 1 + tests-ng/symlink.sh | 1 + tests-ng/transformations-template.sh | 1 + tests-ng/transformations-with-args.sh | 1 + tests-ng/transformations.sh | 1 + tests-ng/update-ignore-relative.sh | 1 + tests-ng/update-ignore.sh | 1 + tests-ng/update-rights.sh | 1 + tests-ng/update-templates.sh | 1 + tests-ng/update-with-key.sh | 1 + tests-ng/update.sh | 1 + tests-ng/variables-include.sh | 1 + tests-ng/variables.sh | 1 + tests-ng/workdir.sh | 1 + 76 files changed, 81 insertions(+), 3 deletions(-) diff --git a/tests-ng/actions-args-template.sh b/tests-ng/actions-args-template.sh index 42dc3c7..f8d469d 100755 --- a/tests-ng/actions-args-template.sh +++ b/tests-ng/actions-args-template.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/actions-args.sh b/tests-ng/actions-args.sh index 6da4f43..b15d9d5 100755 --- a/tests-ng/actions-args.sh +++ b/tests-ng/actions-args.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/actions-default.sh b/tests-ng/actions-default.sh index 5eb4572..e144243 100755 --- a/tests-ng/actions-default.sh +++ b/tests-ng/actions-default.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/actions-pre.sh b/tests-ng/actions-pre.sh index 48dc59a..24ae575 100755 --- a/tests-ng/actions-pre.sh +++ b/tests-ng/actions-pre.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/actions-template.sh b/tests-ng/actions-template.sh index 36a8036..577a28f 100755 --- a/tests-ng/actions-template.sh +++ b/tests-ng/actions-template.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/actions.sh b/tests-ng/actions.sh index c4e40f9..20486f2 100755 --- a/tests-ng/actions.sh +++ b/tests-ng/actions.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/compare-ignore-relative.sh b/tests-ng/compare-ignore-relative.sh index 741aacf..a8c1260 100755 --- a/tests-ng/compare-ignore-relative.sh +++ b/tests-ng/compare-ignore-relative.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/compare-ignore.sh b/tests-ng/compare-ignore.sh index 98c986b..53d86c9 100755 --- a/tests-ng/compare-ignore.sh +++ b/tests-ng/compare-ignore.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh index 7b8039e..15f34e4 100755 --- a/tests-ng/compare.sh +++ b/tests-ng/compare.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/deprecated-link.sh b/tests-ng/deprecated-link.sh index 262c2b5..b63238b 100755 --- a/tests-ng/deprecated-link.sh +++ b/tests-ng/deprecated-link.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/diff-cmd.sh b/tests-ng/diff-cmd.sh index 8e711c0..b09378d 100755 --- a/tests-ng/diff-cmd.sh +++ b/tests-ng/diff-cmd.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" @@ -67,6 +68,9 @@ dotfiles: profiles: _EOF +export DOTDROP_FORCE_NODEBUG=yes +export DOTDROP_NOBANNER=yes + # import echo "[+] import" cd ${ddpath} | ${bin} import -c ${cfg} ${tmpd}/singlefile @@ -74,9 +78,6 @@ cd ${ddpath} | ${bin} import -c ${cfg} ${tmpd}/singlefile # modify the file echo "modified" > ${tmpd}/singlefile -# suppressing the banner, so we can compare dotdrop diff with UNIX diff -export DOTDROP_NOBANNER=yes - # default diff (unified) echo "[+] comparing with default diff (unified)" set +e @@ -121,7 +122,9 @@ grep fakediff ${tmpd}/fake &> /dev/null || exit 1 ## CLEANING rm -rf ${basedir} ${tmpd} + unset DOTDROP_NOBANNER +unset DOTDROP_FORCE_NODEBUG echo "OK" exit 0 diff --git a/tests-ng/dir-import-update.sh b/tests-ng/dir-import-update.sh index dfd0a3f..79dcad6 100755 --- a/tests-ng/dir-import-update.sh +++ b/tests-ng/dir-import-update.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/dotdrop-variables.sh b/tests-ng/dotdrop-variables.sh index f2a7fb8..c5d7639 100755 --- a/tests-ng/dotdrop-variables.sh +++ b/tests-ng/dotdrop-variables.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/dotfile-no-src.sh b/tests-ng/dotfile-no-src.sh index 4c6ecb1..ccfaf37 100755 --- a/tests-ng/dotfile-no-src.sh +++ b/tests-ng/dotfile-no-src.sh @@ -33,6 +33,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/dotfile-sub-variables.sh b/tests-ng/dotfile-sub-variables.sh index d0d0ad2..c03a730 100755 --- a/tests-ng/dotfile-sub-variables.sh +++ b/tests-ng/dotfile-sub-variables.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/dotfile-variables.sh b/tests-ng/dotfile-variables.sh index 533a48e..cda2104 100755 --- a/tests-ng/dotfile-variables.sh +++ b/tests-ng/dotfile-variables.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/dotfiles-all.sh b/tests-ng/dotfiles-all.sh index 8335994..fc5fdfd 100755 --- a/tests-ng/dotfiles-all.sh +++ b/tests-ng/dotfiles-all.sh @@ -33,6 +33,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/dotfiles-dyn-paths.sh b/tests-ng/dotfiles-dyn-paths.sh index 542b92a..7450752 100755 --- a/tests-ng/dotfiles-dyn-paths.sh +++ b/tests-ng/dotfiles-dyn-paths.sh @@ -33,6 +33,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/duplicate-key.sh b/tests-ng/duplicate-key.sh index 9177323..3d11cc2 100755 --- a/tests-ng/duplicate-key.sh +++ b/tests-ng/duplicate-key.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/dynactions.sh b/tests-ng/dynactions.sh index 6c9f1fd..81bb3b3 100755 --- a/tests-ng/dynactions.sh +++ b/tests-ng/dynactions.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/dyndotfilepaths.sh b/tests-ng/dyndotfilepaths.sh index f26c638..9b26d8a 100755 --- a/tests-ng/dyndotfilepaths.sh +++ b/tests-ng/dyndotfilepaths.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/dynextvariables.sh b/tests-ng/dynextvariables.sh index 05a843f..83e9799 100755 --- a/tests-ng/dynextvariables.sh +++ b/tests-ng/dynextvariables.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/dyninclude.sh b/tests-ng/dyninclude.sh index 34b500b..6db0155 100755 --- a/tests-ng/dyninclude.sh +++ b/tests-ng/dyninclude.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/dynvariables.sh b/tests-ng/dynvariables.sh index 8ab70e4..acbb71f 100755 --- a/tests-ng/dynvariables.sh +++ b/tests-ng/dynvariables.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/ext-actions.sh b/tests-ng/ext-actions.sh index f57ff17..968b01b 100755 --- a/tests-ng/ext-actions.sh +++ b/tests-ng/ext-actions.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/extvariables.sh b/tests-ng/extvariables.sh index d51814d..aae8e17 100755 --- a/tests-ng/extvariables.sh +++ b/tests-ng/extvariables.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/filter_file.sh b/tests-ng/filter_file.sh index d82edf2..def9e09 100755 --- a/tests-ng/filter_file.sh +++ b/tests-ng/filter_file.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/force-actions.sh b/tests-ng/force-actions.sh index c0342e8..242904c 100755 --- a/tests-ng/force-actions.sh +++ b/tests-ng/force-actions.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/func_file.sh b/tests-ng/func_file.sh index 7c68a1c..081f747 100755 --- a/tests-ng/func_file.sh +++ b/tests-ng/func_file.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/global-compare-ignore.sh b/tests-ng/global-compare-ignore.sh index f3dde98..056be9a 100755 --- a/tests-ng/global-compare-ignore.sh +++ b/tests-ng/global-compare-ignore.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/global-update-ignore.sh b/tests-ng/global-update-ignore.sh index 53285fb..5a387a6 100755 --- a/tests-ng/global-update-ignore.sh +++ b/tests-ng/global-update-ignore.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/globs.sh b/tests-ng/globs.sh index 6acbf18..a682961 100755 --- a/tests-ng/globs.sh +++ b/tests-ng/globs.sh @@ -35,6 +35,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/header.sh b/tests-ng/header.sh index 5c16ac3..311be3c 100755 --- a/tests-ng/header.sh +++ b/tests-ng/header.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/ignore-empty.sh b/tests-ng/ignore-empty.sh index df80274..92a3d2f 100755 --- a/tests-ng/ignore-empty.sh +++ b/tests-ng/ignore-empty.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/import-as.sh b/tests-ng/import-as.sh index 34d7f98..d5b29e9 100755 --- a/tests-ng/import-as.sh +++ b/tests-ng/import-as.sh @@ -31,6 +31,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/import-configs.sh b/tests-ng/import-configs.sh index e000de6..4c75d4c 100755 --- a/tests-ng/import-configs.sh +++ b/tests-ng/import-configs.sh @@ -31,6 +31,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/import-duplicate.sh b/tests-ng/import-duplicate.sh index 50cd1f2..262e3a5 100755 --- a/tests-ng/import-duplicate.sh +++ b/tests-ng/import-duplicate.sh @@ -31,6 +31,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/import-link-children.sh b/tests-ng/import-link-children.sh index 8fae7c2..036e667 100755 --- a/tests-ng/import-link-children.sh +++ b/tests-ng/import-link-children.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/import-non-existing.sh b/tests-ng/import-non-existing.sh index 8b0f897..9b05efa 100755 --- a/tests-ng/import-non-existing.sh +++ b/tests-ng/import-non-existing.sh @@ -31,6 +31,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/import-profile-dotfiles.sh b/tests-ng/import-profile-dotfiles.sh index ea565e2..1215388 100755 --- a/tests-ng/import-profile-dotfiles.sh +++ b/tests-ng/import-profile-dotfiles.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/import-subfile.sh b/tests-ng/import-subfile.sh index 141cf97..b416a63 100755 --- a/tests-ng/import-subfile.sh +++ b/tests-ng/import-subfile.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/import.sh b/tests-ng/import.sh index 78ab6ff..854388e 100755 --- a/tests-ng/import.sh +++ b/tests-ng/import.sh @@ -31,6 +31,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/imported-configs-variables.sh b/tests-ng/imported-configs-variables.sh index c6e0993..f852c00 100755 --- a/tests-ng/imported-configs-variables.sh +++ b/tests-ng/imported-configs-variables.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/include-actions.sh b/tests-ng/include-actions.sh index 3f292c5..abe4f75 100755 --- a/tests-ng/include-actions.sh +++ b/tests-ng/include-actions.sh @@ -33,6 +33,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/include-order.sh b/tests-ng/include-order.sh index d903f59..c6da500 100755 --- a/tests-ng/include-order.sh +++ b/tests-ng/include-order.sh @@ -33,6 +33,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/include.sh b/tests-ng/include.sh index 8f0d638..0c28fbf 100755 --- a/tests-ng/include.sh +++ b/tests-ng/include.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/inst-link-default.sh b/tests-ng/inst-link-default.sh index b175f98..c3ed955 100755 --- a/tests-ng/inst-link-default.sh +++ b/tests-ng/inst-link-default.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/install-empty.sh b/tests-ng/install-empty.sh index 14e05ee..535d540 100755 --- a/tests-ng/install-empty.sh +++ b/tests-ng/install-empty.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/install-ignore.sh b/tests-ng/install-ignore.sh index edae408..46f5708 100755 --- a/tests-ng/install-ignore.sh +++ b/tests-ng/install-ignore.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/jhelpers.sh b/tests-ng/jhelpers.sh index 0e44adb..f502ed6 100755 --- a/tests-ng/jhelpers.sh +++ b/tests-ng/jhelpers.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/link-import-default.sh b/tests-ng/link-import-default.sh index f46908a..b20f145 100755 --- a/tests-ng/link-import-default.sh +++ b/tests-ng/link-import-default.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/link-templates-dir-home.sh b/tests-ng/link-templates-dir-home.sh index 1b7ed0e..0b58c16 100755 --- a/tests-ng/link-templates-dir-home.sh +++ b/tests-ng/link-templates-dir-home.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/link-templates-dir.sh b/tests-ng/link-templates-dir.sh index cf15d1a..8d57133 100755 --- a/tests-ng/link-templates-dir.sh +++ b/tests-ng/link-templates-dir.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/link-templates.sh b/tests-ng/link-templates.sh index bea1e53..74d7acb 100755 --- a/tests-ng/link-templates.sh +++ b/tests-ng/link-templates.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/link-value-tests.sh b/tests-ng/link-value-tests.sh index f1b7b01..4988e6f 100755 --- a/tests-ng/link-value-tests.sh +++ b/tests-ng/link-value-tests.sh @@ -33,6 +33,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/minversion.sh b/tests-ng/minversion.sh index e827287..2442354 100755 --- a/tests-ng/minversion.sh +++ b/tests-ng/minversion.sh @@ -30,6 +30,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/profile-actions.sh b/tests-ng/profile-actions.sh index 2c5875d..5840021 100755 --- a/tests-ng/profile-actions.sh +++ b/tests-ng/profile-actions.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/profile-dyninclude.sh b/tests-ng/profile-dyninclude.sh index 9040b33..44db6f9 100755 --- a/tests-ng/profile-dyninclude.sh +++ b/tests-ng/profile-dyninclude.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/profile-dynvariables.sh b/tests-ng/profile-dynvariables.sh index bdfb247..9455401 100755 --- a/tests-ng/profile-dynvariables.sh +++ b/tests-ng/profile-dynvariables.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/re-import.sh b/tests-ng/re-import.sh index 5974c31..99e1736 100755 --- a/tests-ng/re-import.sh +++ b/tests-ng/re-import.sh @@ -31,6 +31,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/recinclude.sh b/tests-ng/recinclude.sh index cbef945..af63484 100755 --- a/tests-ng/recinclude.sh +++ b/tests-ng/recinclude.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/recvariables.sh b/tests-ng/recvariables.sh index c1a4f3d..3255718 100755 --- a/tests-ng/recvariables.sh +++ b/tests-ng/recvariables.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/remove.sh b/tests-ng/remove.sh index c8f5c8b..fb7c004 100755 --- a/tests-ng/remove.sh +++ b/tests-ng/remove.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/symlink.sh b/tests-ng/symlink.sh index 829a9e7..818cba4 100755 --- a/tests-ng/symlink.sh +++ b/tests-ng/symlink.sh @@ -31,6 +31,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/transformations-template.sh b/tests-ng/transformations-template.sh index dfad1d4..4c3bafc 100755 --- a/tests-ng/transformations-template.sh +++ b/tests-ng/transformations-template.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/transformations-with-args.sh b/tests-ng/transformations-with-args.sh index faeffec..5815620 100755 --- a/tests-ng/transformations-with-args.sh +++ b/tests-ng/transformations-with-args.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/transformations.sh b/tests-ng/transformations.sh index eae4e47..0bd329f 100755 --- a/tests-ng/transformations.sh +++ b/tests-ng/transformations.sh @@ -34,6 +34,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/update-ignore-relative.sh b/tests-ng/update-ignore-relative.sh index ac9ef60..09d4021 100755 --- a/tests-ng/update-ignore-relative.sh +++ b/tests-ng/update-ignore-relative.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/update-ignore.sh b/tests-ng/update-ignore.sh index 273482f..af9e34c 100755 --- a/tests-ng/update-ignore.sh +++ b/tests-ng/update-ignore.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/update-rights.sh b/tests-ng/update-rights.sh index 5ff2fa3..2e804de 100755 --- a/tests-ng/update-rights.sh +++ b/tests-ng/update-rights.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/update-templates.sh b/tests-ng/update-templates.sh index abdad4c..67bfcbd 100755 --- a/tests-ng/update-templates.sh +++ b/tests-ng/update-templates.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/update-with-key.sh b/tests-ng/update-with-key.sh index 47ba116..8c44d0e 100755 --- a/tests-ng/update-with-key.sh +++ b/tests-ng/update-with-key.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/update.sh b/tests-ng/update.sh index 83ca1d5..2535b3d 100755 --- a/tests-ng/update.sh +++ b/tests-ng/update.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/variables-include.sh b/tests-ng/variables-include.sh index 655fbb3..e87b60d 100755 --- a/tests-ng/variables-include.sh +++ b/tests-ng/variables-include.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/variables.sh b/tests-ng/variables.sh index b0ba712..8f9721d 100755 --- a/tests-ng/variables.sh +++ b/tests-ng/variables.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" diff --git a/tests-ng/workdir.sh b/tests-ng/workdir.sh index 1563921..dae5751 100755 --- a/tests-ng/workdir.sh +++ b/tests-ng/workdir.sh @@ -32,6 +32,7 @@ ddpath="${cur}/../" export PYTHONPATH="${ddpath}:${PYTHONPATH}" bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true echo "dotdrop path: ${ddpath}" echo "pythonpath: ${PYTHONPATH}" From 1b0aac2cf4345af7ac689c27fa61ee37067438c7 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Aug 2020 16:59:29 +0200 Subject: [PATCH 16/47] remove dead code --- dotdrop/cfg_yaml.py | 50 --------------------------------------------- 1 file changed, 50 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 1f179c0..0f5025e 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -261,7 +261,6 @@ class CfgYaml: # TODO self._clear_profile_vars(newvars) self._add_variables(newvars) - # self.variables, self.prokeys = self._merge_variables() # apply variables # self._apply_variables() @@ -563,49 +562,6 @@ class CfgYaml: ######################################################## # parsing helpers ######################################################## - def _merge_variables(self): - """ - resolve all variables across the config - apply them to any needed entries - and return the full list of variables - """ - # TODO remove me ??? - if self._debug: - self._dbg('get local variables') - - # get all variables from local and resolve - var = self._get_variables_dict(self._profile) - - # get all dynvariables from local and resolve - dvar = self._get_dvariables_dict() - - # temporarly resolve all variables for "include" - merged = self._merge_dict(dvar, var) - merged = self._rec_resolve_vars(merged) - if self._debug: - self._debug_dict('variables', merged) - # exec dynvariables - self._shell_exec_dvars(dvar.keys(), merged) - - if self._debug: - self._dbg('local variables resolved') - self._debug_dict('variables', merged) - - # resolve profile included variables/dynvariables - _, pro_var, pro_dvar = self._get_profile_included_vars() - - # merge all and resolve - merged = self._merge_dict(pro_var, merged) - merged = self._merge_dict(pro_dvar, merged) - merged = self._rec_resolve_vars(merged) - - if self._debug: - self._dbg('resolve all uses of variables in config') - self._debug_dict('variables', merged) - - prokeys = list(pro_var.keys()) + list(pro_dvar.keys()) - return merged, prokeys - def _template_include_entry(self): """template all "include" entries""" # import_actions @@ -709,12 +665,6 @@ class CfgYaml: variables['_dotdrop_workdir'] = p return variables - def _get_dvariables_dict(self): - """return dynvariables""" - # TODO remove me ?? - variables = deepcopy(self.ori_dvariables) - return variables - def _get_profile_included_item(self, keyitem): """recursively get included in profile""" profiles = [self._profile] + self._inc_profiles From 7e084f081d48b81875fe3b087b9b073e0c3af5e7 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Aug 2020 16:59:43 +0200 Subject: [PATCH 17/47] fix coverage --- tests.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests.sh b/tests.sh index c8f1a46..eedbfa5 100755 --- a/tests.sh +++ b/tests.sh @@ -25,10 +25,20 @@ which ${nosebin} 2>/dev/null [ "$?" != "0" ] && echo "Install nosetests" && exit 1 set -e +# do not print debugs when running tests (faster) +export DOTDROP_FORCE_NODEBUG=yes + +# coverage file location +cur=`dirname $(readlink -f "${0}")` +export COVERAGE_FILE="${cur}/.coverage" + # execute tests with coverage PYTHONPATH=dotdrop ${nosebin} -s --with-coverage --cover-package=dotdrop #PYTHONPATH=dotdrop python3 -m pytest tests +# enable debug logs +export DOTDROP_DEBUG= +unset DOTDROP_FORCE_NODEBUG # do not print debugs when running tests (faster) #export DOTDROP_FORCE_NODEBUG=yes From 9fcb22a6f3a4f6e0aa76910955dea29cf62205ef Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Aug 2020 16:59:54 +0200 Subject: [PATCH 18/47] remove unused mock --- tests/test_install.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_install.py b/tests/test_install.py index 0f0ce66..2da99be 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -362,8 +362,8 @@ exec bspwm src = '/some/non/existant/file' installer = Installer() - logger = MagicMock() - installer.log.err = logger + # logger = MagicMock() + # installer.log.err = logger res, err = installer.link_children(templater=MagicMock(), src=src, dst='/dev/null', actionexec=None) @@ -381,10 +381,10 @@ exec bspwm src = create_random_file(src_dir)[0] - logger = MagicMock() + # logger = MagicMock() templater = MagicMock() installer = Installer() - installer.log.err = logger + # installer.log.err = logger # pass src file not src dir res, err = installer.link_children(templater=templater, src=src, From 9bb2722345296a224a0ae997c902ff40f3aae709 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Aug 2020 17:08:29 +0200 Subject: [PATCH 19/47] disable debug for travis --- tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests.sh b/tests.sh index eedbfa5..c598ca7 100755 --- a/tests.sh +++ b/tests.sh @@ -37,10 +37,10 @@ PYTHONPATH=dotdrop ${nosebin} -s --with-coverage --cover-package=dotdrop #PYTHONPATH=dotdrop python3 -m pytest tests # enable debug logs -export DOTDROP_DEBUG= -unset DOTDROP_FORCE_NODEBUG +#export DOTDROP_DEBUG= +#unset DOTDROP_FORCE_NODEBUG # do not print debugs when running tests (faster) -#export DOTDROP_FORCE_NODEBUG=yes +export DOTDROP_FORCE_NODEBUG=yes ## execute bash script tests [ "$1" = '--python-only' ] || { From dad31efb1fe2ce3957c444fcc2118395e1dbea43 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Aug 2020 17:25:36 +0200 Subject: [PATCH 20/47] hide debug logs in travis --- tests.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests.sh b/tests.sh index c598ca7..3823a0f 100755 --- a/tests.sh +++ b/tests.sh @@ -37,16 +37,20 @@ PYTHONPATH=dotdrop ${nosebin} -s --with-coverage --cover-package=dotdrop #PYTHONPATH=dotdrop python3 -m pytest tests # enable debug logs -#export DOTDROP_DEBUG= -#unset DOTDROP_FORCE_NODEBUG +export DOTDROP_DEBUG= +unset DOTDROP_FORCE_NODEBUG # do not print debugs when running tests (faster) -export DOTDROP_FORCE_NODEBUG=yes +#export DOTDROP_FORCE_NODEBUG=yes ## execute bash script tests [ "$1" = '--python-only' ] || { log=`mktemp` for scr in tests-ng/*.sh; do - ${scr} > "${log}" 2>&1 & + if [ -z ${TRAVIS} ]; then + ${scr} > "${log}" 2>&1 & + else + ${scr} > "${log}" 2>&1 >/dev/null & + fi tail --pid="$!" -f "${log}" set +e wait "$!" From 627b75e28ae7a54ae214bd85ed7f983f2b107bbb Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Aug 2020 17:30:36 +0200 Subject: [PATCH 21/47] fix travis --- tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests.sh b/tests.sh index 3823a0f..81e7457 100755 --- a/tests.sh +++ b/tests.sh @@ -49,7 +49,7 @@ unset DOTDROP_FORCE_NODEBUG if [ -z ${TRAVIS} ]; then ${scr} > "${log}" 2>&1 & else - ${scr} > "${log}" 2>&1 >/dev/null & + ${scr} > "${log}" >/dev/null 2>&1 & fi tail --pid="$!" -f "${log}" set +e From 1bd52f0d3cd5c4c28c95bac8802abf7705e60a7f Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Aug 2020 17:55:45 +0200 Subject: [PATCH 22/47] clear TODOs --- dotdrop/cfg_yaml.py | 70 +++++++++++++++++---------------------------- 1 file changed, 27 insertions(+), 43 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 0f5025e..b0c9cd0 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -128,9 +128,6 @@ class CfgYaml: # - properly handle the included profile # - document precedence in wiki # - document parsing in CONTRIBUTING.md - # - document dvars are executed in their own config file - # - remove unused functions/methods - # - coverage # def __init__(self, path, profile=None, addprofiles=[], debug=False): @@ -202,7 +199,6 @@ class CfgYaml: self._add_variables(pvd, shell=True, prio=True) self._profilevarskeys.extend(pv.keys()) self._profilevarskeys.extend(pvd.keys()) - # TODO handle prio when importing included profile from somewhere else # template variables self.variables = self._template_dict(self.variables) @@ -249,7 +245,6 @@ class CfgYaml: # process profile include self._resolve_profile_includes() - # TODO TODO # add the current profile variables _, pv, pvd = self._get_profile_included_vars() self._add_variables(pv, prio=True) @@ -258,52 +253,18 @@ class CfgYaml: self._profilevarskeys.extend(pvd.keys()) # resolve variables - # TODO self._clear_profile_vars(newvars) self._add_variables(newvars) - # apply variables - # self._apply_variables() - # process profile ALL self._resolve_profile_all() # patch dotfiles paths self._template_dotfiles_paths() - # TODO ensure no element is left un-templated at the end - if self._debug: self._dbg('########### {} ###########'.format('final config')) self._debug_entries() - def _add_variables(self, new, shell=False, template=True, prio=False): - """ - add new variables - @shell: execute the variable through the shell - @template: template the variable - @prio: new takes priority over existing variables - """ - # TODO move me - if not new: - return - # merge - if prio: - self.variables = self._merge_dict(new, self.variables) - else: - self.variables = self._merge_dict(self.variables, new) - # ensure enriched variables are relative to this config - self.variables = self._enrich_vars(self.variables, self._profile) - # re-create the templater - self._redefine_templater() - if template: - # rec resolve variables with new ones - self._rec_resolve_variables(self.variables) - if shell: - # shell exec - self._shell_exec_dvars(self.variables, keys=new.keys()) - # re-create the templater - self._redefine_templater() - ######################################################## # outside available methods ######################################################## @@ -650,6 +611,33 @@ class CfgYaml: v[self.key_dotfile_noempty] = val return new + def _add_variables(self, new, shell=False, template=True, prio=False): + """ + add new variables + @shell: execute the variable through the shell + @template: template the variable + @prio: new takes priority over existing variables + """ + if not new: + return + # merge + if prio: + self.variables = self._merge_dict(new, self.variables) + else: + self.variables = self._merge_dict(self.variables, new) + # ensure enriched variables are relative to this config + self.variables = self._enrich_vars(self.variables, self._profile) + # re-create the templater + self._redefine_templater() + if template: + # rec resolve variables with new ones + self._rec_resolve_variables(self.variables) + if shell: + # shell exec + self._shell_exec_dvars(self.variables, keys=new.keys()) + # re-create the templater + self._redefine_templater() + def _enrich_vars(self, variables, profile): """return enriched variables""" # add profile variable @@ -1058,7 +1046,6 @@ class CfgYaml: template an item using the templategen will raise an exception if template failed and exc_if_fail """ - # TODO use this across the entire file if not Templategen.var_is_template(item): return item try: @@ -1152,8 +1139,6 @@ class CfgYaml: remove profile variables from dic if found inplace to avoid profile variables being overwriten """ - # TODO - # [dic.pop(k, None) for k in self.prokeys] if not dic: return [dic.pop(k, None) for k in self._profilevarskeys] @@ -1303,7 +1288,6 @@ class CfgYaml: def _shell_exec_dvars(self, dic, keys=[]): """shell execute dynvariables in-place""" - # TODO remove other calls outside initial setup of dvars if not keys: keys = dic.keys() for k in keys: From 8c7904b6e692b63caad63681af4804ad78023d44 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 7 Aug 2020 19:09:33 +0200 Subject: [PATCH 23/47] adding corner cases test --- tests-ng/corner-case.sh | 102 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100755 tests-ng/corner-case.sh diff --git a/tests-ng/corner-case.sh b/tests-ng/corner-case.sh new file mode 100755 index 0000000..1bc4e2a --- /dev/null +++ b/tests-ng/corner-case.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2019, deadc0de6 +# +# the only purpose is to test corner-cases +# not covered by other tests like +# dry +# diff before write +# etc +# +# returns 1 in case of error +# + +# exit on first error +#set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# dotdrop directory +basedir=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +echo "[+] dotdrop dir: ${basedir}" +echo "[+] dotpath dir: ${basedir}/dotfiles" + +# create the config file +cfg="${basedir}/config.yaml" +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: + f_x: + src: /tmp/x + dst: + f_y: + src: /tmp/.i-do-not-exist-dotdrop + dst: /tmp/y +profiles: + p1: + dotfiles: + - f_x + - f_y + +_EOF + +echo "[+] test install dry" +cd ${ddpath} | ${bin} install -c ${cfg} --dry -p p1 --verbose f_x +[ "$?" != "0" ] && exit 1 + +echo "[+] test install show-diff" +cd ${ddpath} | ${bin} install -c ${cfg} -p p1 --verbose f_x +[ "$?" != "0" ] && exit 1 +cd ${ddpath} | ${bin} install -D -c ${cfg} -p p1 --verbose f_x +[ "$?" != "0" ] && exit 1 + +echo "[+] test install not existing src" +cd ${ddpath} | ${bin} install -c ${cfg} --dry -p p1 --verbose f_y + +echo "[+] test install to temp" +cd ${ddpath} | ${bin} install -t -c ${cfg} -p p1 --verbose f_x +[ "$?" != "0" ] && exit 1 + +## CLEANING +rm -rf ${basedir} + +echo "OK" +exit 0 From 59ffae089c7ce9e6b862870ba2228cf8454b8792 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 11 Aug 2020 16:08:28 +0200 Subject: [PATCH 24/47] update contrib --- CONTRIBUTING.md | 67 ++++++++++++++++++++++++++++----------------- dotdrop/cfg_yaml.py | 41 --------------------------- 2 files changed, 42 insertions(+), 66 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 28d3d0f..5b10c66 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,7 +41,25 @@ The configuration file (yaml) is parsed in two layers: Only the higher layer is accessible to other classes of dotdrop. -The lower layer part is only taking care of basic types and +**Rules** + +* `dynvariables` are executed in their own config file +* profile cannot include profiles defined above in the import tree +* config files do not have access to variables + defined above in the import tree + +**Precedence** + +* `dynvariables` > `variables` +* profile `(dyn)variables` > any other `(dyn)variables` +* profile `(dyn)variables` > profile's included `(dyn)variables` +* imported `variables`/`dynvariables` > `(dyn)variables` +* actions/transformations using variables are resolved at runtime + (when action/transformation is executed) and not when loading the config + +## lower layer (cfg_yaml.py) + +The lower layer part is only taking care of basic types does the following: * normalize all config entries * resolve paths (dotfiles src, dotpath, etc) @@ -55,9 +73,11 @@ does the following: * fix any deprecated entries (link_by_default, etc) * clear empty entries -In the end it makes sure the dictionary (or parts of it) accessed +In the end it makes sure the dictionary accessed by the higher layer is clean and normalized. +## higher layer (cfg_aggregator.py) + The higher layer will transform the dictionary parsed by the lower layer into objects (profiles, dotfiles, actions, etc). The higher layer has no notion of inclusion (profile included for example) or @@ -74,36 +94,33 @@ example) won't be *seen* by the higher layer until the config is reloaded. Consi `dirty` flag as a sign the file needs to be written and its representation in higher levels in not accurate anymore. -## Variables resolution +## Variables resolution in the config file -How variables are resolved (pass through jinja2's -templating function) in the config file. +How variables are resolved (through jinja2's +templating) in the config file. -* resolve `include` (the below merge is temporary just to resolve the `includes`) - * `variables` and `dynvariables` are first merged and recursively resolved +* resolve main config file variables + * `variables` and `dynvariables` are recursively templated * `dynvariables` are executed - * they are all merged and `include` paths are resolved + * both `variables` and `dynvariables` are merged + * profile's `variables` and `dynvariables` are merged +* resolve *included* entries (see below) + * paths and entries are templated (allows to use something like `include {{@@ os @@}}.variables.yaml`) -* `variables` and profile's `variables` are merged -* `dynvariables` and profile's `dynvariables` are merged -* `dynvariables` are executed -* they are all merged into the final *local* `variables` +* *included* entries are processed + * dyn-/variables are all resolved in their own file -These are then used to resolve different elements in the config file: +*included* + +* entry *import_actions* +* entry *import_configs* +* entry *import_variables* +* profile's *import* +* profile's *include + +Variables are then used to resolve different elements in the config file: see [this](https://github.com/deadc0de6/dotdrop/wiki/config-variables#config-available-variables) -Then additional variables (`import_variables` and `import_configs`) are -then merged and take precedence over local variables. - -Note: - -* `dynvariables` > `variables` -* profile `(dyn)variables` > any other `(dyn)variables` -* profile `(dyn)variables` > profile's included `(dyn)variables` -* imported `variables`/`dynvariables` > `(dyn)variables` -* actions/transformations using variables are resolved at runtime - (when action/transformation is executed) and not when loading the config - # Testing Dotdrop is tested with the use of the [tests.sh](/tests.sh) script. diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index b0c9cd0..fc49647 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -89,47 +89,6 @@ class CfgYaml: lnk_link = LinkTypes.LINK.name.lower() lnk_children = LinkTypes.LINK_CHILDREN.name.lower() - # TODO - # - # "include" entries: - # - import_actions - # - import_configs - # - import_variables - # - profile's import - # - profile's include - # - # variable precedence: - # 1) profile variable - # 2) "import_variables" variables - # 3) "import_configs" variables - # 4) other variables - # - # rules: - # - dynvariables are executed in their own config file - # - profile cannot include profiles defined above in the import tree - # - config files do not have access to variables - # defined above in the import tree - # - # parse a config file - # - parse settings - # - parse variables - # - interprete dynvariables - # - template the include entries - # - parse profiles - # - parse dotfiles - # - parse other elements (actions, trans_r, trans_w) - # - parse and integrate included elements (see below) - # - # parse "include" entry - # - same as parse config file - # - add new entry to the top dict - # - # TODO - # - properly handle the included profile - # - document precedence in wiki - # - document parsing in CONTRIBUTING.md - # - def __init__(self, path, profile=None, addprofiles=[], debug=False): """ config parser From ba21e499fa6f02e443d6bf20ff10c32d811c35bf Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 11 Aug 2020 18:59:44 +0200 Subject: [PATCH 25/47] fix ret --- dotdrop/installer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 3313eeb..773c1c2 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -240,7 +240,6 @@ class Installer: actionexec = None else: if err: - return ret, err return self._log_install(ret, err) return self._log_install(installed > 0, None) From 72cc87aa4711ed913d1065482d59f5d66d8e44da Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 11 Aug 2020 18:59:58 +0200 Subject: [PATCH 26/47] more tests --- tests-ng/install-empty.sh | 15 +++++-- tests-ng/install-ignore.sh | 2 +- tests-ng/install-to-temp.sh | 89 +++++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 4 deletions(-) create mode 100755 tests-ng/install-to-temp.sh diff --git a/tests-ng/install-empty.sh b/tests-ng/install-empty.sh index 535d540..6773ca7 100755 --- a/tests-ng/install-empty.sh +++ b/tests-ng/install-empty.sh @@ -7,7 +7,7 @@ # # exit on first error -#set -e +set -e # all this crap to get current path rl="readlink -f" @@ -68,17 +68,26 @@ dotfiles: f_z: src: dst: + f_l: + src: + dst: + link: link + f_lc: + src: + dst: + link: link_children profiles: p1: dotfiles: - f_x - f_y - f_z - + - f_l + - f_lc _EOF echo "[+] install" -cd ${ddpath} | ${bin} install -c ${cfg} -p p1 --verbose | grep '^3 dotfile(s) installed.$' +cd ${ddpath} | ${bin} install -c ${cfg} -p p1 --verbose | grep '^5 dotfile(s) installed.$' [ "$?" != "0" ] && exit 1 ## CLEANING diff --git a/tests-ng/install-ignore.sh b/tests-ng/install-ignore.sh index 46f5708..adc4298 100755 --- a/tests-ng/install-ignore.sh +++ b/tests-ng/install-ignore.sh @@ -82,7 +82,7 @@ echo "new data" > ${basedir}/dotfiles/${tmpd}/readmes/README.md # install rm -rf ${tmpd} echo "[+] install normal" -cd ${ddpath} | ${bin} install -c ${cfg} --verbose +cd ${ddpath} | ${bin} install --showdiff -c ${cfg} --verbose [ "$?" != "0" ] && exit 1 nb=`find ${tmpd} -iname 'README.md' | wc -l` echo "(1) found ${nb} README.md file(s)" diff --git a/tests-ng/install-to-temp.sh b/tests-ng/install-to-temp.sh new file mode 100755 index 0000000..902920b --- /dev/null +++ b/tests-ng/install-to-temp.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2019, deadc0de6 +# +# test install to temp +# returns 1 in case of error +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# dotdrop directory +basedir=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${basedir}/dotfiles +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +echo "[+] dotdrop dir: ${basedir}" +echo "[+] dotpath dir: ${basedir}/dotfiles" + +# create the config file +cfg="${basedir}/config.yaml" +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: + f_x: + src: x + dst: ${tmpd}/x + f_y: + src: y + dst: ${tmpd}/y + link: link +profiles: + p1: + dotfiles: + - f_x + - f_y +_EOF + +echo 'test_x' > ${basedir}/dotfiles/x +echo 'test_y' > ${basedir}/dotfiles/y + +echo "[+] install" +cd ${ddpath} | ${bin} install -c ${cfg} -p p1 --showdiff --verbose --temp | grep '^2 dotfile(s) installed.$' +[ "$?" != "0" ] && exit 1 + +## CLEANING +rm -rf ${basedir} + +echo "OK" +exit 0 From e4cbe92428673285fa56a01556b940f44431054f Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 11 Aug 2020 19:20:45 +0200 Subject: [PATCH 27/47] remove dead code --- dotdrop/cfg_yaml.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index fc49647..df31162 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -1274,14 +1274,6 @@ class CfgYaml: err += ' Please update.' raise YamlException(err) - def _profile_yes(self, profile): - """returns True if the profile is to be considered""" - if profile == self._profile: - return True - if profile in self._inc_profiles: - return True - return False - def _debug_entries(self): """debug print all interesting entries""" if not self._debug: From 49953be2bd04f4ef8cbfbd893e252daff5a8fa90 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 24 Aug 2020 10:30:17 +0200 Subject: [PATCH 28/47] adding test for #250 --- tests-ng/profile-undefined-variables.sh | 110 ++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100755 tests-ng/profile-undefined-variables.sh diff --git a/tests-ng/profile-undefined-variables.sh b/tests-ng/profile-undefined-variables.sh new file mode 100755 index 0000000..cad5fb4 --- /dev/null +++ b/tests-ng/profile-undefined-variables.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2017, deadc0de6 +# +# test variables defined in a different profile +# than the one selected +# returns 1 in case of error +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +#echo "dotfile destination: ${tmpd}" + +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: + f_abc: + dst: "${tmpd}/{{@@ defined_in_main @@}}" + src: abc + f_def: + dst: "${tmpd}/{{@@ defined_in_alt @@}}" + src: def +profiles: + pmain: + dynvariables: + defined_in_main: echo abc + dotfiles: + - f_abc + palt: + dynvariables: + defined_in_alt: echo def + dotfiles: + - f_def + pall: + dynvariables: + defined_in_main: echo abcall + defined_in_alt: echo defall + dotfiles: + - ALL +_EOF +#cat ${cfg} + +# create the dotfile +echo "main" > ${tmps}/dotfiles/abc +echo "alt" > ${tmps}/dotfiles/def + +# install +cd ${ddpath} | ${bin} install -f -c ${cfg} -p pmain -V + +grep main ${tmpd}/abc + +# install pall +cd ${ddpath} | ${bin} install -f -c ${cfg} -p pall -V +grep main ${tmpd}/abcall +grep alt ${tmpd}/defall + +## CLEANING +rm -rf ${tmps} ${tmpd} ${scr} ${scr2} + +echo "OK" +exit 0 From 2bf4740f101c2e70d9454eb22364ba3f4b263f22 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 24 Aug 2020 13:54:47 +0200 Subject: [PATCH 29/47] adding selective templating dotfiles for #250 --- dotdrop/cfg_yaml.py | 46 +++++++++++++++++++++---- tests-ng/profile-undefined-variables.sh | 18 ++++++++-- 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index df31162..fe4ab40 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -381,7 +381,7 @@ class CfgYaml: def _parse_blk_settings(self, dic): """parse the "config" block""" - block = self._get_entry(dic, self.key_settings) + block = self._get_entry(dic, self.key_settings).copy() # set defaults settings = Settings(None).serialize().get(self.key_settings) settings.update(block) @@ -412,12 +412,13 @@ class CfgYaml: def _parse_blk_dotfiles(self, dic): """parse the "dotfiles" block""" - dotfiles = self._get_entry(dic, self.key_dotfiles) + dotfiles = self._get_entry(dic, self.key_dotfiles).copy() keys = dotfiles.keys() if len(keys) != len(list(set(keys))): dups = [x for x in keys if x not in list(set(keys))] err = 'duplicate dotfile keys found: {}'.format(dups) raise YamlException(err) + dotfiles = self._norm_dotfiles(dotfiles) if self._debug: self._debug_dict('dotfiles block', dotfiles) @@ -425,7 +426,7 @@ class CfgYaml: def _parse_blk_profiles(self, dic): """parse the "profiles" block""" - profiles = self._get_entry(dic, self.key_profiles) + profiles = self._get_entry(dic, self.key_profiles).copy() profiles = self._norm_profiles(profiles) if self._debug: self._debug_dict('profiles block', profiles) @@ -435,6 +436,8 @@ class CfgYaml: """parse the "actions" block""" actions = self._get_entry(dic, self.key_actions, mandatory=False) + if actions: + actions = actions.copy() actions = self._norm_actions(actions) if self._debug: self._debug_dict('actions block', actions) @@ -449,6 +452,8 @@ class CfgYaml: dic[self.key_trans_r] = dic[self.old_key_trans_r] del dic[self.old_key_trans_r] trans_r = self._get_entry(dic, key, mandatory=False) + if trans_r: + trans_r = trans_r.copy() if self._debug: self._debug_dict('trans_r block', trans_r) return trans_r @@ -457,6 +462,8 @@ class CfgYaml: """parse the "trans_w" block""" trans_w = self._get_entry(dic, self.key_trans_w, mandatory=False) + if trans_w: + trans_w = trans_w.copy() if self._debug: self._debug_dict('trans_w block', trans_w) return trans_w @@ -466,6 +473,8 @@ class CfgYaml: variables = self._get_entry(dic, self.key_variables, mandatory=False) + if variables: + variables = variables.copy() if self._debug: self._debug_dict('variables block', variables) return variables @@ -475,6 +484,8 @@ class CfgYaml: dvariables = self._get_entry(dic, self.key_dvariables, mandatory=False) + if dvariables: + dvariables = dvariables.copy() if self._debug: self._debug_dict('dynvariables block', dvariables) return dvariables @@ -655,7 +666,7 @@ class CfgYaml: continue if self.key_all in dfs: if self._debug: - self._dbg('add ALL to profile {}'.format(k)) + self._dbg('add ALL to profile \"{}\"'.format(k)) v[self.key_profile_dotfiles] = self.dotfiles.keys() def _resolve_profile_includes(self): @@ -1042,7 +1053,30 @@ class CfgYaml: def _template_dotfiles_paths(self): """template dotfiles paths""" - for dotfile in self.dotfiles.values(): + if self._debug: + self._dbg('templating dotfiles paths') + dotfiles = self.dotfiles.copy() + + # only keep dotfiles related to the selected profile + pdfs = [] + pro = self.profiles.get(self._profile) + if pro: + pdfs = pro.get(self.key_profile_dotfiles, []) + for addpro in self._inc_profiles: + pro = self.profiles.get(addpro) + if not pro: + continue + pdfsalt = pro.get(self.key_profile_dotfiles, []) + pdfs.extend(pdfsalt) + if self.key_all not in pdfs: + # take a subset of the dotfiles + newdotfiles = {} + for k, v in dotfiles.items(): + if k in pdfs: + newdotfiles[k] = v + dotfiles = newdotfiles + + for dotfile in dotfiles.values(): # src src = dotfile[self.key_dotfile_src] newsrc = self.resolve_dotfile_src(src, templater=self._tmpl) @@ -1257,7 +1291,7 @@ class CfgYaml: self._log.err(err) raise YamlException(err) if self._debug: - self._dbg('\"{}\": {} -> {}'.format(k, v, out)) + self._dbg('{}: `{}` -> {}'.format(k, v, out)) dic[k] = out def _check_minversion(self, minversion): diff --git a/tests-ng/profile-undefined-variables.sh b/tests-ng/profile-undefined-variables.sh index cad5fb4..157be19 100755 --- a/tests-ng/profile-undefined-variables.sh +++ b/tests-ng/profile-undefined-variables.sh @@ -86,6 +86,9 @@ profiles: defined_in_alt: echo defall dotfiles: - ALL + pinclude: + include: + - pmain _EOF #cat ${cfg} @@ -93,16 +96,27 @@ _EOF echo "main" > ${tmps}/dotfiles/abc echo "alt" > ${tmps}/dotfiles/def -# install +# install pmain +echo "install pmain" cd ${ddpath} | ${bin} install -f -c ${cfg} -p pmain -V - +[ ! -e ${tmpd}/abc ] && echo "dotfile not installed" && exit 1 grep main ${tmpd}/abc # install pall +echo "install pall" cd ${ddpath} | ${bin} install -f -c ${cfg} -p pall -V +[ ! -e ${tmpd}/abcall ] && echo "dotfile not installed" && exit 1 grep main ${tmpd}/abcall +[ ! -e ${tmpd}/defall ] && echo "dotfile not installed" && exit 1 grep alt ${tmpd}/defall +# install pinclude +echo "install pinclude" +rm -f ${tmpd}/abc +cd ${ddpath} | ${bin} install -f -c ${cfg} -p pinclude -V +[ ! -e ${tmpd}/abc ] && echo "dotfile not installed" && exit 1 +grep main ${tmpd}/abc + ## CLEANING rm -rf ${tmps} ${tmpd} ${scr} ${scr2} From 75050cd4b61d4417afb123ef86c08f6e168dd8e2 Mon Sep 17 00:00:00 2001 From: Davide Laezza Date: Tue, 25 Aug 2020 00:01:15 +0200 Subject: [PATCH 30/47] Allowing templating on config variables and redefining templater when changing variables --- dotdrop/cfg_yaml.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index fe4ab40..424b552 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -142,7 +142,7 @@ class CfgYaml: # parse the "variables" block var = self._parse_blk_variables(self._yaml_dict) - self._add_variables(var, template=False) + self._add_variables(var, template=True) # parse the "dynvariables" block dvariables = self._parse_blk_dynvariables(self._yaml_dict) @@ -1100,6 +1100,8 @@ class CfgYaml: val = t.generate_string(val) variables[k] = val t.update_variables(variables) + if variables is self.variables: + self._redefine_templater() def _get_profile_included_vars(self): """resolve profile included variables/dynvariables""" From eea25ecce280e3722098d5d9b3ecee1f874bc1ac Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 25 Aug 2020 07:25:03 +0200 Subject: [PATCH 31/47] adding test for #252 --- tests-ng/variables.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests-ng/variables.sh b/tests-ng/variables.sh index 8f9721d..aa0680c 100755 --- a/tests-ng/variables.sh +++ b/tests-ng/variables.sh @@ -56,6 +56,7 @@ tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` # create the config file cfg="${tmps}/config.yaml" +export dotdrop_test_dst="${tmpd}/def" cat > ${cfg} << _EOF config: @@ -66,14 +67,19 @@ variables: var1: "this is some test" var2: 12 var3: another test + vardst: "{{@@ env['dotdrop_test_dst'] @@}}" dotfiles: f_abc: dst: ${tmpd}/abc src: abc + f_def: + dst: "{{@@ vardst @@}}" + src: def profiles: p1: dotfiles: - f_abc + - f_def _EOF #cat ${cfg} @@ -83,13 +89,19 @@ echo "{{@@ var2 @@}}" >> ${tmps}/dotfiles/abc echo "{{@@ var3 @@}}" >> ${tmps}/dotfiles/abc echo "test" >> ${tmps}/dotfiles/abc -# install -cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 +echo "test_def" > ${tmps}/dotfiles/def +# install +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 --verbose + +[ ! -e ${tmpd}/abc ] && echo "abc not installed" && exit 1 grep '^this is some test' ${tmpd}/abc >/dev/null grep '^12' ${tmpd}/abc >/dev/null grep '^another test' ${tmpd}/abc >/dev/null +[ ! -e ${tmpd}/def ] && echo "def not installed" && exit 1 +grep '^test_def' ${tmpd}/def >/dev/null + #cat ${tmpd}/abc ## CLEANING From 09aac991a4e986741a81dc686935c07045fada50 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 25 Aug 2020 08:14:08 +0200 Subject: [PATCH 32/47] fix cyclic ref between var and dvar for #252 --- CONTRIBUTING.md | 25 +++++++++++++++++++++++-- dotdrop/cfg_yaml.py | 22 ++++++++++++++++++++-- tests-ng/dynvariables.sh | 6 +++++- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5b10c66..e713955 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,10 +44,31 @@ Only the higher layer is accessible to other classes of dotdrop. **Rules** * `dynvariables` are executed in their own config file +* since `variables` and `dynvariables` are templated before the `dynvariables` + are executed, this means that `dynvariables` can safely reference `variables` however + `variables` referencing `dynvariables` will result with the none executed value of the + referenced `dynvariables` (see examples below) +* a `dynvariables` can reference a `variables` however the opposite is not possible * profile cannot include profiles defined above in the import tree * config files do not have access to variables defined above in the import tree +This will result with `var0 = "echo test"` and not `var0 = test` +```yaml +variables: + var0: "{{@@ dvar4 @@}}" +dynvariables: + dvar0: "echo test" +``` + +This will result with `dvar0 = "test"` +```yaml +variables: + var0: "test" +dynvariables: + dvar0: "echo {{@@ var0 @@}}" +``` + **Precedence** * `dynvariables` > `variables` @@ -100,9 +121,9 @@ How variables are resolved (through jinja2's templating) in the config file. * resolve main config file variables - * `variables` and `dynvariables` are recursively templated + * merge `variables` and `dynvariables` (allowing cycling reference) + * recursively template merged `variables` and `dynvariables` * `dynvariables` are executed - * both `variables` and `dynvariables` are merged * profile's `variables` and `dynvariables` are merged * resolve *included* entries (see below) * paths and entries are templated diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 424b552..719ddc2 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -140,13 +140,31 @@ class CfgYaml: self.variables = self._enrich_vars(self.variables, self._profile) self._redefine_templater() + # variables and dynvariables need to be first merged + # before being templated in order to allow cyclic + # references between them + # parse the "variables" block var = self._parse_blk_variables(self._yaml_dict) - self._add_variables(var, template=True) + self._add_variables(var, template=False) # parse the "dynvariables" block dvariables = self._parse_blk_dynvariables(self._yaml_dict) - self._add_variables(dvariables, shell=True) + self._add_variables(dvariables, template=False) + + # now template variables and dynvariables from the same pool + self._rec_resolve_variables(self.variables) + # and execute dvariables + # since this is done after recursively resolving variables + # and dynvariables this means that variables referencing + # dynvariables will result with the not executed value + if dvariables.keys(): + self._shell_exec_dvars(self.variables, keys=dvariables.keys()) + # finally redefine the template + self._redefine_templater() + + if self._debug: + self._debug_dict('current variables defined', self.variables) # parse the "profiles" block self.profiles = self._parse_blk_profiles(self._yaml_dict) diff --git a/tests-ng/dynvariables.sh b/tests-ng/dynvariables.sh index acbb71f..89527ba 100755 --- a/tests-ng/dynvariables.sh +++ b/tests-ng/dynvariables.sh @@ -69,10 +69,12 @@ config: dotpath: dotfiles variables: var1: "this is some test" + var2: "the_dvar4" dynvariables: dvar1: head -1 /proc/meminfo dvar2: "echo 'this is some test' | rev | tr ' ' ','" dvar3: ${scr} + dvar4: "echo {{@@ var2 @@}} | rev" dotfiles: f_abc: dst: ${tmpd}/abc @@ -89,17 +91,19 @@ echo "{{@@ var1 @@}}" > ${tmps}/dotfiles/abc echo "{{@@ dvar1 @@}}" >> ${tmps}/dotfiles/abc echo "{{@@ dvar2 @@}}" >> ${tmps}/dotfiles/abc echo "{{@@ dvar3 @@}}" >> ${tmps}/dotfiles/abc +echo "{{@@ dvar4 @@}}" >> ${tmps}/dotfiles/abc echo "test" >> ${tmps}/dotfiles/abc # install cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V -#cat ${tmpd}/abc +cat ${tmpd}/abc grep '^this is some test' ${tmpd}/abc >/dev/null grep "^MemTotal" ${tmpd}/abc >/dev/null grep '^tset,emos,si,siht' ${tmpd}/abc >/dev/null grep "^${TESTENV}" ${tmpd}/abc > /dev/null +grep '^4ravd_eht' ${tmpd}/abc >/dev/null #cat ${tmpd}/abc From e1280dca9beffc97ea5afe87ac14df68bcdeb9c4 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 25 Aug 2020 08:15:18 +0200 Subject: [PATCH 33/47] fix doc --- CONTRIBUTING.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e713955..74aa6a3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,6 @@ Only the higher layer is accessible to other classes of dotdrop. are executed, this means that `dynvariables` can safely reference `variables` however `variables` referencing `dynvariables` will result with the none executed value of the referenced `dynvariables` (see examples below) -* a `dynvariables` can reference a `variables` however the opposite is not possible * profile cannot include profiles defined above in the import tree * config files do not have access to variables defined above in the import tree From 958f9b052eec8096f59416cddf13f5c5ce76a4e0 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 25 Aug 2020 08:24:09 +0200 Subject: [PATCH 34/47] update doc --- CONTRIBUTING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 74aa6a3..72dbea0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,15 +52,15 @@ Only the higher layer is accessible to other classes of dotdrop. * config files do not have access to variables defined above in the import tree -This will result with `var0 = "echo test"` and not `var0 = test` +This will result with `dvar0 = "test"` and `var0 = "echo test"` (**not** `var0 = test`) ```yaml variables: - var0: "{{@@ dvar4 @@}}" + var0: "{{@@ dvar0 @@}}" dynvariables: dvar0: "echo test" ``` -This will result with `dvar0 = "test"` +This will result with `dvar0 = "test"` and `var0 = "test"` ```yaml variables: var0: "test" From 546e13b4a6ea657064b7884ded039e6fc5b19605 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 28 Aug 2020 17:42:01 +0200 Subject: [PATCH 35/47] adding test case for #250 --- tests-ng/import-variables.sh | 113 +++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100755 tests-ng/import-variables.sh diff --git a/tests-ng/import-variables.sh b/tests-ng/import-variables.sh new file mode 100755 index 0000000..cc9340e --- /dev/null +++ b/tests-ng/import-variables.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2019, deadc0de6 +# +# import variables from file +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` + +# create the config file +cfg1="${tmps}/config1.yaml" +cfg2="${tmps}/config2.yaml" +varf="${tmps}/variables.yaml" + +cat > ${cfg1} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + import_configs: + - ${cfg2} +dotfiles: + f_abc: + dst: ${tmpd}/abc + src: abc +profiles: + p0: + dotfiles: + - f_abc +_EOF + +cat > ${cfg2} << _EOF +config: + backup: true + create: true + dotpath: dotfiles-other + import_variables: + - ${varf} +dotfiles: +profiles: +_EOF + +cat > ${varf} << _EOF +variables: + var1: var1value +dynvariables: + dvar1: "echo dvar1value" +_EOF + +# create the source +mkdir -p ${tmps}/dotfiles/ +echo "start" > ${tmps}/dotfiles/abc +echo "{{@@ var1 @@}}" >> ${tmps}/dotfiles/abc +echo "{{@@ dvar1 @@}}" >> ${tmps}/dotfiles/abc +echo "end" >> ${tmps}/dotfiles/abc + +# install +cd ${ddpath} | ${bin} install -c ${cfg1} -p p0 -V + +# test file content +cat ${tmpd}/abc +grep 'var1value' ${tmpd}/abc >/dev/null 2>&1 +grep 'dvar1value' ${tmpd}/abc >/dev/null 2>&1 + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 From bf3f5b9c9cc8065a33ebfa37ecedf73452cd6fad Mon Sep 17 00:00:00 2001 From: Davide Laezza Date: Sat, 29 Aug 2020 19:50:34 +0200 Subject: [PATCH 36/47] Refactoring "import-variables" test to detect the real problem, "macro-with-globals", as explained in #250 --- ...ort-variables.sh => macro-with-globals.sh} | 49 +++++++------------ 1 file changed, 19 insertions(+), 30 deletions(-) rename tests-ng/{import-variables.sh => macro-with-globals.sh} (72%) diff --git a/tests-ng/import-variables.sh b/tests-ng/macro-with-globals.sh similarity index 72% rename from tests-ng/import-variables.sh rename to tests-ng/macro-with-globals.sh index cc9340e..9a0f62c 100755 --- a/tests-ng/import-variables.sh +++ b/tests-ng/macro-with-globals.sh @@ -52,17 +52,13 @@ mkdir -p ${tmps}/dotfiles tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` # create the config file -cfg1="${tmps}/config1.yaml" -cfg2="${tmps}/config2.yaml" -varf="${tmps}/variables.yaml" +cfg="${tmps}/config.yaml" -cat > ${cfg1} << _EOF +cat > ${cfg} << _EOF config: backup: true create: true dotpath: dotfiles - import_configs: - - ${cfg2} dotfiles: f_abc: dst: ${tmpd}/abc @@ -71,40 +67,33 @@ profiles: p0: dotfiles: - f_abc -_EOF - -cat > ${cfg2} << _EOF -config: - backup: true - create: true - dotpath: dotfiles-other - import_variables: - - ${varf} -dotfiles: -profiles: -_EOF - -cat > ${varf} << _EOF variables: - var1: var1value -dynvariables: - dvar1: "echo dvar1value" + global: global_var + local: local_var _EOF # create the source mkdir -p ${tmps}/dotfiles/ -echo "start" > ${tmps}/dotfiles/abc -echo "{{@@ var1 @@}}" >> ${tmps}/dotfiles/abc -echo "{{@@ dvar1 @@}}" >> ${tmps}/dotfiles/abc -echo "end" >> ${tmps}/dotfiles/abc + +cat > ${tmps}/dotfiles/macro_file << _EOF +{%@@ macro macro(var) @@%} +{{@@ global @@}} +{{@@ var @@}} +{%@@ endmacro @@%} +_EOF + +cat > ${tmps}/dotfiles/abc << _EOF +{%@@ from 'macro_file' import macro @@%} +{{@@ macro(local) @@}} +_EOF # install -cd ${ddpath} | ${bin} install -c ${cfg1} -p p0 -V +cd ${ddpath} | ${bin} install -c ${cfg} -p p0 -V # test file content cat ${tmpd}/abc -grep 'var1value' ${tmpd}/abc >/dev/null 2>&1 -grep 'dvar1value' ${tmpd}/abc >/dev/null 2>&1 +grep 'global_var' ${tmpd}/abc >/dev/null 2>&1 +grep 'local_var' ${tmpd}/abc >/dev/null 2>&1 ## CLEANING rm -rf ${tmps} ${tmpd} From fe5cec3427dbefcb06c302c1631ae52074f9edfd Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 30 Aug 2020 11:23:05 +0200 Subject: [PATCH 37/47] add "with context" to access variables with macro --- tests-ng/macro-with-globals.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests-ng/macro-with-globals.sh b/tests-ng/macro-with-globals.sh index 9a0f62c..48bb1d3 100755 --- a/tests-ng/macro-with-globals.sh +++ b/tests-ng/macro-with-globals.sh @@ -83,7 +83,7 @@ cat > ${tmps}/dotfiles/macro_file << _EOF _EOF cat > ${tmps}/dotfiles/abc << _EOF -{%@@ from 'macro_file' import macro @@%} +{%@@ from 'macro_file' import macro with context @@%} {{@@ macro(local) @@}} _EOF From 2512eeb65a35de9e3b7e3f238e378987fb778c5a Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 3 Sep 2020 08:32:57 +0200 Subject: [PATCH 38/47] adding test case for #250 --- tests-ng/include.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests-ng/include.sh b/tests-ng/include.sh index 0c28fbf..526ffa4 100755 --- a/tests-ng/include.sh +++ b/tests-ng/include.sh @@ -109,6 +109,9 @@ cd ${ddpath} | ${bin} files -c ${cfg} -p p2 | grep f_abc cd ${ddpath} | ${bin} files -c ${cfg} -p p3 | grep f_abc cd ${ddpath} | ${bin} files -c ${cfg} -p p0 | grep f_abc +cnt=`cd ${ddpath} | ${bin} files -c ${cfg} -p p0 | grep f_abc | wc -l` +[ "${cnt}" != "1" ] && echo "dotfiles displayed more than once" && exit 1 + # count cnt=`cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -b | grep '^f_' | wc -l` [ "${cnt}" != "1" ] && exit 1 From 0813eccbe1f2c9ab0cf2c982049024b6ac7cb524 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 3 Sep 2020 14:21:30 +0200 Subject: [PATCH 39/47] fix bug for #250 --- dotdrop/cfg_yaml.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 719ddc2..49b32fe 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -1077,15 +1077,17 @@ class CfgYaml: # only keep dotfiles related to the selected profile pdfs = [] - pro = self.profiles.get(self._profile) + pro = self.profiles.get(self._profile, []) if pro: - pdfs = pro.get(self.key_profile_dotfiles, []) + pdfs = list(pro.get(self.key_profile_dotfiles, [])) for addpro in self._inc_profiles: - pro = self.profiles.get(addpro) + pro = self.profiles.get(addpro, []) if not pro: continue pdfsalt = pro.get(self.key_profile_dotfiles, []) pdfs.extend(pdfsalt) + pdfs = uniq_list(pdfs) + if self.key_all not in pdfs: # take a subset of the dotfiles newdotfiles = {} From f44fdfdc24363646a0e83654c64c6db4f97aea4c Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 3 Sep 2020 14:21:51 +0200 Subject: [PATCH 40/47] refactoring --- tests.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests.sh b/tests.sh index 81e7457..16be406 100755 --- a/tests.sh +++ b/tests.sh @@ -6,24 +6,24 @@ set -ev # PEP8 tests -which pycodestyle 2>/dev/null +which pycodestyle >/dev/null 2>&1 [ "$?" != "0" ] && echo "Install pycodestyle" && exit 1 +echo "testing with pycodestyle" pycodestyle --ignore=W503,W504,W605 dotdrop/ pycodestyle tests/ pycodestyle scripts/ # pyflakes tests +echo "testing with pyflakes" pyflakes dotdrop/ pyflakes tests/ # retrieve the nosetests binary -set +e nosebin="nosetests" -which ${nosebin} 2>/dev/null +which ${nosebin} >/dev/null 2>&1 [ "$?" != "0" ] && nosebin="nosetests3" -which ${nosebin} 2>/dev/null +which ${nosebin} >/dev/null 2>&1 [ "$?" != "0" ] && echo "Install nosetests" && exit 1 -set -e # do not print debugs when running tests (faster) export DOTDROP_FORCE_NODEBUG=yes @@ -33,8 +33,8 @@ cur=`dirname $(readlink -f "${0}")` export COVERAGE_FILE="${cur}/.coverage" # execute tests with coverage -PYTHONPATH=dotdrop ${nosebin} -s --with-coverage --cover-package=dotdrop -#PYTHONPATH=dotdrop python3 -m pytest tests +PYTHONPATH="dotdrop" ${nosebin} -s --with-coverage --cover-package=dotdrop +#PYTHONPATH="dotdrop" python3 -m pytest tests # enable debug logs export DOTDROP_DEBUG= @@ -44,6 +44,7 @@ unset DOTDROP_FORCE_NODEBUG ## execute bash script tests [ "$1" = '--python-only' ] || { + echo "doing extended tests" log=`mktemp` for scr in tests-ng/*.sh; do if [ -z ${TRAVIS} ]; then From 2b576eaee2b20f920cefb32d0934d47e6c8c9121 Mon Sep 17 00:00:00 2001 From: Davide Laezza Date: Wed, 2 Sep 2020 22:34:34 +0200 Subject: [PATCH 41/47] Adding test for variables from an imported config that are used in directly a config file --- tests-ng/imported-variables-from-config.sh | 110 +++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100755 tests-ng/imported-variables-from-config.sh diff --git a/tests-ng/imported-variables-from-config.sh b/tests-ng/imported-variables-from-config.sh new file mode 100755 index 0000000..004c5df --- /dev/null +++ b/tests-ng/imported-variables-from-config.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# author: davla (https://github.com/davls) +# Copyright (c) 2020, davla +# +# test variables imported from config and used in the importing yaml config +# returns 1 in case of error +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` + +# create the config file +cfg="${tmps}/config.yaml" +subcfg="${tmps}/subconfig.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + import_configs: + - ${subcfg} +dotfiles: + f_abc: + dst: ${tmpd}/abc + src: '{{@@ abc_dyn_src @@}}{{@@ abc_src @@}}' +profiles: + p1: + dotfiles: + - f_abc +_EOF +cat ${cfg} + +# create the subconfig file +cat > ${subcfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +variables: + abc_src: c +dynvariables: + abc_dyn_src: 'echo ab' +dotfiles: [] +profiles: [] +_EOF + +# create the dotfile +dirname ${tmps}/dotfiles/abc | xargs mkdir -p +cat > ${tmps}/dotfiles/abc << _EOF +Hell yeah +_EOF + +# install +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V + +# test file existence and content +[ -f "${tmpd}/abc" ] || { + echo 'Dotfile not installed' + exit 1 +} + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 From 8c210d4fcb3a18a8e25e57ae9ad1a6f32b981dbd Mon Sep 17 00:00:00 2001 From: Davide Laezza Date: Wed, 2 Sep 2020 22:30:14 +0200 Subject: [PATCH 42/47] Properly adding variables for imported config (and redefining templater) --- dotdrop/cfg_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 719ddc2..24bda2d 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -854,7 +854,7 @@ class CfgYaml: if self._debug: self._debug_dict('add import_configs var', sub.variables) - self.variables = self._merge_dict(sub.variables, self.variables) + self._add_variables(sub.variables, prio=True) def _import_configs(self): """import configs from external files""" From 69cf14e302ad87ba99f59c58f2a90d9aa221883c Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 4 Sep 2020 07:10:41 +0200 Subject: [PATCH 43/47] fix compare bug --- dotdrop/cfg_aggregator.py | 5 +++-- dotdrop/dotdrop.py | 15 +++++++++++---- dotdrop/installer.py | 7 ++++--- dotdrop/templategen.py | 6 ++++-- tests/test_compare.py | 4 ++-- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/dotdrop/cfg_aggregator.py b/dotdrop/cfg_aggregator.py index 2be9297..ecc3d9c 100644 --- a/dotdrop/cfg_aggregator.py +++ b/dotdrop/cfg_aggregator.py @@ -135,8 +135,9 @@ class CfgAggregator: objects.append(o) if not islist: objects = objects[0] - if self.debug: - self.log.dbg('patching {}.{} with {}'.format(c, keys, objects)) + # if self.debug: + # er = 'patching {}.{} with {}' + # self.log.dbg(er.format(c, keys, objects)) setattr(c, keys, objects) def del_dotfile(self, dotfile): diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index efe618d..d27a437 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -230,6 +230,7 @@ def cmd_compare(o, tmp): newvars = dotfile.get_dotfile_variables() t.add_tmp_vars(newvars=newvars) + # dotfiles does not exist / not installed if o.debug: LOG.dbg('comparing {}'.format(dotfile)) src = dotfile.src @@ -239,9 +240,9 @@ def cmd_compare(o, tmp): same = False continue + # apply transformation tmpsrc = None if dotfile.trans_r: - # apply transformation if o.debug: LOG.dbg('applying transformation before comparing') tmpsrc = apply_trans(o.dotpath, dotfile, t, debug=o.debug) @@ -261,20 +262,26 @@ def cmd_compare(o, tmp): LOG.dbg('points to itself') continue - # install dotfile to temporary dir - ret, insttmp = inst.install_to_temp(t, tmp, src, dotfile.dst) + # install dotfile to temporary dir and compare + ret, err, insttmp = inst.install_to_temp(t, tmp, src, dotfile.dst) if not ret: # failed to install to tmp + line = '=> compare {}: error' + LOG.log(line.format(dotfile.key, err)) + LOG.err(err) same = False continue ignores = list(set(o.compare_ignore + dotfile.cmpignore)) ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) diff = comp.compare(insttmp, dotfile.dst, ignore=ignores) + + # clean tmp transformed dotfile if any if tmpsrc: - # clean tmp transformed dotfile if any tmpsrc = os.path.join(o.dotpath, tmpsrc) if os.path.exists(tmpsrc): remove(tmpsrc) + + # print diff result if diff == '': if o.debug: line = '=> compare {}: diffing with \"{}\"' diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 773c1c2..ef0e0a9 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -551,9 +551,10 @@ class Installer: src = os.path.expanduser(src) dst = os.path.expanduser(dst) if self.debug: - self.log.dbg('tmp install {} to {}'.format(src, dst)) + self.log.dbg('tmp install {} (defined dst: {})'.format(src, dst)) # install the dotfile to a temp directory for comparing - ret, tmpdst = self._install_to_temp(templater, src, dst, tmpdir) + r, tmpdst = self._install_to_temp(templater, src, dst, tmpdir) + ret, err = r if self.debug: self.log.dbg('tmp installed in {}'.format(tmpdst)) # reset flags @@ -561,4 +562,4 @@ class Installer: self.diff = diffsaved self.comparing = False self.create = createsaved - return ret, tmpdst + return ret, err, tmpdst diff --git a/dotdrop/templategen.py b/dotdrop/templategen.py index ff2f06b..feb7db0 100644 --- a/dotdrop/templategen.py +++ b/dotdrop/templategen.py @@ -90,7 +90,8 @@ class Templategen: try: return self._handle_file(src) except UndefinedError as e: - raise UndefinedException(e.message) + err = 'undefined variable: {}'.format(e.message) + raise UndefinedException(err) def generate_string(self, string): """ @@ -103,7 +104,8 @@ class Templategen: try: return self.env.from_string(string).render(self.variables) except UndefinedError as e: - raise UndefinedException(e.message) + err = 'undefined variable: {}'.format(e.message) + raise UndefinedException(err) def add_tmp_vars(self, newvars={}): """add vars to the globals, make sure to call restore_vars""" diff --git a/tests/test_compare.py b/tests/test_compare.py index 6c24db4..b436c92 100644 --- a/tests/test_compare.py +++ b/tests/test_compare.py @@ -36,8 +36,8 @@ class TestCompare(unittest.TestCase): results = {} for dotfile in dotfiles: path = os.path.expanduser(dotfile.dst) - ret, insttmp = inst.install_to_temp(t, tmp, dotfile.src, - dotfile.dst) + ret, err, insttmp = inst.install_to_temp(t, tmp, dotfile.src, + dotfile.dst) if not ret: results[path] = False continue From 6918bfd51a72b0814f92dfebf4584cbc73459ca0 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 7 Sep 2020 15:25:54 +0200 Subject: [PATCH 44/47] update contributing doc --- CONTRIBUTING.md | 185 ++++++++++++++++++++++++++---------------------- 1 file changed, 101 insertions(+), 84 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 72dbea0..8b0613c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,17 @@ +Content + +* [code base](#code-base) +* [config parsing](#config-parsing) + * [lower layer](#lower-layer) + * [higher layer](#higher-layer) + * [Precedence](#precedence) + * [variables resolution](#variables-resolution) + * [rules](#rules) +* [testing](#testing) + * [testing with unittest](#testing-with-unittest) + * [testing with bash scripts](#testing-with-bash-scripts) +* [documentation](#documentation) + Thanks for helping out! Feature requests, bug reports and PRs are always welcome! @@ -6,9 +20,9 @@ This file provides a few pointers on how to contribute to dotdrop and where to find information. For any question, feel free to open an issue. For PR adding new features, I'd be very thankful if you could add either -a unittest testing the added feature or a bash script test, thanks! +a unittest testing the added feature or a bash script test ((see [testing](#testing), thanks! -# Code base +# code base Dotdrop's code base is located in the [dotdrop directory](/dotdrop). @@ -32,25 +46,100 @@ Here's an overview of the different files and their role: * **updater.py**: the class handling the update of dotfiles for `update` * **utils.py**: some useful methods -## Config parsing +# config parsing -The configuration file (yaml) is parsed in two layers: +The configuration file (yaml) is parsed using two layers: - * the lower layer in `cfg_yaml.py` - * the higher layer in `cfg_aggregator.py` + * first in the lower layer in [cfg_yaml.py](/dotdrop/cfg_yaml.py) + * then in the higher layer in [cfg_aggregator.py](/dotdrop/cfg_aggregator.py) Only the higher layer is accessible to other classes of dotdrop. -**Rules** +## lower layer + +This is done in [cfg_yaml.py](/dotdrop/cfg_yaml.py) + +The lower layer part is only taking care of basic types +and does the following: + * normalize all config entries + * resolve paths (dotfiles src, dotpath, etc) + * refactor actions/transformations to a common format + * etc + * import any data from external files (configs, variables, etc) + * apply variable substitutions + * complete any data if needed (add the "profile" variable, etc) + * execute intrepreted variables through the shell + * write new entries (dotfile, profile) into the dictionary and save it to a file + * fix any deprecated entries (link_by_default, etc) + * clear empty entries + +In the end it builds a cleaned and normalized dictionary to be accessed by the higher layer. + +## higher layer + +This is done in [cfg_aggregator.py](/dotdrop/cfg_aggregator.py) + +The higher layer will transform the dictionary parsed by the lower layer +into objects (profiles, dotfiles, actions, transformations, etc). +The higher layer has no notion of inclusion (profile included for example) or +file importing (import actions, etc) or even interpreted variables +(it only sees variables that have already been interpreted). + +It does the following: + * transform dictionaries into objects + * patch list of keys with its corresponding object (for example dotfile's actions) + * provide getters for every other classes of dotdrop needing to access elements + +Note that any change to the yaml dictionary (adding a new profile or a new dotfile for +example) won't be *seen* by the higher layer until the config is reloaded. Consider the +`dirty` flag as a sign the file needs to be written and its representation in higher +levels in not accurate anymore. + +## precedence + +* `dynvariables` > `variables` +* profile `(dyn)variables` > any other `(dyn)variables` +* profile `(dyn)variables` > profile's included `(dyn)variables` +* imported `variables`/`dynvariables` > `(dyn)variables` + +## variables resolution + +How variables are resolved (through jinja2's +templating) in the config file. + +* resolve main config file variables + * merge `variables` and `dynvariables` (allowing cycling reference) + * recursively template merged `variables` and `dynvariables` + * `dynvariables` are executed + * profile's `variables` and `dynvariables` are merged +* resolve *included* entries (see below) + * paths and entries are templated + (allows to use something like `include {{@@ os @@}}.variables.yaml`) +* *included* entries are processed + * dyn-/variables are all resolved in their own file + +potential *included* entries + +* entry *import_actions* +* entry *import_configs* +* entry *import_variables* +* profile's *import* +* profile's *include + +Variables are then used to resolve different elements in the config file: +see [this](https://github.com/deadc0de6/dotdrop/wiki/config-variables#config-available-variables) + +## rules * `dynvariables` are executed in their own config file * since `variables` and `dynvariables` are templated before the `dynvariables` are executed, this means that `dynvariables` can safely reference `variables` however - `variables` referencing `dynvariables` will result with the none executed value of the + `variables` referencing `dynvariables` will result with the *not-executed* value of the referenced `dynvariables` (see examples below) * profile cannot include profiles defined above in the import tree -* config files do not have access to variables - defined above in the import tree +* config files do not have access to variables defined above in the import tree +* actions/transformations using variables are resolved at runtime + (when action/transformation is executed) and not when loading the config This will result with `dvar0 = "test"` and `var0 = "echo test"` (**not** `var0 = test`) ```yaml @@ -68,80 +157,8 @@ dynvariables: dvar0: "echo {{@@ var0 @@}}" ``` -**Precedence** -* `dynvariables` > `variables` -* profile `(dyn)variables` > any other `(dyn)variables` -* profile `(dyn)variables` > profile's included `(dyn)variables` -* imported `variables`/`dynvariables` > `(dyn)variables` -* actions/transformations using variables are resolved at runtime - (when action/transformation is executed) and not when loading the config - -## lower layer (cfg_yaml.py) - -The lower layer part is only taking care of basic types -does the following: - * normalize all config entries - * resolve paths (dotfiles src, dotpath, etc) - * refactor actions to a common format - * etc - * import any data from external files (configs, variables, etc) - * apply variable substitutions - * complete any data if needed (add the "profile" variable, etc) - * execute intrepreted variables through the shell - * write new entries (dotfile, profile) into the dictionary and save it to a file - * fix any deprecated entries (link_by_default, etc) - * clear empty entries - -In the end it makes sure the dictionary accessed -by the higher layer is clean and normalized. - -## higher layer (cfg_aggregator.py) - -The higher layer will transform the dictionary parsed by the lower layer -into objects (profiles, dotfiles, actions, etc). -The higher layer has no notion of inclusion (profile included for example) or -file importing (import actions, etc) or even interpreted variables -(it only sees variables that have already been interpreted). - -It does the following: - * transform dictionaries into objects - * patch list of keys with its corresponding object (for example dotfile's actions) - * provide getters for every other classes of dotdrop needing to access elements - -Note that any change to the yaml dictionary (adding a new profile or a new dotfile for -example) won't be *seen* by the higher layer until the config is reloaded. Consider the -`dirty` flag as a sign the file needs to be written and its representation in higher -levels in not accurate anymore. - -## Variables resolution in the config file - -How variables are resolved (through jinja2's -templating) in the config file. - -* resolve main config file variables - * merge `variables` and `dynvariables` (allowing cycling reference) - * recursively template merged `variables` and `dynvariables` - * `dynvariables` are executed - * profile's `variables` and `dynvariables` are merged -* resolve *included* entries (see below) - * paths and entries are templated - (allows to use something like `include {{@@ os @@}}.variables.yaml`) -* *included* entries are processed - * dyn-/variables are all resolved in their own file - -*included* - -* entry *import_actions* -* entry *import_configs* -* entry *import_variables* -* profile's *import* -* profile's *include - -Variables are then used to resolve different elements in the config file: -see [this](https://github.com/deadc0de6/dotdrop/wiki/config-variables#config-available-variables) - -# Testing +# testing Dotdrop is tested with the use of the [tests.sh](/tests.sh) script. @@ -166,6 +183,6 @@ for different use-cases (usually described in their filename). Each script starts with the same boiler plate code that you can paste at the start of your new test (see the head of the file down to `# this is the test`). -# Documentation +# documentation Most of dotdrop documentation is hosted in [its wiki](https://github.com/deadc0de6/dotdrop/wiki) From d9dc7388213ca7e1c47fa791177847eb9cf07149 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 7 Sep 2020 16:03:28 +0200 Subject: [PATCH 45/47] update readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9194de0..85a2484 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Features: Check also the [blog post](https://deadc0de.re/articles/dotfiles.html), the [example](#getting-started), the [wiki](https://github.com/deadc0de6/dotdrop/wiki) or -how [people are using dotdrop](https://github.com/deadc0de6/dotdrop/wiki/people-using-dotdrop) +how [people are using dotdrop](https://github.com/deadc0de6/dotdrop/wiki/meta-people-using-dotdrop) for more. Quick start: @@ -261,8 +261,8 @@ That's it, a single repository with all your dotfiles for your different hosts. You can then -* [create actions](https://github.com/deadc0de6/dotdrop/wiki/usage-actions) -* [use transformations](https://github.com/deadc0de6/dotdrop/wiki/usage-transformations) +* [create actions](https://github.com/deadc0de6/dotdrop/wiki/config-actions) +* [use transformations](https://github.com/deadc0de6/dotdrop/wiki/config-transformations) * [use variables](https://github.com/deadc0de6/dotdrop/wiki/config-variables) * [symlink dotfiles](https://github.com/deadc0de6/dotdrop/wiki/symlinked-dotfiles) * [and more](https://github.com/deadc0de6/dotdrop/wiki) From 8d9521534d9a20868637213a44a8d75476df95ed Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 10 Sep 2020 17:59:55 +0200 Subject: [PATCH 46/47] adding test for #256 --- tests-ng/import-with-empty.sh | 106 ++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100755 tests-ng/import-with-empty.sh diff --git a/tests-ng/import-with-empty.sh b/tests-ng/import-with-empty.sh new file mode 100755 index 0000000..0c2c50b --- /dev/null +++ b/tests-ng/import-with-empty.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2019, deadc0de6 +# +# test import new dotfiles with empty dst/src on existing dotfiles +# returns 1 in case of error +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# dotdrop directory +basedir=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +echo "[+] dotdrop dir: ${basedir}" +echo "[+] dotpath dir: ${basedir}/dotfiles" +# the temp directory +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` + +# create a dotfile +dftoimport="${tmpd}/a_dotfile" +echo 'some content' > ${dftoimport} + +# create the config file +cfg="${basedir}/config.yaml" +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: + f_x: + src: /tmp/x + dst: + f_y: + src: + dst: /tmp/y + f_z: + src: + dst: + f_l: + src: + dst: + link: link + f_lc: + src: + dst: + link: link_children +profiles: + p1: + dotfiles: + - f_x + - f_y + - f_z + - f_l + - f_lc +_EOF + +echo "[+] import" +#cd ${ddpath} | ${bin} install -c ${cfg} -p p1 --verbose | grep '^5 dotfile(s) installed.$' +cd ${ddpath} | ${bin} import -c ${cfg} -p p1 --verbose ${dftoimport} +[ "$?" != "0" ] && exit 1 + +cat ${cfg} + +## CLEANING +rm -rf ${basedir} ${tmpd} + +echo "OK" +exit 0 From 91b220162a9482abf2a46af844f005897e3e06ef Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 10 Sep 2020 18:09:28 +0200 Subject: [PATCH 47/47] fix bug #256 --- dotdrop/cfg_yaml.py | 3 +++ tests-ng/import-with-empty.sh | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 19af396..ce514b9 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -1258,6 +1258,9 @@ class CfgYaml: """recursively delete all none/empty values in a dictionary.""" new = {} for k, v in dic.items(): + if k == self.key_dotfiles and v: + new[k] = v + continue newv = v if isinstance(v, dict): newv = self._clear_none(v) diff --git a/tests-ng/import-with-empty.sh b/tests-ng/import-with-empty.sh index 0c2c50b..b43fb93 100755 --- a/tests-ng/import-with-empty.sh +++ b/tests-ng/import-with-empty.sh @@ -93,11 +93,18 @@ profiles: _EOF echo "[+] import" -#cd ${ddpath} | ${bin} install -c ${cfg} -p p1 --verbose | grep '^5 dotfile(s) installed.$' cd ${ddpath} | ${bin} import -c ${cfg} -p p1 --verbose ${dftoimport} [ "$?" != "0" ] && exit 1 -cat ${cfg} +echo "[+] install" +cd ${ddpath} | ${bin} install -c ${cfg} -p p1 --verbose | grep '^5 dotfile(s) installed.$' +rm -f ${dftoimport} +cd ${ddpath} | ${bin} install -c ${cfg} -p p1 --verbose | grep '^6 dotfile(s) installed.$' + +nb=`cd ${ddpath} | ${bin} files -c ${cfg} -p p1 --verbose | grep '^[a-zA-Z]' | wc -l` +[ "${nb}" != "6" ] && echo 'error in dotfile list' && exit 1 + +#cat ${cfg} ## CLEANING rm -rf ${basedir} ${tmpd}