diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bd49d9f..209f64b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -98,8 +98,9 @@ then merged and take precedence over local variables. Note: * `dynvariables` > `variables` -* profile variables > (`variables` or `dynvariables`) -* imported `variables`/`dynvariables` > any other `variables` or `dynvariables` +* profile `(dyn)variables` > any other `(dyn)variables` +* profile `(dyn)variables` > profile's included `(dyn)variables` +* imported `variables`/`dynvariables` > `(dyn)variables` * actions using variables are resolved at runtime (when action is executed) and not when loading the config diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index efb5b00..25190c6 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -91,7 +91,7 @@ class CfgYaml: self.log.dbg('before normalization: {}'.format(self.yaml_dict)) # resolve variables - self.variables = self._merge_variables() + self.variables, self.prokeys = self._merge_variables() # apply variables self._apply_variables() @@ -252,30 +252,24 @@ class CfgYaml: v[self.key_profile_include] = new # now get the included ones - incl_var = self._get_included_variables(self.profile, - seen=[self.profile]) - incl_dvar = self._get_included_dvariables(self.profile, - seen=[self.profile]) + pro_var = self._get_included_variables(self.profile, + seen=[self.profile]) + pro_dvar = self._get_included_dvariables(self.profile, + seen=[self.profile]) # exec incl dynvariables - self._shell_exec_dvars(incl_dvar.keys(), incl_dvar) + self._shell_exec_dvars(pro_dvar.keys(), pro_dvar) # merge all and resolve - merged = self._merge_dict(incl_var, merged) - merged = self._merge_dict(incl_dvar, merged) + merged = self._merge_dict(pro_var, merged) + merged = self._merge_dict(pro_dvar, merged) merged = self._rec_resolve_vars(merged) - if self.debug: - self.log.dbg('with included variables') - self._debug_vars(merged) - - if self.debug: - self.log.dbg('with included variables') - self._debug_vars(merged) if self.debug: self.log.dbg('resolve all uses of variables in config') self._debug_vars(merged) - return merged + prokeys = list(pro_var.keys()) + list(pro_dvar.keys()) + return merged, prokeys def _apply_variables(self): """template any needed parts of the config""" @@ -416,7 +410,7 @@ class CfgYaml: variables.update(new) cur = pentry.get(self.key_profile_variables, {}) - return self._merge_dict(variables, cur) + return self._merge_dict(cur, variables) def _get_included_dvariables(self, profile, seen): """return included dynvariables""" @@ -440,7 +434,7 @@ class CfgYaml: variables.update(new) cur = pentry.get(self.key_profile_dvariables, {}) - return self._merge_dict(variables, cur) + return self._merge_dict(cur, variables) def _resolve_profile_all(self): """resolve some other parts of the config""" @@ -449,9 +443,9 @@ class CfgYaml: dfs = v.get(self.key_profile_dotfiles, None) if not dfs: continue - if self.debug: - self.log.dbg('add ALL to profile {}'.format(k)) if self.key_all in dfs: + if self.debug: + self.log.dbg('add ALL to profile {}'.format(k)) v[self.key_profile_dotfiles] = self.dotfiles.keys() def _resolve_profile_includes(self): @@ -538,8 +532,13 @@ class CfgYaml: merged = self._rec_resolve_vars(merged) # execute dvar self._shell_exec_dvars(dvar.keys(), merged) + self._clear_profile_vars(merged) self.variables = self._merge_dict(merged, self.variables) + def _clear_profile_vars(self, dic): + """remove profile variables from dic if found""" + [dic.pop(k, None) for k in self.prokeys] + def _import_actions(self): """import external actions from paths""" paths = self.settings.get(self.key_import_actions, None) @@ -571,26 +570,32 @@ class CfgYaml: mandatory=False) v[self.key_dotfiles] = new + current + def _import_config(self, path): + """import config from path""" + path = self._norm_path(path) + if self.debug: + self.log.dbg('import config from {}'.format(path)) + sub = CfgYaml(path, profile=self.profile, debug=self.debug) + # settings is ignored + self.dotfiles = self._merge_dict(self.dotfiles, sub.dotfiles) + self.profiles = self._merge_dict(self.profiles, sub.profiles) + self.actions = self._merge_dict(self.actions, sub.actions) + self.trans_r = self._merge_dict(self.trans_r, sub.trans_r) + self.trans_w = self._merge_dict(self.trans_w, sub.trans_w) + self._clear_profile_vars(sub.variables) + if self.debug: + self.log.dbg('add import_configs var: {}'.format(sub.variables)) + self.variables = self._merge_dict(sub.variables, self.variables) + def _import_configs(self): - """import configs from external file""" + """import configs from external files""" # settings -> import_configs imp = self.settings.get(self.key_import_configs, None) if not imp: return paths = self._glob_paths(imp) for path in paths: - path = self._norm_path(path) - if self.debug: - self.log.dbg('import config from {}'.format(path)) - sub = CfgYaml(path, debug=self.debug) - # settings is ignored - self.dotfiles = self._merge_dict(self.dotfiles, sub.dotfiles) - self.profiles = self._merge_dict(self.profiles, sub.profiles) - self.actions = self._merge_dict(self.actions, sub.actions) - self.trans_r = self._merge_dict(self.trans_r, sub.trans_r) - self.trans_w = self._merge_dict(self.trans_w, sub.trans_w) - self.variables = self._merge_dict(self.variables, - sub.variables) + self._import_config(path) def _import_sub(self, path, key, mandatory=False, patch_func=None): diff --git a/tests-ng/extvariables.sh b/tests-ng/extvariables.sh index 180102d..3c49742 100755 --- a/tests-ng/extvariables.sh +++ b/tests-ng/extvariables.sh @@ -116,7 +116,7 @@ grep '^var3: var1 var2 var3' ${tmpd}/abc >/dev/null grep '^dvar3: dvar1 dvar2 dvar3' ${tmpd}/abc >/dev/null grep '^var4: echo var1 var2 var3' ${tmpd}/abc >/dev/null grep '^dvar4: var1 var2 var3' ${tmpd}/abc >/dev/null -grep '^varx: exttest' ${tmpd}/abc >/dev/null +grep '^varx: profvarx' ${tmpd}/abc >/dev/null grep '^evar1: extevar1' ${tmpd}/abc >/dev/null grep '^provar: provar' ${tmpd}/abc >/dev/null @@ -178,7 +178,7 @@ grep '^var3: extvar1 var2 var3' ${tmpd}/abc >/dev/null grep '^dvar3: extdvar1 dvar2 dvar3' ${tmpd}/abc >/dev/null grep '^var4: echo extvar1 var2 var3' ${tmpd}/abc >/dev/null grep '^dvar4: extvar1 var2 var3' ${tmpd}/abc >/dev/null -grep '^varx: exttest' ${tmpd}/abc >/dev/null +grep '^varx: profvarx' ${tmpd}/abc >/dev/null grep '^vary: profvary' ${tmpd}/abc >/dev/null ## CLEANING diff --git a/tests-ng/imported-configs-variables.sh b/tests-ng/imported-configs-variables.sh new file mode 100755 index 0000000..e74450f --- /dev/null +++ b/tests-ng/imported-configs-variables.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2019, deadc0de6 +# +# test external config's variables +# returns 1 in case of error +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "\e[96m\e[1m==> RUNNING $(basename $BASH_SOURCE) <==\e[0m" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests'` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests'` +#echo "dotfile destination: ${tmpd}" + +# create the config file +extcfg="${tmps}/ext-config.yaml" +cfg="${tmps}/config.yaml" +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + import_configs: + - $(basename ${extcfg}) +variables: + varx: "test" + provar: "local" +dynvariables: + dvarx: "echo dtest" + dprovar: "echo dlocal" +dotfiles: + f_abc: + dst: ${tmpd}/abc + src: abc +profiles: + p1: + dotfiles: + - f_abc + variables: + varx: profvarx + provar: provar + dynvariables: + dvarx: echo dprofvarx + dprovar: echo dprovar +_EOF +cat ${cfg} + +# create the external variables file +cat > ${extcfg} << _EOF +config: +profiles: + p2: + dotfiles: + - f_abc + variables: + varx: extprofvarx + provar: extprovar + dynvariables: + dvarx: echo extdprofvarx + dprovar: echo extdprovar +dotfiles: +_EOF +ls -l ${extcfg} +cat ${extcfg} + +# create the dotfile +echo "varx: {{@@ varx @@}}" > ${tmps}/dotfiles/abc +echo "provar: {{@@ provar @@}}" >> ${tmps}/dotfiles/abc +echo "dvarx: {{@@ dvarx @@}}" >> ${tmps}/dotfiles/abc +echo "dprovar: {{@@ dprovar@@}}" >> ${tmps}/dotfiles/abc + +#cat ${tmps}/dotfiles/abc + +# install +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p2 -V + +echo "test1" +cat ${tmpd}/abc +grep '^varx: extprofvarx' ${tmpd}/abc >/dev/null +grep '^provar: extprovar' ${tmpd}/abc >/dev/null +grep '^dvarx: extdprofvarx' ${tmpd}/abc >/dev/null +grep '^dprovar: extdprovar' ${tmpd}/abc >/dev/null + +rm -f ${tmpd}/abc +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V + +echo "test2" +cat ${tmpd}/abc +grep '^varx: profvarx' ${tmpd}/abc >/dev/null +grep '^provar: provar' ${tmpd}/abc >/dev/null +grep '^dvarx: dprofvarx' ${tmpd}/abc >/dev/null +grep '^dprovar: dprovar' ${tmpd}/abc >/dev/null + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 diff --git a/tests/test_yamlcfg.py b/tests/test_yamlcfg.py index 33570b1..909feca 100644 --- a/tests/test_yamlcfg.py +++ b/tests/test_yamlcfg.py @@ -583,9 +583,11 @@ profiles: )) # test variables + # since variables get merged they are + # the same in both configs imported_vars = imported_cfg.variables self.assertFalse(any( - imported_vars[k] == v + imported_vars[k] != v for k, v in importing_cfg.variables.items() if not k.startswith('_') ))