profile variables get precedence (#169)

CONTRIBUTING.md

@@ -98,8 +98,9 @@ then merged and take precedence over local variables.
 Note:
 
 * `dynvariables` > `variables`
-* profile variables > (`variables` or `dynvariables`)
+* profile `(dyn)variables` > any other `(dyn)variables`
-* imported `variables`/`dynvariables` > any other `variables` or `dynvariables`
+* profile `(dyn)variables` > profile's included `(dyn)variables`
+* imported `variables`/`dynvariables` > `(dyn)variables`
 * actions using variables are resolved at runtime (when action is executed)
   and not when loading the config
 

dotdrop/cfg_yaml.py

@@ -91,7 +91,7 @@ class CfgYaml:
             self.log.dbg('before normalization: {}'.format(self.yaml_dict))
 
         # resolve variables
-        self.variables = self._merge_variables()
+        self.variables, self.prokeys = self._merge_variables()
 
         # apply variables
         self._apply_variables()
@@ -252,30 +252,24 @@ class CfgYaml:
                 v[self.key_profile_include] = new
 
         # now get the included ones
-        incl_var = self._get_included_variables(self.profile,
+        pro_var = self._get_included_variables(self.profile,
-                                                seen=[self.profile])
+                                               seen=[self.profile])
-        incl_dvar = self._get_included_dvariables(self.profile,
+        pro_dvar = self._get_included_dvariables(self.profile,
-                                                  seen=[self.profile])
+                                                 seen=[self.profile])
         # exec incl dynvariables
-        self._shell_exec_dvars(incl_dvar.keys(), incl_dvar)
+        self._shell_exec_dvars(pro_dvar.keys(), pro_dvar)
 
         # merge all and resolve
-        merged = self._merge_dict(incl_var, merged)
+        merged = self._merge_dict(pro_var, merged)
-        merged = self._merge_dict(incl_dvar, merged)
+        merged = self._merge_dict(pro_dvar, merged)
         merged = self._rec_resolve_vars(merged)
-        if self.debug:
-            self.log.dbg('with included variables')
-            self._debug_vars(merged)
-
-        if self.debug:
-            self.log.dbg('with included variables')
-            self._debug_vars(merged)
 
         if self.debug:
             self.log.dbg('resolve all uses of variables in config')
             self._debug_vars(merged)
 
-        return merged
+        prokeys = list(pro_var.keys()) + list(pro_dvar.keys())
+        return merged, prokeys
 
     def _apply_variables(self):
         """template any needed parts of the config"""
@@ -416,7 +410,7 @@ class CfgYaml:
             variables.update(new)
 
         cur = pentry.get(self.key_profile_variables, {})
-        return self._merge_dict(variables, cur)
+        return self._merge_dict(cur, variables)
 
     def _get_included_dvariables(self, profile, seen):
         """return included dynvariables"""
@@ -440,7 +434,7 @@ class CfgYaml:
             variables.update(new)
 
         cur = pentry.get(self.key_profile_dvariables, {})
-        return self._merge_dict(variables, cur)
+        return self._merge_dict(cur, variables)
 
     def _resolve_profile_all(self):
         """resolve some other parts of the config"""
@@ -449,9 +443,9 @@ class CfgYaml:
             dfs = v.get(self.key_profile_dotfiles, None)
             if not dfs:
                 continue
-            if self.debug:
-                self.log.dbg('add ALL to profile {}'.format(k))
             if self.key_all in dfs:
+                if self.debug:
+                    self.log.dbg('add ALL to profile {}'.format(k))
                 v[self.key_profile_dotfiles] = self.dotfiles.keys()
 
     def _resolve_profile_includes(self):
@@ -538,8 +532,13 @@ class CfgYaml:
             merged = self._rec_resolve_vars(merged)
             # execute dvar
             self._shell_exec_dvars(dvar.keys(), merged)
+            self._clear_profile_vars(merged)
             self.variables = self._merge_dict(merged, self.variables)
 
+    def _clear_profile_vars(self, dic):
+        """remove profile variables from dic if found"""
+        [dic.pop(k, None) for k in self.prokeys]
+
     def _import_actions(self):
         """import external actions from paths"""
         paths = self.settings.get(self.key_import_actions, None)
@@ -571,26 +570,32 @@ class CfgYaml:
                                        mandatory=False)
                 v[self.key_dotfiles] = new + current
 
+    def _import_config(self, path):
+        """import config from path"""
+        path = self._norm_path(path)
+        if self.debug:
+            self.log.dbg('import config from {}'.format(path))
+        sub = CfgYaml(path, profile=self.profile, debug=self.debug)
+        # settings is ignored
+        self.dotfiles = self._merge_dict(self.dotfiles, sub.dotfiles)
+        self.profiles = self._merge_dict(self.profiles, sub.profiles)
+        self.actions = self._merge_dict(self.actions, sub.actions)
+        self.trans_r = self._merge_dict(self.trans_r, sub.trans_r)
+        self.trans_w = self._merge_dict(self.trans_w, sub.trans_w)
+        self._clear_profile_vars(sub.variables)
+        if self.debug:
+            self.log.dbg('add import_configs var: {}'.format(sub.variables))
+        self.variables = self._merge_dict(sub.variables, self.variables)
+
     def _import_configs(self):
-        """import configs from external file"""
+        """import configs from external files"""
         # settings -> import_configs
         imp = self.settings.get(self.key_import_configs, None)
         if not imp:
             return
         paths = self._glob_paths(imp)
         for path in paths:
-            path = self._norm_path(path)
+            self._import_config(path)
-            if self.debug:
-                self.log.dbg('import config from {}'.format(path))
-            sub = CfgYaml(path, debug=self.debug)
-            # settings is ignored
-            self.dotfiles = self._merge_dict(self.dotfiles, sub.dotfiles)
-            self.profiles = self._merge_dict(self.profiles, sub.profiles)
-            self.actions = self._merge_dict(self.actions, sub.actions)
-            self.trans_r = self._merge_dict(self.trans_r, sub.trans_r)
-            self.trans_w = self._merge_dict(self.trans_w, sub.trans_w)
-            self.variables = self._merge_dict(self.variables,
-                                              sub.variables)
 
     def _import_sub(self, path, key,
                     mandatory=False, patch_func=None):

tests-ng/extvariables.sh

@@ -116,7 +116,7 @@ grep '^var3: var1 var2 var3' ${tmpd}/abc >/dev/null
 grep '^dvar3: dvar1 dvar2 dvar3' ${tmpd}/abc >/dev/null
 grep '^var4: echo var1 var2 var3' ${tmpd}/abc >/dev/null
 grep '^dvar4: var1 var2 var3' ${tmpd}/abc >/dev/null
-grep '^varx: exttest' ${tmpd}/abc >/dev/null
+grep '^varx: profvarx' ${tmpd}/abc >/dev/null
 grep '^evar1: extevar1' ${tmpd}/abc >/dev/null
 grep '^provar: provar' ${tmpd}/abc >/dev/null
 
@@ -178,7 +178,7 @@ grep '^var3: extvar1 var2 var3' ${tmpd}/abc >/dev/null
 grep '^dvar3: extdvar1 dvar2 dvar3' ${tmpd}/abc >/dev/null
 grep '^var4: echo extvar1 var2 var3' ${tmpd}/abc >/dev/null
 grep '^dvar4: extvar1 var2 var3' ${tmpd}/abc >/dev/null
-grep '^varx: exttest' ${tmpd}/abc >/dev/null
+grep '^varx: profvarx' ${tmpd}/abc >/dev/null
 grep '^vary: profvary' ${tmpd}/abc >/dev/null
 
 ## CLEANING

tests-ng/imported-configs-variables.sh

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+# author: deadc0de6 (https://github.com/deadc0de6)
+# Copyright (c) 2019, deadc0de6
+#
+# test external config's variables
+# returns 1 in case of error
+#
+
+# exit on first error
+set -e
+
+# all this crap to get current path
+rl="readlink -f"
+if ! ${rl} "${0}" >/dev/null 2>&1; then
+  rl="realpath"
+
+  if ! hash ${rl}; then
+    echo "\"${rl}\" not found !" && exit 1
+  fi
+fi
+cur=$(dirname "$(${rl} "${0}")")
+
+#hash dotdrop >/dev/null 2>&1
+#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1
+
+#echo "called with ${1}"
+
+# dotdrop path can be pass as argument
+ddpath="${cur}/../"
+[ "${1}" != "" ] && ddpath="${1}"
+[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1
+
+export PYTHONPATH="${ddpath}:${PYTHONPATH}"
+bin="python3 -m dotdrop.dotdrop"
+
+echo "dotdrop path: ${ddpath}"
+echo "pythonpath: ${PYTHONPATH}"
+
+# get the helpers
+source ${cur}/helpers
+
+echo -e "\e[96m\e[1m==> RUNNING $(basename $BASH_SOURCE) <==\e[0m"
+
+################################################################
+# this is the test
+################################################################
+
+# the dotfile source
+tmps=`mktemp -d --suffix='-dotdrop-tests'`
+mkdir -p ${tmps}/dotfiles
+# the dotfile destination
+tmpd=`mktemp -d --suffix='-dotdrop-tests'`
+#echo "dotfile destination: ${tmpd}"
+
+# create the config file
+extcfg="${tmps}/ext-config.yaml"
+cfg="${tmps}/config.yaml"
+cat > ${cfg} << _EOF
+config:
+  backup: true
+  create: true
+  dotpath: dotfiles
+  import_configs:
+  - $(basename ${extcfg})
+variables:
+  varx: "test"
+  provar: "local"
+dynvariables:
+  dvarx: "echo dtest"
+  dprovar: "echo dlocal"
+dotfiles:
+  f_abc:
+    dst: ${tmpd}/abc
+    src: abc
+profiles:
+  p1:
+    dotfiles:
+    - f_abc
+    variables:
+      varx: profvarx
+      provar: provar
+    dynvariables:
+      dvarx: echo dprofvarx
+      dprovar: echo dprovar
+_EOF
+cat ${cfg}
+
+# create the external variables file
+cat > ${extcfg} << _EOF
+config:
+profiles:
+  p2:
+    dotfiles:
+    - f_abc
+    variables:
+      varx: extprofvarx
+      provar: extprovar
+    dynvariables:
+      dvarx: echo extdprofvarx
+      dprovar: echo extdprovar
+dotfiles:
+_EOF
+ls -l ${extcfg}
+cat ${extcfg}
+
+# create the dotfile
+echo "varx: {{@@ varx @@}}" > ${tmps}/dotfiles/abc
+echo "provar: {{@@ provar @@}}" >> ${tmps}/dotfiles/abc
+echo "dvarx: {{@@ dvarx @@}}" >> ${tmps}/dotfiles/abc
+echo "dprovar: {{@@ dprovar@@}}" >> ${tmps}/dotfiles/abc
+
+#cat ${tmps}/dotfiles/abc
+
+# install
+cd ${ddpath} | ${bin} install -f -c ${cfg} -p p2 -V
+
+echo "test1"
+cat ${tmpd}/abc
+grep '^varx: extprofvarx' ${tmpd}/abc >/dev/null
+grep '^provar: extprovar' ${tmpd}/abc >/dev/null
+grep '^dvarx: extdprofvarx' ${tmpd}/abc >/dev/null
+grep '^dprovar: extdprovar' ${tmpd}/abc >/dev/null
+
+rm -f ${tmpd}/abc
+cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V
+
+echo "test2"
+cat ${tmpd}/abc
+grep '^varx: profvarx' ${tmpd}/abc >/dev/null
+grep '^provar: provar' ${tmpd}/abc >/dev/null
+grep '^dvarx: dprofvarx' ${tmpd}/abc >/dev/null
+grep '^dprovar: dprovar' ${tmpd}/abc >/dev/null
+
+## CLEANING
+rm -rf ${tmps} ${tmpd}
+
+echo "OK"
+exit 0

tests/test_yamlcfg.py

@@ -583,9 +583,11 @@ profiles:
         ))
 
         # test variables
+        # since variables get merged they are
+        # the same in both configs
         imported_vars = imported_cfg.variables
         self.assertFalse(any(
-            imported_vars[k] == v
+            imported_vars[k] != v
             for k, v in importing_cfg.variables.items()
             if not k.startswith('_')
         ))