From 02365a19e5b0c3a4fd0a9aa2aeb3c0f494364878 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 2 Jul 2020 14:34:19 +0200 Subject: [PATCH 1/3] adding test case for #246 --- tests-ng/profile-dyninclude.sh | 138 +++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100755 tests-ng/profile-dyninclude.sh diff --git a/tests-ng/profile-dyninclude.sh b/tests-ng/profile-dyninclude.sh new file mode 100755 index 0000000..097d479 --- /dev/null +++ b/tests-ng/profile-dyninclude.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2017, deadc0de6 +# +# test profile dynvariables and included dynvariables +# returns 1 in case of error +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` + +# create the config file +cfg="${tmps}/config.yaml" +cfg2="${tmps}/sub.yaml" + +cat > ${cfg} << _EOF +config: + dotpath: dotfiles + import_configs: + - sub.yaml +variables: + mainvar: 'not-that' + subvar: 'not-that-either' +dynvariables: + maindyn: 'echo wont-work' + subdyn: 'echo wont-work-either' +dotfiles: +profiles: + profile_1: + include: + - subprofile + dynvariables: + maindyn: 'echo maindyncontent' + variables: + mainvar: 'maincontent' + profile_2: + include: + - subignore +_EOF +#cat ${cfg} + +cat > ${cfg2} << _EOF +config: +dotfiles: + f_abc: + dst: ${tmpd}/abc + src: abc + f_def: + dst: ${tmpd}/def + src: def +variables: + mainvar: 'bad0' + subvar: 'bad1' +dynvariables: + maindyn: 'echo bad2' + subdyn: 'echo bad3' +profiles: + subprofile: + dotfiles: + - f_abc + dynvariables: + subdyn: 'echo subdyncontent' + variables: + subvar: 'subcontent' + subignore: + dotfiles: + - f_def +_EOF +#cat ${cfg2} + +# create the dotfile +echo "start" > ${tmps}/dotfiles/abc +echo "{{@@ mainvar @@}}" >> ${tmps}/dotfiles/abc +echo "{{@@ maindyn @@}}" >> ${tmps}/dotfiles/abc +echo "{{@@ subdyn @@}}" >> ${tmps}/dotfiles/abc +echo "{{@@ subvar @@}}" >> ${tmps}/dotfiles/abc +echo "end" >> ${tmps}/dotfiles/abc +cat ${tmps}/dotfiles/abc + +# install +cd ${ddpath} | ${bin} install -f -c ${cfg} -p profile_1 --verbose + +# check dotfile exists +[ ! -e ${tmpd}/abc ] && exit 1 +cat ${tmpd}/abc +grep 'maincontent' ${tmpd}/abc >/dev/null || (echo "variables 1 not resolved" && exit 1) +grep 'maindyncontent' ${tmpd}/abc >/dev/null || (echo "dynvariables 1 not resolved" && exit 1) +grep 'subcontent' ${tmpd}/abc >/dev/null || (echo "variables 2 not resolved" && exit 1) +grep 'subdyncontent' ${tmpd}/abc >/dev/null || (echo "dynvariables 2 not resolved" && exit 1) +#cat ${tmpd}/abc + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 From 8bfdfa28266eea647f70506de8943d78f4dc9ac7 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 3 Jul 2020 10:14:25 +0200 Subject: [PATCH 2/3] fix bug #246 --- dotdrop/cfg_yaml.py | 153 +++++++++++++++++++++++++------------------- 1 file changed, 88 insertions(+), 65 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index f3c4090..9d7c5f7 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -287,6 +287,31 @@ class CfgYaml: t.update_variables(variables) return variables + def _get_profile_included_vars(self, tvars): + """resolve profile included variables/dynvariables""" + t = Templategen(variables=tvars, + func_file=self.settings[Settings.key_func_file], + filter_file=self.settings[Settings.key_filter_file]) + + for k, v in self.profiles.items(): + if self.key_profile_include in v: + new = [] + for x in v[self.key_profile_include]: + new.append(t.generate_string(x)) + v[self.key_profile_include] = new + + # now get the included ones + pro_var = self._get_profile_included_item(self.profile, + self.key_profile_variables, + seen=[self.profile]) + pro_dvar = self._get_profile_included_item(self.profile, + self.key_profile_dvariables, + seen=[self.profile]) + + # exec incl dynvariables + self._shell_exec_dvars(pro_dvar.keys(), pro_dvar) + return pro_var, pro_dvar + def _merge_variables(self): """ resolve all variables across the config @@ -314,25 +339,8 @@ class CfgYaml: self.log.dbg('local variables resolved') self._debug_dict('variables', merged) - # resolve profile includes - t = Templategen(variables=merged, - func_file=self.settings[Settings.key_func_file], - filter_file=self.settings[Settings.key_filter_file]) - - for k, v in self.profiles.items(): - if self.key_profile_include in v: - new = [] - for x in v[self.key_profile_include]: - new.append(t.generate_string(x)) - v[self.key_profile_include] = new - - # now get the included ones - pro_var = self._get_profile_included_variables(self.profile, - seen=[self.profile]) - pro_dvar = self._get_profile_included_dvariables(self.profile, - seen=[self.profile]) - # exec incl dynvariables - self._shell_exec_dvars(pro_dvar.keys(), pro_dvar) + # resolve profile included variables/dynvariables + pro_var, pro_dvar = self._get_profile_included_vars(merged) # merge all and resolve merged = self._merge_dict(pro_var, merged) @@ -459,54 +467,29 @@ class CfgYaml: variables = deepcopy(self.ori_dvariables) return variables - def _get_profile_included_variables(self, profile, seen): - """return included variables from profile""" - variables = {} + def _get_profile_included_item(self, profile, item, seen): + """recursively get included from profile""" + items = {} if not profile or profile not in self.profiles.keys(): - return variables + return items - # profile entry + # considered profile entry pentry = self.profiles.get(profile) - # inherite profile variables + # recursively get from inherited profile for inherited_profile in pentry.get(self.key_profile_include, []): if inherited_profile == profile or inherited_profile in seen: raise YamlException('\"include\" loop') seen.append(inherited_profile) - new = self._get_profile_included_variables(inherited_profile, - seen) + new = self._get_profile_included_item(inherited_profile, + item, seen) if self.debug: - msg = 'included vars from {}: {}' - self.log.dbg(msg.format(inherited_profile, new)) - variables.update(new) + msg = 'included {} from {}: {}' + self.log.dbg(msg.format(item, inherited_profile, new)) + items.update(new) - cur = pentry.get(self.key_profile_variables, {}) - return self._merge_dict(cur, variables) - - def _get_profile_included_dvariables(self, profile, seen): - """return included dynvariables from profile""" - variables = {} - - if not profile or profile not in self.profiles.keys(): - return variables - - # profile entry - pentry = self.profiles.get(profile) - - # inherite profile dynvariables - for inherited_profile in pentry.get(self.key_profile_include, []): - if inherited_profile == profile or inherited_profile in seen: - raise YamlException('\"include loop\"') - seen.append(inherited_profile) - new = self._get_profile_included_dvariables(inherited_profile, - seen) - if self.debug: - msg = 'included dvars from {}: {}' - self.log.dbg(msg.format(inherited_profile, new)) - variables.update(new) - - cur = pentry.get(self.key_profile_dvariables, {}) - return self._merge_dict(cur, variables) + cur = pentry.get(item, {}) + return self._merge_dict(cur, items) def _resolve_profile_all(self): """resolve some other parts of the config""" @@ -530,22 +513,39 @@ class CfgYaml: recursively resolve include of other profiles's: * dotfiles * actions + * variables + * dynvariables + variables/dynvariables are directly merged with the + global variables (self.variables) if these are + included in the selected profile + returns dotfiles, actions, variables, dynvariables """ this_profile = self.profiles[profile] + # the profiles included by the selected profile + included = [] + if self.profile and self.profile in self.profiles: + included = self.profiles[self.profile] \ + .get(self.key_profile_include, []) or [] - # include + # considered profile content dotfiles = this_profile.get(self.key_profile_dotfiles, []) or [] actions = this_profile.get(self.key_profile_actions, []) or [] includes = this_profile.get(self.key_profile_include, []) or [] + pvars = this_profile.get(self.key_profile_variables, []) or [] + pdvars = this_profile.get(self.key_profile_dvariables, []) or [] if not includes: # nothing to include - return dotfiles, actions + return dotfiles, actions, pvars, pdvars if self.debug: - self.log.dbg('{} includes: {}'.format(profile, ','.join(includes))) + self.log.dbg('{} includes {}'.format(profile, ','.join(includes))) self.log.dbg('{} dotfiles before include: {}'.format(profile, dotfiles)) self.log.dbg('{} actions before include: {}'.format(profile, actions)) + self.log.dbg('{} variables before include: {}'.format(profile, + pvars)) + self.log.dbg('{} dynvariables before include: {}'.format(profile, + pdvars)) seen = [] for i in uniq_list(includes): @@ -557,28 +557,51 @@ class CfgYaml: if i not in self.profiles.keys(): self.log.warn('include unknown profile: {}'.format(i)) continue + # recursive resolve - o_dfs, o_actions = self._rec_resolve_profile_include(i) + o_dfs, o_actions, o_v, o_dv = self._rec_resolve_profile_include(i) + # merge dotfile keys dotfiles.extend(o_dfs) this_profile[self.key_profile_dotfiles] = uniq_list(dotfiles) + # merge actions keys actions.extend(o_actions) this_profile[self.key_profile_actions] = uniq_list(actions) + # merge variables + pvars = self._merge_dict(o_v, pvars) + this_profile[self.key_profile_variables] = pvars + + # merge dynvariables + pdvars = self._merge_dict(o_dv, pdvars) + self._shell_exec_dvars(pdvars.keys(), pdvars) + this_profile[self.key_profile_dvariables] = pdvars + + if i in included: + # only merge variables/dynvariables with the global variables + # if the considered profile is included by the selected profile + self.variables = self._merge_dict(pvars, self.variables) + self.variables = self._merge_dict(pdvars, self.variables) + dotfiles = this_profile.get(self.key_profile_dotfiles, []) actions = this_profile.get(self.key_profile_actions, []) + pvars = this_profile.get(self.key_profile_variables, []) or [] + pdvars = this_profile.get(self.key_profile_dvariables, []) or [] if self.debug: self.log.dbg('{} dotfiles after include: {}'.format(profile, dotfiles)) self.log.dbg('{} actions after include: {}'.format(profile, actions)) + self.log.dbg('{} variables after include: {}'.format(profile, + pvars)) + self.log.dbg('{} dynvariables after include: {}'.format(profile, + pdvars)) - # since dotfiles and actions are resolved here - # and variables have been already done at the beginning - # of the parsing, we can clear these include + # since included items are resolved here + # we can clear these include self.profiles[profile][self.key_profile_include] = None - return dotfiles, actions + return dotfiles, actions, pvars, pdvars ######################################################## # handle imported entries From fb69e154c43ded5d84ab92a83a5c20c83d7822bd Mon Sep 17 00:00:00 2001 From: Davide Laezza Date: Sun, 5 Jul 2020 20:22:52 +0200 Subject: [PATCH 3/3] Moving execution of dynamic variables and merging of dynvariables/variables after the include loop in _rec_resolve_profile_include --- dotdrop/cfg_yaml.py | 49 +++++++++++++++++++++++----------- tests-ng/profile-dyninclude.sh | 3 +-- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 9d7c5f7..5dfa50a 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -521,21 +521,17 @@ class CfgYaml: returns dotfiles, actions, variables, dynvariables """ this_profile = self.profiles[profile] - # the profiles included by the selected profile - included = [] - if self.profile and self.profile in self.profiles: - included = self.profiles[self.profile] \ - .get(self.key_profile_include, []) or [] # considered profile content dotfiles = this_profile.get(self.key_profile_dotfiles, []) or [] actions = this_profile.get(self.key_profile_actions, []) or [] includes = this_profile.get(self.key_profile_include, []) or [] - pvars = this_profile.get(self.key_profile_variables, []) or [] - pdvars = this_profile.get(self.key_profile_dvariables, []) or [] + pvars = this_profile.get(self.key_profile_variables, {}) or {} + pdvars = this_profile.get(self.key_profile_dvariables, {}) or {} if not includes: # nothing to include return dotfiles, actions, pvars, pdvars + if self.debug: self.log.dbg('{} includes {}'.format(profile, ','.join(includes))) self.log.dbg('{} dotfiles before include: {}'.format(profile, @@ -549,6 +545,10 @@ class CfgYaml: seen = [] for i in uniq_list(includes): + if self.debug: + self.log.dbg('resolving includes "{}" <- "{}"' + .format(profile, i)) + # ensure no include loop occurs if i in seen: raise YamlException('\"include loop\"') @@ -559,35 +559,45 @@ class CfgYaml: continue # recursive resolve + if self.debug: + self.log.dbg('recursively resolving includes for profile "{}"' + .format(i)) o_dfs, o_actions, o_v, o_dv = self._rec_resolve_profile_include(i) # merge dotfile keys + if self.debug: + self.log.dbg('Merging dotfiles {} <- {}: {} <- {}' + .format(profile, i, dotfiles, o_dfs)) dotfiles.extend(o_dfs) this_profile[self.key_profile_dotfiles] = uniq_list(dotfiles) # merge actions keys + if self.debug: + self.log.dbg('Merging actions {} <- {}: {} <- {}' + .format(profile, i, actions, o_actions)) actions.extend(o_actions) this_profile[self.key_profile_actions] = uniq_list(actions) # merge variables + if self.debug: + self.log.dbg('Merging variables {} <- {}: {} <- {}' + .format(profile, i, dict(pvars), dict(o_v))) pvars = self._merge_dict(o_v, pvars) this_profile[self.key_profile_variables] = pvars # merge dynvariables + if self.debug: + self.log.dbg('Merging dynamic variables {} <- {}: {} <- {}' + .format(profile, i, dict(pdvars), + dict(o_dv))) pdvars = self._merge_dict(o_dv, pdvars) - self._shell_exec_dvars(pdvars.keys(), pdvars) this_profile[self.key_profile_dvariables] = pdvars - if i in included: - # only merge variables/dynvariables with the global variables - # if the considered profile is included by the selected profile - self.variables = self._merge_dict(pvars, self.variables) - self.variables = self._merge_dict(pdvars, self.variables) - dotfiles = this_profile.get(self.key_profile_dotfiles, []) actions = this_profile.get(self.key_profile_actions, []) - pvars = this_profile.get(self.key_profile_variables, []) or [] - pdvars = this_profile.get(self.key_profile_dvariables, []) or [] + pvars = this_profile.get(self.key_profile_variables, {}) or {} + pdvars = this_profile.get(self.key_profile_dvariables, {}) or {} + if self.debug: self.log.dbg('{} dotfiles after include: {}'.format(profile, dotfiles)) @@ -598,6 +608,13 @@ class CfgYaml: self.log.dbg('{} dynvariables after include: {}'.format(profile, pdvars)) + if profile == self.profile: + # Only for the selected profile, we execute dynamic variables and + # we merge variables/dynvariables into the global variables + self._shell_exec_dvars(pdvars.keys(), pdvars) + self.variables = self._merge_dict(pvars, self.variables) + self.variables = self._merge_dict(pdvars, self.variables) + # since included items are resolved here # we can clear these include self.profiles[profile][self.key_profile_include] = None diff --git a/tests-ng/profile-dyninclude.sh b/tests-ng/profile-dyninclude.sh index 097d479..b21022e 100755 --- a/tests-ng/profile-dyninclude.sh +++ b/tests-ng/profile-dyninclude.sh @@ -117,14 +117,13 @@ echo "{{@@ maindyn @@}}" >> ${tmps}/dotfiles/abc echo "{{@@ subdyn @@}}" >> ${tmps}/dotfiles/abc echo "{{@@ subvar @@}}" >> ${tmps}/dotfiles/abc echo "end" >> ${tmps}/dotfiles/abc -cat ${tmps}/dotfiles/abc +#cat ${tmps}/dotfiles/abc # install cd ${ddpath} | ${bin} install -f -c ${cfg} -p profile_1 --verbose # check dotfile exists [ ! -e ${tmpd}/abc ] && exit 1 -cat ${tmpd}/abc grep 'maincontent' ${tmpd}/abc >/dev/null || (echo "variables 1 not resolved" && exit 1) grep 'maindyncontent' ${tmpd}/abc >/dev/null || (echo "dynvariables 1 not resolved" && exit 1) grep 'subcontent' ${tmpd}/abc >/dev/null || (echo "variables 2 not resolved" && exit 1)