diff --git a/README.md b/README.md index 16ce677..dbc002c 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ why [dotdrop](https://github.com/deadc0de6/dotdrop) rocks. * [Include dotfiles from another profile](#include-dotfiles-from-another-profile) * [Templating](#templating) * [Available variables](#available-variables) + * [Variables from file](#variables-from-file) * [Available methods](#available-methods) * [Dynamic dotfile paths](#dynamic-dotfile-paths) * [Dynamic actions](#dynamic-actions) @@ -655,6 +656,8 @@ the following entries: (absolute path or relative to the config file location, defaults to *~/.config/dotdrop*) * `showdiff`: on install show a diff before asking to overwrite (see `--showdiff`) (default *false*) * `ignoreempty`: do not deploy template if empty (default *false*) + * `import_variables`: list of paths to load variables from + (absolute path or relative to the config file location). * **dotfiles** entry: a list of dotfiles * `dst`: where this dotfile needs to be deployed (can use `variables` and `dynvariables`, make sure to quote). @@ -858,7 +861,7 @@ will result in the following available variables * var4: `echo var1 var2 var3` * dvar4: `var1 var2 var3` -## Variables +### Variables Variables can be added in the config file under the `variables` entry. The variables added there are directly reachable in any templates. @@ -895,7 +898,7 @@ profiles: - f_gitconfig ``` -## Interpreted variables +### Interpreted variables It is also possible to have *dynamic* variables in the sense that their content will be interpreted by the shell before being replaced in the templates. @@ -916,7 +919,7 @@ These can be used as any variables in the templates As for variables (see [Variables](#variables)) profile dynvariables will take precedence over globally defined dynvariables. -## Environment variables +### Environment variables It's possible to access environment variables inside the templates. ``` @@ -948,6 +951,35 @@ alias dotdrop='eval $(grep -v "^#" ~/dotfiles/.env) /usr/bin/dotdrop --cfg=~/dot The above aliases load all the variables from `~/dotfiles/.env` (while omitting lines starting with `#`) before calling dotdrop. +## Variables from file + +Variables can be loaded from external files by specifying their +paths in the config entry `import_variables`. + +`config.yaml` +```yaml +config: + import_variables: + - variables.yaml +variables: + v1: var2 +``` + +`variables.yaml` +```yaml +variables: + v1: var1 + v2: var2 +dynvariables: + dv1: "echo test" +``` + +External variables will take precedence over variables defined within +the source config file. + +This can be useful for example if you have sensitive information stored in variables +and want those to be encrypted when versioned. + ## Available methods Beside jinja2 global functions diff --git a/dotdrop/config.py b/dotdrop/config.py index 14e021b..06980ed 100644 --- a/dotdrop/config.py +++ b/dotdrop/config.py @@ -33,6 +33,7 @@ class Cfg: key_showdiff = 'showdiff' key_deflink = 'link_by_default' key_workdir = 'workdir' + key_include_vars = 'import_variables' # actions keys key_actions = 'actions' @@ -120,7 +121,11 @@ class Cfg: # NOT linked inside the yaml dict (self.content) self.prodots = {} - if not self._load_file(): + # represents all variables from external files + self.ext_variables = {} + self.ext_dynvariables = {} + + if not self._load_config(): raise ValueError('config is not valid') def eval_dotfiles(self, profile, variables, debug=False): @@ -141,14 +146,26 @@ class Cfg: action.action = t.generate_string(action.action) return dotfiles - def _load_file(self): + def _load_config(self): """load the yaml file""" - with open(self.cfgpath, 'r') as f: - self.content = yaml.safe_load(f) + self.content = self._load_yaml(self.cfgpath) if not self._is_valid(): return False return self._parse() + def _load_yaml(self, path): + """load a yaml file to a dict""" + content = {} + if not os.path.exists(path): + return content + with open(path, 'r') as f: + try: + content = yaml.safe_load(f) + except Exception as e: + self.log.err(e) + return {} + return content + def _is_valid(self): """test the yaml dict (self.content) is valid""" if self.key_profiles not in self.content: @@ -350,8 +367,38 @@ class Cfg: self.lnk_settings[self.key_workdir] = \ self._abs_path(self.curworkdir) + # load external variables/dynvariables + if self.key_include_vars in self.lnk_settings: + paths = self.lnk_settings[self.key_include_vars] + self._load_ext_variables(paths) + return True + def _load_ext_variables(self, paths): + """load external variables""" + variables = {} + dvariables = {} + for path in paths: + path = self._abs_path(path) + if self.debug: + self.log.dbg('loading variables from {}'.format(path)) + content = self._load_yaml(path) + if not content: + self.log.warn('\"{}\" does not exist'.format(path)) + continue + # variables + if self.key_variables in content: + variables.update(content[self.key_variables]) + # dynamic variables + if self.key_dynvariables in content: + dvariables.update(content[self.key_dynvariables]) + self.ext_variables = variables + if self.debug: + self.log.dbg('loaded ext variables: {}'.format(variables)) + self.ext_dynvariables = dvariables + if self.debug: + self.log.dbg('loaded ext dynvariables: {}'.format(dvariables)) + def _abs_path(self, path): """return absolute path of path relative to the confpath""" path = os.path.expanduser(path) @@ -669,6 +716,9 @@ class Cfg: if self.key_variables in self.content: variables.update(self.content[self.key_variables]) + # external variables + variables.update(self.ext_variables) + if profile not in self.lnk_profiles: return variables @@ -689,6 +739,9 @@ class Cfg: # interpret dynamic variables variables.update(self.content[self.key_dynvariables]) + # external variables + variables.update(self.ext_dynvariables) + if profile not in self.lnk_profiles: return variables diff --git a/tests-ng/extvariables.sh b/tests-ng/extvariables.sh new file mode 100755 index 0000000..db97e88 --- /dev/null +++ b/tests-ng/extvariables.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2017, deadc0de6 +# +# test external variables +# returns 1 in case of error +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "\e[96m\e[1m==> RUNNING $(basename $BASH_SOURCE) <==\e[0m" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests'` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests'` +#echo "dotfile destination: ${tmpd}" + +# create the config file +extvars="${tmps}/variables.yaml" +cfg="${tmps}/config.yaml" +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + import_variables: + - $(basename ${extvars}) +variables: + var1: "var1" + var2: "{{@@ var1 @@}} var2" + var3: "{{@@ var2 @@}} var3" + var4: "{{@@ dvar4 @@}}" + varx: "test" +dynvariables: + dvar1: "echo dvar1" + dvar2: "{{@@ dvar1 @@}} dvar2" + dvar3: "{{@@ dvar2 @@}} dvar3" + dvar4: "echo {{@@ var3 @@}}" +dotfiles: + f_abc: + dst: ${tmpd}/abc + src: abc +profiles: + p1: + dotfiles: + - f_abc + variables: + varx: profvarx +_EOF +#cat ${cfg} + +# create the external variables file +cat > ${extvars} << _EOF +variables: + var1: "extvar1" + varx: "exttest" +dynvariables: + dvar1: "echo extdvar1" +_EOF + +# create the dotfile +echo "var3: {{@@ var3 @@}}" > ${tmps}/dotfiles/abc +echo "dvar3: {{@@ dvar3 @@}}" >> ${tmps}/dotfiles/abc +echo "var4: {{@@ var4 @@}}" >> ${tmps}/dotfiles/abc +echo "dvar4: {{@@ dvar4 @@}}" >> ${tmps}/dotfiles/abc +echo "varx: {{@@ varx @@}}" >> ${tmps}/dotfiles/abc + +#cat ${tmps}/dotfiles/abc + +# install +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V + +#cat ${tmpd}/abc + +grep '^var3: extvar1 var2 var3' ${tmpd}/abc >/dev/null +grep '^dvar3: extdvar1 dvar2 dvar3' ${tmpd}/abc >/dev/null +grep '^var4: echo extvar1 var2 var3' ${tmpd}/abc >/dev/null +grep '^dvar4: extvar1 var2 var3' ${tmpd}/abc >/dev/null +grep '^varx: profvarx' ${tmpd}/abc >/dev/null + +rm -f ${tmpd}/abc + +#cat ${tmpd}/abc +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + import_variables: + - $(basename ${extvars}) +dotfiles: + f_abc: + dst: ${tmpd}/abc + src: abc +profiles: + p1: + dotfiles: + - f_abc + variables: + varx: profvarx +_EOF +#cat ${cfg} + +# create the external variables file +cat > ${extvars} << _EOF +variables: + var1: "extvar1" + varx: "exttest" + var2: "{{@@ var1 @@}} var2" + var3: "{{@@ var2 @@}} var3" + var4: "{{@@ dvar4 @@}}" +dynvariables: + dvar1: "echo extdvar1" + dvar2: "{{@@ dvar1 @@}} dvar2" + dvar3: "{{@@ dvar2 @@}} dvar3" + dvar4: "echo {{@@ var3 @@}}" +_EOF + +# create the dotfile +echo "var3: {{@@ var3 @@}}" > ${tmps}/dotfiles/abc +echo "dvar3: {{@@ dvar3 @@}}" >> ${tmps}/dotfiles/abc +echo "var4: {{@@ var4 @@}}" >> ${tmps}/dotfiles/abc +echo "dvar4: {{@@ dvar4 @@}}" >> ${tmps}/dotfiles/abc +echo "varx: {{@@ varx @@}}" >> ${tmps}/dotfiles/abc + +#cat ${tmps}/dotfiles/abc + +# install +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V + +#cat ${tmpd}/abc + +grep '^var3: extvar1 var2 var3' ${tmpd}/abc >/dev/null +grep '^dvar3: extdvar1 dvar2 dvar3' ${tmpd}/abc >/dev/null +grep '^var4: echo extvar1 var2 var3' ${tmpd}/abc >/dev/null +grep '^dvar4: extvar1 var2 var3' ${tmpd}/abc >/dev/null +grep '^varx: profvarx' ${tmpd}/abc >/dev/null + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0