Merge branch 'extvariables'

README.md

@@ -84,6 +84,7 @@ why [dotdrop](https://github.com/deadc0de6/dotdrop) rocks.
   * [Include dotfiles from another profile](#include-dotfiles-from-another-profile)
 * [Templating](#templating)
   * [Available variables](#available-variables)
+  * [Variables from file](#variables-from-file)
   * [Available methods](#available-methods)
   * [Dynamic dotfile paths](#dynamic-dotfile-paths)
   * [Dynamic actions](#dynamic-actions)
@@ -655,6 +656,8 @@ the following entries:
     (absolute path or relative to the config file location, defaults to *~/.config/dotdrop*)
   * `showdiff`: on install show a diff before asking to overwrite (see `--showdiff`) (default *false*)
   * `ignoreempty`: do not deploy template if empty (default *false*)
+  * `import_variables`: list of paths to load variables from
+    (absolute path or relative to the config file location).
 
 * **dotfiles** entry: a list of dotfiles
   * `dst`: where this dotfile needs to be deployed (can use `variables` and `dynvariables`, make sure to quote).
@@ -858,7 +861,7 @@ will result in the following available variables
 * var4: `echo var1 var2 var3`
 * dvar4: `var1 var2 var3`
 
-## Variables
+### Variables
 
 Variables can be added in the config file under the `variables` entry.
 The variables added there are directly reachable in any templates.
@@ -895,7 +898,7 @@ profiles:
     - f_gitconfig
 ```
 
-## Interpreted variables
+### Interpreted variables
 
 It is also possible to have *dynamic* variables in the sense that their
 content will be interpreted by the shell before being replaced in the templates.
@@ -916,7 +919,7 @@ These can be used as any variables in the templates
 As for variables (see [Variables](#variables)) profile dynvariables will take
 precedence over globally defined dynvariables.
 
-## Environment variables
+### Environment variables
 
 It's possible to access environment variables inside the templates.
 ```
@@ -948,6 +951,35 @@ alias dotdrop='eval $(grep -v "^#" ~/dotfiles/.env) /usr/bin/dotdrop --cfg=~/dot
 The above aliases load all the variables from `~/dotfiles/.env`
 (while omitting lines starting with `#`) before calling dotdrop.
 
+## Variables from file
+
+Variables can be loaded from external files by specifying their
+paths in the config entry `import_variables`.
+
+`config.yaml`
+```yaml
+config:
+  import_variables:
+  - variables.yaml
+variables:
+  v1: var2
+```
+
+`variables.yaml`
+```yaml
+variables:
+  v1: var1
+  v2: var2
+dynvariables:
+  dv1: "echo test"
+```
+
+External variables will take precedence over variables defined within
+the source config file.
+
+This can be useful for example if you have sensitive information stored in variables
+and want those to be encrypted when versioned.
+
 ## Available methods
 
 Beside jinja2 global functions

dotdrop/config.py

@@ -33,6 +33,7 @@ class Cfg:
     key_showdiff = 'showdiff'
     key_deflink = 'link_by_default'
     key_workdir = 'workdir'
+    key_include_vars = 'import_variables'
 
     # actions keys
     key_actions = 'actions'
@@ -120,7 +121,11 @@ class Cfg:
         # NOT linked inside the yaml dict (self.content)
         self.prodots = {}
 
-        if not self._load_file():
+        # represents all variables from external files
+        self.ext_variables = {}
+        self.ext_dynvariables = {}
+
+        if not self._load_config():
             raise ValueError('config is not valid')
 
     def eval_dotfiles(self, profile, variables, debug=False):
@@ -141,14 +146,26 @@ class Cfg:
                     action.action = t.generate_string(action.action)
         return dotfiles
 
-    def _load_file(self):
+    def _load_config(self):
         """load the yaml file"""
-        with open(self.cfgpath, 'r') as f:
+        self.content = self._load_yaml(self.cfgpath)
-            self.content = yaml.safe_load(f)
         if not self._is_valid():
             return False
         return self._parse()
 
+    def _load_yaml(self, path):
+        """load a yaml file to a dict"""
+        content = {}
+        if not os.path.exists(path):
+            return content
+        with open(path, 'r') as f:
+            try:
+                content = yaml.safe_load(f)
+            except Exception as e:
+                self.log.err(e)
+                return {}
+        return content
+
     def _is_valid(self):
         """test the yaml dict (self.content) is valid"""
         if self.key_profiles not in self.content:
@@ -350,8 +367,38 @@ class Cfg:
         self.lnk_settings[self.key_workdir] = \
             self._abs_path(self.curworkdir)
 
+        # load external variables/dynvariables
+        if self.key_include_vars in self.lnk_settings:
+            paths = self.lnk_settings[self.key_include_vars]
+            self._load_ext_variables(paths)
+
         return True
 
+    def _load_ext_variables(self, paths):
+        """load external variables"""
+        variables = {}
+        dvariables = {}
+        for path in paths:
+            path = self._abs_path(path)
+            if self.debug:
+                self.log.dbg('loading variables from {}'.format(path))
+            content = self._load_yaml(path)
+            if not content:
+                self.log.warn('\"{}\" does not exist'.format(path))
+                continue
+            # variables
+            if self.key_variables in content:
+                variables.update(content[self.key_variables])
+            # dynamic variables
+            if self.key_dynvariables in content:
+                dvariables.update(content[self.key_dynvariables])
+        self.ext_variables = variables
+        if self.debug:
+            self.log.dbg('loaded ext variables: {}'.format(variables))
+        self.ext_dynvariables = dvariables
+        if self.debug:
+            self.log.dbg('loaded ext dynvariables: {}'.format(dvariables))
+
     def _abs_path(self, path):
         """return absolute path of path relative to the confpath"""
         path = os.path.expanduser(path)
@@ -669,6 +716,9 @@ class Cfg:
         if self.key_variables in self.content:
             variables.update(self.content[self.key_variables])
 
+        # external variables
+        variables.update(self.ext_variables)
+
         if profile not in self.lnk_profiles:
             return variables
 
@@ -689,6 +739,9 @@ class Cfg:
             # interpret dynamic variables
             variables.update(self.content[self.key_dynvariables])
 
+        # external variables
+        variables.update(self.ext_dynvariables)
+
         if profile not in self.lnk_profiles:
             return variables
 

tests-ng/extvariables.sh

@@ -0,0 +1,180 @@
+#!/usr/bin/env bash
+# author: deadc0de6 (https://github.com/deadc0de6)
+# Copyright (c) 2017, deadc0de6
+#
+# test external variables
+# returns 1 in case of error
+#
+
+# exit on first error
+set -e
+
+# all this crap to get current path
+rl="readlink -f"
+if ! ${rl} "${0}" >/dev/null 2>&1; then
+  rl="realpath"
+
+  if ! hash ${rl}; then
+    echo "\"${rl}\" not found !" && exit 1
+  fi
+fi
+cur=$(dirname "$(${rl} "${0}")")
+
+#hash dotdrop >/dev/null 2>&1
+#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1
+
+#echo "called with ${1}"
+
+# dotdrop path can be pass as argument
+ddpath="${cur}/../"
+[ "${1}" != "" ] && ddpath="${1}"
+[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1
+
+export PYTHONPATH="${ddpath}:${PYTHONPATH}"
+bin="python3 -m dotdrop.dotdrop"
+
+echo "dotdrop path: ${ddpath}"
+echo "pythonpath: ${PYTHONPATH}"
+
+# get the helpers
+source ${cur}/helpers
+
+echo -e "\e[96m\e[1m==> RUNNING $(basename $BASH_SOURCE) <==\e[0m"
+
+################################################################
+# this is the test
+################################################################
+
+# the dotfile source
+tmps=`mktemp -d --suffix='-dotdrop-tests'`
+mkdir -p ${tmps}/dotfiles
+# the dotfile destination
+tmpd=`mktemp -d --suffix='-dotdrop-tests'`
+#echo "dotfile destination: ${tmpd}"
+
+# create the config file
+extvars="${tmps}/variables.yaml"
+cfg="${tmps}/config.yaml"
+cat > ${cfg} << _EOF
+config:
+  backup: true
+  create: true
+  dotpath: dotfiles
+  import_variables:
+  - $(basename ${extvars})
+variables:
+  var1: "var1"
+  var2: "{{@@ var1 @@}} var2"
+  var3: "{{@@ var2 @@}} var3"
+  var4: "{{@@ dvar4 @@}}"
+  varx: "test"
+dynvariables:
+  dvar1: "echo dvar1"
+  dvar2: "{{@@ dvar1 @@}} dvar2"
+  dvar3: "{{@@ dvar2 @@}} dvar3"
+  dvar4: "echo {{@@ var3 @@}}"
+dotfiles:
+  f_abc:
+    dst: ${tmpd}/abc
+    src: abc
+profiles:
+  p1:
+    dotfiles:
+    - f_abc
+    variables:
+      varx: profvarx
+_EOF
+#cat ${cfg}
+
+# create the external variables file
+cat > ${extvars} << _EOF
+variables:
+  var1: "extvar1"
+  varx: "exttest"
+dynvariables:
+  dvar1: "echo extdvar1"
+_EOF
+
+# create the dotfile
+echo "var3: {{@@ var3 @@}}" > ${tmps}/dotfiles/abc
+echo "dvar3: {{@@ dvar3 @@}}" >> ${tmps}/dotfiles/abc
+echo "var4: {{@@ var4 @@}}" >> ${tmps}/dotfiles/abc
+echo "dvar4: {{@@ dvar4 @@}}" >> ${tmps}/dotfiles/abc
+echo "varx: {{@@ varx @@}}" >> ${tmps}/dotfiles/abc
+
+#cat ${tmps}/dotfiles/abc
+
+# install
+cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V
+
+#cat ${tmpd}/abc
+
+grep '^var3: extvar1 var2 var3' ${tmpd}/abc >/dev/null
+grep '^dvar3: extdvar1 dvar2 dvar3' ${tmpd}/abc >/dev/null
+grep '^var4: echo extvar1 var2 var3' ${tmpd}/abc >/dev/null
+grep '^dvar4: extvar1 var2 var3' ${tmpd}/abc >/dev/null
+grep '^varx: profvarx' ${tmpd}/abc >/dev/null
+
+rm -f ${tmpd}/abc
+
+#cat ${tmpd}/abc
+cat > ${cfg} << _EOF
+config:
+  backup: true
+  create: true
+  dotpath: dotfiles
+  import_variables:
+  - $(basename ${extvars})
+dotfiles:
+  f_abc:
+    dst: ${tmpd}/abc
+    src: abc
+profiles:
+  p1:
+    dotfiles:
+    - f_abc
+    variables:
+      varx: profvarx
+_EOF
+#cat ${cfg}
+
+# create the external variables file
+cat > ${extvars} << _EOF
+variables:
+  var1: "extvar1"
+  varx: "exttest"
+  var2: "{{@@ var1 @@}} var2"
+  var3: "{{@@ var2 @@}} var3"
+  var4: "{{@@ dvar4 @@}}"
+dynvariables:
+  dvar1: "echo extdvar1"
+  dvar2: "{{@@ dvar1 @@}} dvar2"
+  dvar3: "{{@@ dvar2 @@}} dvar3"
+  dvar4: "echo {{@@ var3 @@}}"
+_EOF
+
+# create the dotfile
+echo "var3: {{@@ var3 @@}}" > ${tmps}/dotfiles/abc
+echo "dvar3: {{@@ dvar3 @@}}" >> ${tmps}/dotfiles/abc
+echo "var4: {{@@ var4 @@}}" >> ${tmps}/dotfiles/abc
+echo "dvar4: {{@@ dvar4 @@}}" >> ${tmps}/dotfiles/abc
+echo "varx: {{@@ varx @@}}" >> ${tmps}/dotfiles/abc
+
+#cat ${tmps}/dotfiles/abc
+
+# install
+cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V
+
+#cat ${tmpd}/abc
+
+grep '^var3: extvar1 var2 var3' ${tmpd}/abc >/dev/null
+grep '^dvar3: extdvar1 dvar2 dvar3' ${tmpd}/abc >/dev/null
+grep '^var4: echo extvar1 var2 var3' ${tmpd}/abc >/dev/null
+grep '^dvar4: extvar1 var2 var3' ${tmpd}/abc >/dev/null
+grep '^varx: profvarx' ${tmpd}/abc >/dev/null
+
+## CLEANING
+rm -rf ${tmps} ${tmpd}
+
+echo "OK"
+exit 0