diff --git a/README.md b/README.md index 9f2390a..91911af 100644 --- a/README.md +++ b/README.md @@ -403,13 +403,19 @@ when xinitrc is installed. ## Use transformations -Transformations are used to transform a dotfile before it is -installed. These are executed before the dotfile is installed to transform the source. +There are two types of transformations available: -Transformation commands have two arguments: +* **read transformations** ([Config](#config) key *trans*): used to transform dotfiles before they are installed + (used for commands `install` and `compare`). They have two arguments: -* **{0}** will be replaced with the dotfile to process -* **{1}** will be replaced with a temporary file to store the result of the transformation + * **{0}** will be replaced with the dotfile to process + * **{1}** will be replaced with a temporary file to store the result of the transformation + +* **write transformations** ([Config](#config) key *trans_write**): used to transform files before updating a dotfile + (used for command `update`). They have two arguments + + * **{0}** will be replaced with the file path to update the dotfile with + * **{1}** will be replaced with a temporary file to store the result of the transformation A typical use-case for transformations is when the dotfile needs to be stored encrypted. @@ -420,42 +426,19 @@ dotfiles: f_secret: dst: ~/.secret src: secret - trans: - - gpg + trans: gpg trans: gpg: gpg2 -q --for-your-eyes-only --no-tty -d {0} > {1} ``` -The above config allows to store the dotfile `~/.secret` encrypted in the *dotfiles* +The above config allows to store the dotfile `~/.secret` encrypted in the *dotpath* directory and uses gpg to decrypt it when `install` is run. -Here's how to deploy the above solution: - -* import the clear dotfile (what creates the correct entries in the config file) - -```bash -$ dotdrop import ~/.secret -``` - -* encrypt the original dotfile - -```bash -$ ~/.secret -``` - -* overwrite the dotfile with the encrypted version - -```bash -$ cp dotfiles/secret -``` - -* edit the config file and add the transformation to the dotfile - (as shown in the example above) - -* commit and push the changes +See the wiki page for a walkthrough on how to deploy this solution as well +as more information on transformations: +[wiki transformation page](https://github.com/deadc0de6/dotdrop/wiki/transformations). Note that transformations cannot be used if the dotfiles is to be linked (`link: true`). -Also `compare` won't work on dotfiles using transformations. ## Update dotdrop @@ -482,17 +465,17 @@ $ sudo pip3 install dotdrop --upgrade Dotfiles managed by dotdrop can be updated using the `update` command. When updating, only dotfiles that have differences with the stored version are updated. A confirmation is requested from the user before any overwrite/update unless the -`--force` switch is used. +`-f --force` switch is used. Either provide the path of the file containing the new version of the dotfile or -provide the dotfile key to update (as found in the config file) along with the `--key` switch. -When using the `--key` switch and no key is provided, all dotfiles for that profile are updated. +provide the dotfile key to update (as found in the config file) along with the `-k --key` switch. +When using the `-k --key` switch and no key is provided, all dotfiles for that profile are updated. ```bash # update by path $ dotdrop update ~/.vimrc -# update by key -$ dotdrop update f_vimrc +# update by key with the --key switch +$ dotdrop update --key f_vimrc ``` There are two cases when updating a dotfile: @@ -585,7 +568,8 @@ the following entries: * `link`: if true dotdrop will create a symlink instead of copying (default *false*). * `cmpignore`: list of pattern to ignore when comparing (enclose in quotes when using wildcards). * `actions`: list of action keys that need to be defined in the **actions** entry below. - * `trans`: list of transformation keys that need to be defined in the **trans** entry below. + * `trans`: transformation key to apply when installing this dotfile (must be defined in the **trans** entry below). + * `trans_write`: transformation key to apply when updating this dotfile (must be defined in the **trans_write** entry below). * `ignoreempty`: if true empty template will not be deployed (defaults to the value of `ignoreempty` above) ```yaml @@ -599,8 +583,8 @@ the following entries: - "" actions: - - trans: - - + trans: + trans_write: ``` * **profiles** entry: a list of profiles with the different dotfiles that @@ -632,6 +616,12 @@ the following entries: : ``` +* **trans_write** entry (optional): a list of write transformations (see [Use transformations](#use-transformations)) + +``` + : +``` + * **variables** entry (optional): a list of template variables (see [Variables](#variables)) ``` diff --git a/dotdrop/action.py b/dotdrop/action.py index 391152e..414b420 100644 --- a/dotdrop/action.py +++ b/dotdrop/action.py @@ -60,15 +60,15 @@ class Transform(Cmd): """execute transformation with {0} and {1} where {0} is the file to transform and {1} is the result file""" - if os.path.exists(arg1): - msg = 'transformation destination exists: {}' - self.log.warn(msg.format(arg1)) - return False ret = 1 cmd = self.action.format(arg0, arg1) + if os.path.exists(arg1): + msg = 'transformation \"{}\": destination exists: {}' + self.log.warn(msg.format(cmd, arg1)) + return False self.log.sub('transforming with \"{}\"'.format(cmd)) try: ret = subprocess.call(cmd, shell=True) except KeyboardInterrupt: - self.log.warn('action interrupted') + self.log.warn('transformation interrupted') return ret == 0 diff --git a/dotdrop/config.py b/dotdrop/config.py index ca56bc3..b039cc8 100644 --- a/dotdrop/config.py +++ b/dotdrop/config.py @@ -42,7 +42,8 @@ class Cfg: key_actions_post = 'post' # transformations keys - key_trans = 'trans' + key_trans_r = 'trans' + key_trans_w = 'trans_write' # template variables key_variables = 'variables' @@ -57,7 +58,8 @@ class Cfg: key_dotfiles_noempty = 'ignoreempty' key_dotfiles_cmpignore = 'cmpignore' key_dotfiles_actions = 'actions' - key_dotfiles_trans = 'trans' + key_dotfiles_trans_r = 'trans' + key_dotfiles_trans_w = 'trans_write' # profiles keys key_profiles = 'profiles' @@ -101,9 +103,13 @@ class Cfg: # NOT linked inside the yaml dict (self.content) self.actions = {} - # dict of all transformation objects by trans key + # dict of all read transformation objects by trans key # NOT linked inside the yaml dict (self.content) - self.trans = {} + self.trans_r = {} + + # dict of all write transformation objects by trans key + # NOT linked inside the yaml dict (self.content) + self.trans_w = {} # represents all dotfiles per profile by profile key # NOT linked inside the yaml dict (self.content) @@ -174,11 +180,17 @@ class Cfg: self.actions[self.key_actions_post] = {} self.actions[self.key_actions_post][k] = Action(k, v) - # parse all transformations - if self.key_trans in self.content: - if self.content[self.key_trans] is not None: - for k, v in self.content[self.key_trans].items(): - self.trans[k] = Transform(k, v) + # parse read transformations + if self.key_trans_r in self.content: + if self.content[self.key_trans_r] is not None: + for k, v in self.content[self.key_trans_r].items(): + self.trans_r[k] = Transform(k, v) + + # parse write transformations + if self.key_trans_w in self.content: + if self.content[self.key_trans_w] is not None: + for k, v in self.content[self.key_trans_w].items(): + self.trans_w[k] = Transform(k, v) # parse the profiles self.lnk_profiles = self.content[self.key_profiles] @@ -213,20 +225,60 @@ class Cfg: itsactions = v[self.key_dotfiles_actions] if \ self.key_dotfiles_actions in v else [] actions = self._parse_actions(itsactions) - itstrans = v[self.key_dotfiles_trans] if \ - self.key_dotfiles_trans in v else [] - trans = self._parse_trans(itstrans) - if len(trans) > 0 and link: + + # parse read transformation + itstrans_r = v[self.key_dotfiles_trans_r] if \ + self.key_dotfiles_trans_r in v else None + trans_r = None + if itstrans_r: + if type(itstrans_r) is list: + msg = 'One transformation allowed per dotfile' + msg += ', error on dotfile \"{}\"' + self.log.err(msg.format(k)) + msg = 'Please modify your config file to: \"trans: {}\"' + self.log.err(msg.format(itstrans_r[0])) + return False + trans_r = self._parse_trans(itstrans_r, read=True) + if not trans_r: + msg = 'unknown trans \"{}\" for \"{}\"' + self.log.err(msg.format(itstrans_r, k)) + return False + + # parse write transformation + itstrans_w = v[self.key_dotfiles_trans_w] if \ + self.key_dotfiles_trans_w in v else None + trans_w = None + if itstrans_w: + if type(itstrans_w) is list: + msg = 'One write transformation allowed per dotfile' + msg += ', error on dotfile \"{}\"' + self.log.err(msg.format(k)) + msg = 'Please modify your config file: \"trans_write: {}\"' + self.log.err(msg.format(itstrans_w[0])) + return False + trans_w = self._parse_trans(itstrans_w, read=False) + if not trans_w: + msg = 'unknown trans_write \"{}\" for \"{}\"' + self.log.err(msg.format(itstrans_w, k)) + return False + + # disable transformation when link is true + if link and (trans_r or trans_w): msg = 'transformations disabled for \"{}\"'.format(dst) msg += ' because link is True' self.log.warn(msg) - trans = [] + trans_r = None + trans_w = None + + # parse ignore pattern ignores = v[self.key_dotfiles_cmpignore] if \ self.key_dotfiles_cmpignore in v else [] + + # create new dotfile self.dotfiles[k] = Dotfile(k, dst, src, link=link, actions=actions, - trans=trans, cmpignore=ignores, - noempty=noempty) + trans_r=trans_r, trans_w=trans_w, + cmpignore=ignores, noempty=noempty) # assign dotfiles to each profile for k, v in self.lnk_profiles.items(): @@ -315,16 +367,14 @@ class Cfg: res[key].append(action) return res - def _parse_trans(self, entries): - """parse transformations specified for an element - where entries are the ones defined for this dotfile""" - res = [] - for entry in entries: - if entry not in self.trans.keys(): - self.log.warn('unknown trans \"{}\"'.format(entry)) - continue - res.append(self.trans[entry]) - return res + def _parse_trans(self, trans, read=True): + """parse transformation key specified for a dotfile""" + transformations = self.trans_r + if not read: + transformations = self.trans_w + if trans not in transformations.keys(): + return None + return transformations[trans] def _complete_settings(self): """set settings defaults if not present""" diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index b7ff39f..13b1cac 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -113,7 +113,7 @@ def cmd_install(opts, conf, temporary=False, keys=[]): else: src = dotfile.src tmp = None - if dotfile.trans: + if dotfile.trans_r: tmp = apply_trans(opts, dotfile) if not tmp: continue @@ -173,7 +173,7 @@ def cmd_compare(opts, conf, tmp, focus=[], ignore=[]): LOG.emph('\"{}\" does not exist on local\n'.format(dotfile.dst)) tmpsrc = None - if dotfile.trans: + if dotfile.trans_r: # apply transformation tmpsrc = apply_trans(opts, dotfile) if not tmpsrc: @@ -387,22 +387,18 @@ def _select(selections, dotfiles): def apply_trans(opts, dotfile): - """apply the transformation to the dotfile + """apply the read transformation to the dotfile return None if fails and new source if succeed""" src = dotfile.src new_src = '{}.{}'.format(src, TRANS_SUFFIX) - err = False - for trans in dotfile.trans: - if opts['debug']: - LOG.dbg('executing transformation {}'.format(trans)) - s = os.path.join(opts['dotpath'], src) - temp = os.path.join(opts['dotpath'], new_src) - if not trans.transform(s, temp): - msg = 'transformation \"{}\" failed for {}' - LOG.err(msg.format(trans.key, dotfile.key)) - err = True - break - if err: + trans = dotfile.trans_r + if opts['debug']: + LOG.dbg('executing transformation {}'.format(trans)) + s = os.path.join(opts['dotpath'], src) + temp = os.path.join(opts['dotpath'], new_src) + if not trans.transform(s, temp): + msg = 'transformation \"{}\" failed for {}' + LOG.err(msg.format(trans.key, dotfile.key)) if new_src and os.path.exists(new_src): remove(new_src) return None @@ -422,7 +418,7 @@ def main(): try: conf = Cfg(os.path.expanduser(args['--cfg'])) except ValueError as e: - LOG.err('error: {}'.format(str(e))) + LOG.err('Config format error: {}'.format(str(e))) return False opts = conf.get_settings() diff --git a/dotdrop/dotfile.py b/dotdrop/dotfile.py index b45dc56..45e14ee 100644 --- a/dotdrop/dotfile.py +++ b/dotdrop/dotfile.py @@ -9,9 +9,8 @@ represents a dotfile in dotdrop class Dotfile: def __init__(self, key, dst, src, - actions={}, trans=[], - link=False, cmpignore=[], - noempty=False): + actions={}, trans_r=None, trans_w=None, + link=False, cmpignore=[], noempty=False): # key of dotfile in the config self.key = key # path where to install this dotfile @@ -22,8 +21,10 @@ class Dotfile: self.link = link # list of actions self.actions = actions - # list of transformations - self.trans = trans + # read transformation + self.trans_r = trans_r + # write transformation + self.trans_w = trans_w # pattern to ignore when comparing self.cmpignore = cmpignore # do not deploy empty file diff --git a/dotdrop/updater.py b/dotdrop/updater.py index b63e507..448695a 100644 --- a/dotdrop/updater.py +++ b/dotdrop/updater.py @@ -56,12 +56,39 @@ class Updater: def _update(self, path, dotfile): """update dotfile from file pointed by path""" + ret = False + new_path = None left = os.path.expanduser(path) right = os.path.join(self.conf.abs_dotpath(self.dotpath), dotfile.src) right = os.path.expanduser(right) - if os.path.isdir(path): - return self._handle_dir(left, right) - return self._handle_file(left, right) + if dotfile.trans_w: + # apply write transformation if any + new_path = self._apply_trans_w(path, dotfile) + if not new_path: + return False + left = new_path + if os.path.isdir(left): + ret = self._handle_dir(left, right) + else: + ret = self._handle_file(left, right) + # clean temporary files + if new_path and os.path.exists(new_path): + utils.remove(new_path) + return ret + + def _apply_trans_w(self, path, dotfile): + """apply write transformation to dotfile""" + trans = dotfile.trans_w + if self.debug: + self.log.dbg('executing write transformation {}'.format(trans)) + tmp = utils.get_unique_tmp_name() + if not trans.transform(path, tmp): + msg = 'transformation \"{}\" failed for {}' + self.log.err(msg.format(trans.key, dotfile.key)) + if os.path.exists(tmp): + utils.remove(tmp) + return None + return tmp def _normalize(self, path): """normalize the path to match dotfile""" diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 6e1c606..b949a75 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -8,6 +8,7 @@ utilities import subprocess import tempfile import os +import uuid import shlex from shutil import rmtree @@ -61,6 +62,12 @@ def get_tmpfile(): return path +def get_unique_tmp_name(): + """get a unique file name (not created)""" + unique = str(uuid.uuid4()) + return os.path.join(tempfile.gettempdir(), unique) + + def remove(path): """remove a file/directory/symlink""" if not os.path.lexists(path): diff --git a/tests-ng/transformations.sh b/tests-ng/transformations.sh index 7fbd005..8c729db 100755 --- a/tests-ng/transformations.sh +++ b/tests-ng/transformations.sh @@ -9,6 +9,7 @@ # exit on first error set -e +#set -v # all this crap to get current path rl="readlink -f" @@ -59,10 +60,16 @@ cfg="${tmps}/config.yaml" # token token="test-base64" +tokend="compressed archive" +touched="touched" cat > ${cfg} << _EOF trans: base64: cat {0} | base64 -d > {1} + uncompress: mkdir -p {1} && tar -xf {0} -C {1} +trans_write: + base64: cat {0} | base64 > {1} + compress: tar -cf {1} -C {0} . config: backup: true create: true @@ -74,43 +81,117 @@ dotfiles: f_abc: dst: ${tmpd}/abc src: abc - trans: - - base64 + trans: base64 + trans_write: base64 + d_ghi: + dst: ${tmpd}/ghi + src: ghi + trans: uncompress + trans_write: compress profiles: p1: dotfiles: - f_abc - f_def + - d_ghi _EOF cat ${cfg} -# create the dotfile +# create the base64 dotfile tmpf=`mktemp` -echo ${token} > ${tmpf} +echo ${token} > ${tmpf} cat ${tmpf} | base64 > ${tmps}/dotfiles/abc rm -f ${tmpf} +# create the canary dotfile echo 'marker' > ${tmps}/dotfiles/def -# install -cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -b +# create the compressed dotfile +tmpx=`mktemp -d` +mkdir -p ${tmpx}/{a,b,c} +mkdir -p ${tmpx}/a/{dir1,dir2} +# ambiguous redirect ?? +#echo ${tokend} > ${tmpd}/{a,b,c}/somefile +echo ${tokend} > ${tmpx}/a/somefile +echo ${tokend} > ${tmpx}/b/somefile +echo ${tokend} > ${tmpx}/c/somefile +echo ${tokend} > ${tmpx}/a/dir1/otherfile +tar -cf ${tmps}/dotfiles/ghi -C ${tmpx} . +rm -rf ${tmpx} +tar -tf ${tmps}/dotfiles/ghi -# checks +########################### +# test install and compare +########################### + +# install +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -b -V + +# check canary dotfile +[ ! -e ${tmpd}/def ] && exit 1 + +# check base64 dotfile [ ! -e ${tmpd}/abc ] && exit 1 content=`cat ${tmpd}/abc` [ "${content}" != "${token}" ] && exit 1 +# check directory dotfile +[ ! -e ${tmpd}/ghi/a/dir1/otherfile ] && exit 1 +content=`cat ${tmpd}/ghi/a/somefile` +[ "${content}" != "${tokend}" ] && exit 1 +content=`cat ${tmpd}/ghi/a/dir1/otherfile` +[ "${content}" != "${tokend}" ] && exit 1 + # compare -cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 -b +cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 -b -V [ "$?" != "0" ] && exit 1 -# change file -echo 'touched' >> ${tmpd}/abc +# change base64 deployed file +echo ${touched} > ${tmpd}/abc set +e -cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 -b +cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 -b -V [ "$?" != "1" ] && exit 1 set -e +# change uncompressed deployed dotfile +echo ${touched} > ${tmpd}/ghi/a/somefile +set +e +cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 -b -V +[ "$?" != "1" ] && exit 1 +set -e + +########################### +# test update +########################### + +# update single file +cd ${ddpath} | ${bin} update -f -k -c ${cfg} -p p1 -b -V f_abc +[ "$?" != "0" ] && exit 1 + +# test updated file +[ ! -e ${tmps}/dotfiles/abc ] && exit 1 +content=`cat ${tmps}/dotfiles/abc` +bcontent=`echo ${touched} | base64` +[ "${content}" != "${bcontent}" ] && exit 1 + +# update directory +echo ${touched} > ${tmpd}/ghi/b/newfile +rm -r ${tmpd}/ghi/c +cd ${ddpath} | ${bin} update -f -k -c ${cfg} -p p1 -b -V d_ghi +[ "$?" != "0" ] && exit 1 + +# test updated directory +tar -tf ${tmps}/dotfiles/ghi | grep './b/newfile' +tar -tf ${tmps}/dotfiles/ghi | grep './a/dir1/otherfile' + +tmpy=`mktemp -d` +tar -xf ${tmps}/dotfiles/ghi -C ${tmpy} +content=`cat ${tmpy}/a/somefile` +[ "${content}" != "${touched}" ] && exit 1 + +# check canary dotfile +[ ! -e ${tmps}/dotfiles/def ] && exit 1 + ## CLEANING rm -rf ${tmps} ${tmpd} diff --git a/tests/test_install.py b/tests/test_install.py index acad9b5..5ebef41 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -55,10 +55,8 @@ exec bspwm f.write(' actions:\n') for action in d.actions: f.write(' - {}\n'.format(action.key)) - if len(d.trans) > 0: - f.write(' trans:\n') - for action in d.trans: - f.write(' - {}\n'.format(action.key)) + if d.trans_r: + f.write(' trans: {}\n'.format(d.trans_r.key)) f.write('profiles:\n') f.write(' {}:\n'.format(profile)) for d in dotfiles: @@ -154,7 +152,7 @@ exec bspwm tr = Action('testtrans', cmd) f9, c9 = create_random_file(tmp, content=trans1) dst9 = os.path.join(dst, get_string(6)) - d9 = Dotfile(get_string(6), dst9, os.path.basename(f9), trans=[tr]) + d9 = Dotfile(get_string(6), dst9, os.path.basename(f9), trans_r=tr) # to test template f10, _ = create_random_file(tmp, content='{{@@ profile @@}}')