Merge pull request #295 from MarcelRobitaille/feat-update-ignore-missing

Option to ignore missing files update and compare

.gitignore

@@ -21,3 +21,5 @@ stage/
 
 # Source archive packed by `snapcraft cleanbuild` before pushing to the LXD container
 /packages/*_source.tar.bz2
+
+venv

docs/config-details.md

@@ -645,3 +645,21 @@ dotfiles:
     trans_read: r_echo_var
     trans_write: w_echo_var
 ```
+
+## Ignoring missing files
+
+The [ignore missing files setting](usage.md#ignoring-missing-files)
+can be configured globally or on a specific dotfile.
+
+To configure globally, place the following in `config.yaml`:
+```yaml
+config:
+  ignore_missing_in_dotdrop: True
+```
+
+To configure per dotfile:
+```yaml
+dotfiles:
+  f_abc:
+    ignore_missing_in_dotdrop: True
+```

docs/usage.md

@@ -108,7 +108,8 @@ It is also possible to install all dotfiles for a specific profile
 in a temporary directory in order to manually compare them with
 the local version by using `install` and the `-t` switch.
 
-For more options, see the usage with `dotdrop --help`
+For more options, see the usage with `dotdrop --help`.
+See also [ignoring missing files](#ignoring-missing-files).
 
 ## List profiles
 
@@ -218,6 +219,8 @@ Installed to tmp /tmp/dotdrop-6ajz7565
 $ diff ~/.vimrc /tmp/dotdrop-6ajz7565/home/user/.vimrc
 ```
 
+See also [ignoring missing files](#ignoring-missing-files).
+
 ## Remove dotfiles
 
 The command `remove` allows to stop managing a specific dotfile with
@@ -287,3 +290,19 @@ export DOTDROP_WORKDIR="/tmp/dotdrop-workdir"
 ```bash
 export DOTDROP_WORKERS="10"
 ```
+
+## Ignoring missing files
+
+Sometimes, it is nice to have [`update`](#update-dotfiles) not copy all the files in the installed directory
+or [`compare`](#compare-dotfiles) diff them.
+
+For example,
+maybe you only want to include a single configuration file in your repository
+and don't want to include other files the program uses,
+such as a cache.
+Maybe you only want to change one file and don't want the others cluttering your repository.
+Maybe the program changes these files quite often and creates unnecessary diffs in your dotfiles.
+
+In these cases, you can use the `ingore-missing` option.
+This option is available as a flag (`--ignore-missing` or `-z`) to the `update` and `compare` commands,
+or [as a configuration option either globally or on a specific dotfile](config-details.md#ignoring-missing-files).

dotdrop/cfg_yaml.py

@@ -59,6 +59,7 @@ class CfgYaml:
     key_dotfile_noempty = 'ignoreempty'
     key_dotfile_template = 'template'
     key_dotfile_chmod = 'chmod'
+    key_dotfile_ignore_missing = 'ignore_missing_in_dotdrop'
 
     # profile
     key_profile_dotfiles = 'dotfiles'

dotdrop/comparator.py

@@ -16,7 +16,8 @@ from dotdrop.utils import must_ignore, uniq_list, diff, \
 
 class Comparator:
 
-    def __init__(self, diff_cmd='', debug=False):
+    def __init__(self, diff_cmd='', debug=False,
+                 ignore_missing_in_dotdrop=False):
         """constructor
         @diff_cmd: diff command to use
         @debug: enable debug
@@ -24,104 +25,139 @@ class Comparator:
         self.diff_cmd = diff_cmd
         self.debug = debug
         self.log = Logger()
+        self.ignore_missing_in_dotdrop = ignore_missing_in_dotdrop
 
-    def compare(self, left, right, ignore=[]):
+    def compare(self, local_path, deployed_path, ignore=[]):
-        """diff left (dotdrop dotfile) and right (deployed file)"""
+        """diff local_path (dotdrop dotfile) and
-        left = os.path.expanduser(left)
+        deployed_path (destination file)"""
-        right = os.path.expanduser(right)
+        local_path = os.path.expanduser(local_path)
+        deployed_path = os.path.expanduser(deployed_path)
         if self.debug:
-            self.log.dbg('comparing {} and {}'.format(left, right))
+            self.log.dbg('comparing {} and {}'.format(
+                local_path,
+                deployed_path,
+            ))
             self.log.dbg('ignore pattern(s): {}'.format(ignore))
 
         # test type of file
-        if os.path.isdir(left) and not os.path.isdir(right):
+        if os.path.isdir(local_path) and not os.path.isdir(deployed_path):
-            return '\"{}\" is a dir while \"{}\" is a file\n'.format(left,
+            return '\"{}\" is a dir while \"{}\" is a file\n'.format(
-                                                                     right)
+                local_path,
-        if not os.path.isdir(left) and os.path.isdir(right):
+                deployed_path,
-            return '\"{}\" is a file while \"{}\" is a dir\n'.format(left,
+            )
-                                                                     right)
+        if not os.path.isdir(local_path) and os.path.isdir(deployed_path):
+            return '\"{}\" is a file while \"{}\" is a dir\n'.format(
+                local_path,
+                deployed_path,
+            )
 
         # test content
-        if not os.path.isdir(left):
+        if not os.path.isdir(local_path):
             if self.debug:
-                self.log.dbg('{} is a file'.format(left))
+                self.log.dbg('{} is a file'.format(local_path))
             if self.debug:
                 self.log.dbg('is file')
-            ret = self._comp_file(left, right, ignore)
+            ret = self._comp_file(local_path, deployed_path, ignore)
             if not ret:
-                ret = self._comp_mode(left, right)
+                ret = self._comp_mode(local_path, deployed_path)
             return ret
 
         if self.debug:
-            self.log.dbg('{} is a directory'.format(left))
+            self.log.dbg('{} is a directory'.format(local_path))
 
-        ret = self._comp_dir(left, right, ignore)
+        ret = self._comp_dir(local_path, deployed_path, ignore)
         if not ret:
-            ret = self._comp_mode(left, right)
+            ret = self._comp_mode(local_path, deployed_path)
         return ret
 
-    def _comp_mode(self, left, right):
+    def _comp_mode(self, local_path, deployed_path):
         """compare mode"""
-        left_mode = get_file_perm(left)
+        local_mode = get_file_perm(local_path)
-        right_mode = get_file_perm(right)
+        deployed_mode = get_file_perm(deployed_path)
-        if left_mode == right_mode:
+        if local_mode == deployed_mode:
             return ''
         if self.debug:
             msg = 'mode differ {} ({:o}) and {} ({:o})'
-            self.log.dbg(msg.format(left, left_mode, right, right_mode))
+            self.log.dbg(msg.format(local_path, local_mode, deployed_path,
+                                    deployed_mode))
         ret = 'modes differ for {} ({:o}) vs {:o}\n'
-        return ret.format(right, right_mode, left_mode)
+        return ret.format(deployed_path, deployed_mode, local_mode)
 
-    def _comp_file(self, left, right, ignore):
+    def _comp_file(self, local_path, deployed_path, ignore):
         """compare a file"""
         if self.debug:
-            self.log.dbg('compare file {} with {}'.format(left, right))
+            self.log.dbg('compare file {} with {}'.format(
-        if must_ignore([left, right], ignore, debug=self.debug):
+                local_path,
+                deployed_path,
+            ))
+        if (self.ignore_missing_in_dotdrop and not
+                os.path.exists(local_path)) \
+                or must_ignore([local_path, deployed_path], ignore,
+                               debug=self.debug):
             if self.debug:
-                self.log.dbg('ignoring diff {} and {}'.format(left, right))
+                self.log.dbg('ignoring diff {} and {}'.format(
+                    local_path,
+                    deployed_path,
+                ))
             return ''
-        return self._diff(left, right)
+        return self._diff(local_path, deployed_path)
 
-    def _comp_dir(self, left, right, ignore):
+    def _comp_dir(self, local_path, deployed_path, ignore):
         """compare a directory"""
         if self.debug:
-            self.log.dbg('compare directory {} with {}'.format(left, right))
+            self.log.dbg('compare directory {} with {}'.format(
-        if not os.path.exists(right):
+                local_path,
+                deployed_path,
+            ))
+        if not os.path.exists(deployed_path):
             return ''
-        if must_ignore([left, right], ignore, debug=self.debug):
+        if (self.ignore_missing_in_dotdrop and not
+                os.path.exists(local_path)) \
+                or must_ignore([local_path, deployed_path], ignore,
+                               debug=self.debug):
             if self.debug:
-                self.log.dbg('ignoring diff {} and {}'.format(left, right))
+                self.log.dbg('ignoring diff {} and {}'.format(
+                    local_path,
+                    deployed_path,
+                ))
             return ''
-        if not os.path.isdir(right):
+        if not os.path.isdir(deployed_path):
-            return '\"{}\" is a file\n'.format(right)
+            return '\"{}\" is a file\n'.format(deployed_path)
         if self.debug:
-            self.log.dbg('compare {} and {}'.format(left, right))
+            self.log.dbg('compare {} and {}'.format(local_path, deployed_path))
         ret = []
-        comp = filecmp.dircmp(left, right)
+        comp = filecmp.dircmp(local_path, deployed_path)
 
         # handle files only in deployed dir
         for i in comp.left_only:
-            if must_ignore([os.path.join(left, i)],
+            if self.ignore_missing_in_dotdrop:
+                continue
+            if must_ignore([os.path.join(local_path, i)],
                            ignore, debug=self.debug):
                 continue
             ret.append('=> \"{}\" does not exist on destination\n'.format(i))
 
         # handle files only in dotpath dir
         for i in comp.right_only:
-            if must_ignore([os.path.join(right, i)],
+            if must_ignore([os.path.join(deployed_path, i)],
                            ignore, debug=self.debug):
                 continue
+
+            if not self.ignore_missing_in_dotdrop:
                 ret.append('=> \"{}\" does not exist in dotdrop\n'.format(i))
 
-        # same left and right but different type
+        # same local_path and deployed_path but different type
         funny = comp.common_funny
         for i in funny:
-            lfile = os.path.join(left, i)
+            source_file = os.path.join(local_path, i)
-            rfile = os.path.join(right, i)
+            deployed_file = os.path.join(deployed_path, i)
-            if must_ignore([lfile, rfile],
+            if self.ignore_missing_in_dotdrop and \
+                    not os.path.exists(source_file):
+                continue
+            if must_ignore([source_file, deployed_file],
                            ignore, debug=self.debug):
                 continue
-            short = os.path.basename(lfile)
+            short = os.path.basename(source_file)
             # file vs dir
             ret.append('=> different type: \"{}\"\n'.format(short))
 
@@ -130,27 +166,29 @@ class Comparator:
         funny.extend(comp.funny_files)
         funny = uniq_list(funny)
         for i in funny:
-            lfile = os.path.join(left, i)
+            source_file = os.path.join(local_path, i)
-            rfile = os.path.join(right, i)
+            deployed_file = os.path.join(deployed_path, i)
-            if must_ignore([lfile, rfile],
+            if self.ignore_missing_in_dotdrop and \
+                    not os.path.exists(source_file):
+                continue
+            if must_ignore([source_file, deployed_file],
                            ignore, debug=self.debug):
                 continue
-            diff = self._diff(lfile, rfile, header=True)
+            ret.append(self._diff(source_file, deployed_file, header=True))
-            ret.append(diff)
 
         # recursively compare subdirs
         for i in comp.common_dirs:
-            subleft = os.path.join(left, i)
+            sublocal_path = os.path.join(local_path, i)
-            subright = os.path.join(right, i)
+            subdeployed_path = os.path.join(deployed_path, i)
-            ret.extend(self._comp_dir(subleft, subright, ignore))
+            ret.extend(self._comp_dir(sublocal_path, subdeployed_path, ignore))
 
         return ''.join(ret)
 
-    def _diff(self, left, right, header=False):
+    def _diff(self, local_path, deployed_path, header=False):
         """diff two files"""
-        out = diff(modified=left, original=right,
+        out = diff(modified=local_path, original=deployed_path,
                    diff_cmd=self.diff_cmd, debug=self.debug)
         if header:
-            lshort = os.path.basename(left)
+            lshort = os.path.basename(local_path)
             out = '=> diff \"{}\":\n{}'.format(lshort, out)
         return out

dotdrop/dotdrop.py

@@ -80,7 +80,8 @@ def _dotfile_update(o, path, key=False):
     updater = Updater(o.dotpath, o.variables, o.conf,
                       dry=o.dry, safe=o.safe, debug=o.debug,
                       ignore=o.update_ignore,
-                      showpatch=o.update_showpatch)
+                      showpatch=o.update_showpatch,
+                      ignore_missing_in_dotdrop=o.ignore_missing_in_dotdrop)
     if key:
         return updater.update_key(path)
     return updater.update_path(path)
@@ -92,12 +93,15 @@ def _dotfile_compare(o, dotfile, tmp):
     returns True if same
     """
     t = _get_templater(o)
+    ignore_missing_in_dotdrop = o.ignore_missing_in_dotdrop or \
+        dotfile.ignore_missing_in_dotdrop
     inst = Installer(create=o.create, backup=o.backup,
                      dry=o.dry, base=o.dotpath,
                      workdir=o.workdir, debug=o.debug,
                      backup_suffix=o.install_backup_suffix,
                      diff_cmd=o.diff_command)
-    comp = Comparator(diff_cmd=o.diff_command, debug=o.debug)
+    comp = Comparator(diff_cmd=o.diff_command, debug=o.debug,
+                      ignore_missing_in_dotdrop=ignore_missing_in_dotdrop)
 
     # add dotfile variables
     newvars = dotfile.get_dotfile_variables()

dotdrop/dotfile.py

@@ -22,7 +22,8 @@ class Dotfile(DictParser):
                  actions=[], trans_r=None, trans_w=None,
                  link=LinkTypes.NOLINK, noempty=False,
                  cmpignore=[], upignore=[],
-                 instignore=[], template=True, chmod=None):
+                 instignore=[], template=True, chmod=None,
+                 ignore_missing_in_dotdrop=False):
         """
         constructor
         @key: dotfile key
@@ -52,6 +53,7 @@ class Dotfile(DictParser):
         self.instignore = instignore
         self.template = template
         self.chmod = chmod
+        self.ignore_missing_in_dotdrop = ignore_missing_in_dotdrop
 
         if self.link != LinkTypes.NOLINK and \
                 (

dotdrop/options.py

@@ -43,7 +43,7 @@ OPT_LINK = {
     LinkTypes.LINK.name.lower(): LinkTypes.LINK,
     LinkTypes.LINK_CHILDREN.name.lower(): LinkTypes.LINK_CHILDREN}
 
-BANNER = """     _       _      _
+BANNER = r"""     _       _      _
   __| | ___ | |_ __| |_ __ ___  _ __
  / _` |/ _ \| __/ _` | '__/ _ \| '_ |
  \__,_|\___/ \__\__,_|_|  \___/| .__/  v{}
@@ -57,9 +57,9 @@ Usage:
                                 [-w <nb>] [<key>...]
   dotdrop import    [-Vbdfm]    [-c <path>] [-p <profile>] [-s <path>]
                                 [-l <link>] [-i <pattern>...] <path>...
-  dotdrop compare   [-LVb]      [-c <path>] [-p <profile>]
+  dotdrop compare   [-LVbz]     [-c <path>] [-p <profile>]
                                 [-w <nb>] [-C <file>...] [-i <pattern>...]
-  dotdrop update    [-VbfdkP]   [-c <path>] [-p <profile>]
+  dotdrop update    [-VbfdkPz]  [-c <path>] [-p <profile>]
                                 [-w <nb>] [-i <pattern>...] [<path>...]
   dotdrop remove    [-Vbfdk]    [-c <path>] [-p <profile>] [<path>...]
   dotdrop files     [-VbTG]     [-c <path>] [-p <profile>]
@@ -73,6 +73,7 @@ Options:
   -b --no-banner          Do not display the banner.
   -c --cfg=<path>         Path to the config.
   -C --file=<path>        Path of dotfile to compare.
+  -z --ignore-missing     Ignore files in installed folders that are missing.
   -d --dry                Dry run.
   -D --showdiff           Show a diff before overwriting.
   -f --force              Do not ask user confirmation for anything.
@@ -264,6 +265,8 @@ class Options(AttrMonitor):
         self.compare_ignore.append('*{}'.format(self.install_backup_suffix))
         self.compare_ignore = uniq_list(self.compare_ignore)
         self.compare_fileonly = self.args['--file-only']
+        self.ignore_missing_in_dotdrop = self.ignore_missing_in_dotdrop or \
+            self.args['--ignore-missing']
 
         # "import" specifics
         self.import_path = self.args['<path>']

dotdrop/settings.py

@@ -41,6 +41,7 @@ class Settings(DictParser):
     key_filter_file = 'filter_file'
     key_diff_command = 'diff_command'
     key_template_dotfile_default = 'template_dotfile_default'
+    key_ignore_missing_in_dotdrop = 'ignore_missing_in_dotdrop'
 
     # import keys
     key_import_actions = 'import_actions'
@@ -57,7 +58,8 @@ class Settings(DictParser):
                  workdir='~/.config/dotdrop', showdiff=False,
                  minversion=None, func_file=[], filter_file=[],
                  diff_command='diff -r -u {0} {1}',
-                 template_dotfile_default=True):
+                 template_dotfile_default=True,
+                 ignore_missing_in_dotdrop=False):
         self.backup = backup
         self.banner = banner
         self.create = create
@@ -84,6 +86,7 @@ class Settings(DictParser):
         self.filter_file = filter_file
         self.diff_command = diff_command
         self.template_dotfile_default = template_dotfile_default
+        self.ignore_missing_in_dotdrop = ignore_missing_in_dotdrop
 
     def _serialize_seq(self, name, dic):
         """serialize attribute 'name' into 'dic'"""
@@ -107,6 +110,7 @@ class Settings(DictParser):
             self.key_minversion: self.minversion,
             self.key_diff_command: self.diff_command,
             self.key_template_dotfile_default: self.template_dotfile_default,
+            self.key_ignore_missing_in_dotdrop: self.ignore_missing_in_dotdrop,
         }
         self._serialize_seq(self.key_default_actions, dic)
         self._serialize_seq(self.key_import_actions, dic)

dotdrop/updater.py

@@ -25,7 +25,8 @@ class Updater:
 
     def __init__(self, dotpath, variables, conf,
                  dry=False, safe=True, debug=False,
-                 ignore=[], showpatch=False):
+                 ignore=[], showpatch=False,
+                 ignore_missing_in_dotdrop=False):
         """constructor
         @dotpath: path where dotfiles are stored
         @variables: dictionary of variables for the templates
@@ -43,7 +44,9 @@ class Updater:
         self.safe = safe
         self.debug = debug
         self.ignore = ignore
+        self.ignores = None
         self.showpatch = showpatch
+        self.ignore_missing_in_dotdrop = ignore_missing_in_dotdrop
         self.templater = Templategen(variables=self.variables,
                                      base=self.dotpath,
                                      debug=self.debug)
@@ -92,49 +95,52 @@ class Updater:
         if self.debug:
             self.log.dbg('ignore pattern(s): {}'.format(self.ignores))
 
-        path = os.path.expanduser(path)
+        deployed_path = os.path.expanduser(path)
-        dtpath = os.path.join(self.dotpath, dotfile.src)
+        local_path = os.path.join(self.dotpath, dotfile.src)
-        dtpath = os.path.expanduser(dtpath)
+        local_path = os.path.expanduser(local_path)
 
-        if not os.path.exists(path):
+        if not os.path.exists(deployed_path):
             msg = '\"{}\" does not exist'
-            self.log.err(msg.format(path))
+            self.log.err(msg.format(deployed_path))
             return False
 
-        if not os.path.exists(dtpath):
+        if not os.path.exists(local_path):
             msg = '\"{}\" does not exist, import it first'
-            self.log.err(msg.format(dtpath))
+            self.log.err(msg.format(local_path))
             return False
 
-        if self._ignore([path, dtpath]):
+        ignore_missing_in_dotdrop = self.ignore_missing_in_dotdrop or \
+            dotfile.ignore_missing_in_dotdrop
+        if (ignore_missing_in_dotdrop and not os.path.exists(local_path)) or \
+                self._ignore([deployed_path, local_path]):
             self.log.sub('\"{}\" ignored'.format(dotfile.key))
             return True
         # apply write transformation if any
-        new_path = self._apply_trans_w(path, dotfile)
+        new_path = self._apply_trans_w(deployed_path, dotfile)
         if not new_path:
             return False
 
         # save current rights
-        fsmode = get_file_perm(path)
+        deployed_mode = get_file_perm(deployed_path)
-        dfmode = get_file_perm(dtpath)
+        local_mode = get_file_perm(local_path)
 
         # handle the pointed file
         if os.path.isdir(new_path):
-            ret = self._handle_dir(new_path, dtpath)
+            ret = self._handle_dir(new_path, local_path, dotfile)
         else:
-            ret = self._handle_file(new_path, dtpath)
+            ret = self._handle_file(new_path, local_path, dotfile)
 
-        if fsmode != dfmode:
+        if deployed_mode != local_mode:
             # mirror rights
             if self.debug:
                 m = 'adopt mode {:o} for {}'
-                self.log.dbg(m.format(fsmode, dotfile.key))
+                self.log.dbg(m.format(deployed_mode, dotfile.key))
-            r = self.conf.update_dotfile(dotfile.key, fsmode)
+            r = self.conf.update_dotfile(dotfile.key, deployed_mode)
             if r:
                 ret = True
 
         # clean temporary files
-        if new_path != path and os.path.exists(new_path):
+        if new_path != deployed_path and os.path.exists(new_path):
             removepath(new_path, logger=self.log)
         return ret
 
@@ -203,64 +209,84 @@ class Updater:
         except OSError as e:
             self.log.err(e)
 
-    def _handle_file(self, path, dtpath, compare=True):
+    def _handle_file(self, deployed_path, local_path, dotfile, compare=True):
-        """sync path (deployed file) and dtpath (dotdrop dotfile path)"""
+        """sync path (deployed file) and local_path (dotdrop dotfile path)"""
-        if self._ignore([path, dtpath]):
+        if self._ignore([deployed_path, local_path]):
-            self.log.sub('\"{}\" ignored'.format(dtpath))
+            self.log.sub('\"{}\" ignored'.format(local_path))
             return True
         if self.debug:
-            self.log.dbg('update for file {} and {}'.format(path, dtpath))
+            self.log.dbg('update for file {} and {}'.format(
-        if self._is_template(dtpath):
+                deployed_path,
+                local_path,
+            ))
+        if self._is_template(local_path):
             # dotfile is a template
             if self.debug:
-                self.log.dbg('{} is a template'.format(dtpath))
+                self.log.dbg('{} is a template'.format(local_path))
             if self.showpatch:
                 try:
-                    self._show_patch(path, dtpath)
+                    self._show_patch(deployed_path, local_path)
                 except UndefinedException as e:
-                    msg = 'unable to show patch for {}: {}'.format(path, e)
+                    msg = 'unable to show patch for {}: {}'.format(
+                        deployed_path,
+                        e,
+                    )
                     self.log.warn(msg)
             return False
-        if compare and filecmp.cmp(path, dtpath, shallow=False) and \
+        if compare and \
-                self._same_rights(path, dtpath):
+                filecmp.cmp(deployed_path, local_path, shallow=False) and \
+                self._same_rights(deployed_path, local_path):
             # no difference
             if self.debug:
-                self.log.dbg('identical files: {} and {}'.format(path, dtpath))
+                self.log.dbg('identical files: {} and {}'.format(
+                    deployed_path,
+                    local_path,
+                ))
             return True
-        if not self._overwrite(path, dtpath):
+        if not self._overwrite(deployed_path, local_path):
             return False
         try:
             if self.dry:
-                self.log.dry('would cp {} {}'.format(path, dtpath))
+                self.log.dry('would cp {} {}'.format(
+                    deployed_path,
+                    local_path,
+                ))
             else:
                 if self.debug:
-                    self.log.dbg('cp {} {}'.format(path, dtpath))
+                    self.log.dbg('cp {} {}'.format(deployed_path, local_path))
-                shutil.copyfile(path, dtpath)
+                shutil.copyfile(deployed_path, local_path)
-                self._mirror_rights(path, dtpath)
+                self._mirror_rights(deployed_path, local_path)
-                self.log.sub('\"{}\" updated'.format(dtpath))
+                self.log.sub('\"{}\" updated'.format(local_path))
         except IOError as e:
-            self.log.warn('{} update failed, do manually: {}'.format(path, e))
+            self.log.warn('{} update failed, do manually: {}'.format(
+                deployed_path,
+                e
+            ))
             return False
         return True
 
-    def _handle_dir(self, path, dtpath):
+    def _handle_dir(self, deployed_path, local_path, dotfile):
-        """sync path (deployed dir) and dtpath (dotdrop dir path)"""
+        """sync path (local dir) and local_path (dotdrop dir path)"""
         if self.debug:
-            self.log.dbg('handle update for dir {} to {}'.format(path, dtpath))
+            self.log.dbg('handle update for dir {} to {}'.format(
+                deployed_path,
+                local_path,
+            ))
         # paths must be absolute (no tildes)
-        path = os.path.expanduser(path)
+        deployed_path = os.path.expanduser(deployed_path)
-        dtpath = os.path.expanduser(dtpath)
+        local_path = os.path.expanduser(local_path)
-        if self._ignore([path, dtpath]):
+
-            self.log.sub('\"{}\" ignored'.format(dtpath))
+        if self._ignore([deployed_path, local_path]):
+            self.log.sub('\"{}\" ignored'.format(local_path))
             return True
         # find the differences
-        diff = filecmp.dircmp(path, dtpath, ignore=None)
+        diff = filecmp.dircmp(deployed_path, local_path, ignore=None)
         # handle directories diff
-        ret = self._merge_dirs(diff)
+        ret = self._merge_dirs(diff, dotfile)
-        self._mirror_rights(path, dtpath)
+        self._mirror_rights(deployed_path, local_path)
         return ret
 
-    def _merge_dirs(self, diff):
+    def _merge_dirs(self, diff, dotfile):
         """Synchronize directories recursively."""
         left, right = diff.left, diff.right
         if self.debug:
@@ -268,6 +294,9 @@ class Updater:
         if self._ignore([left, right]):
             return True
 
+        ignore_missing_in_dotdrop = self.ignore_missing_in_dotdrop or \
+            dotfile.ignore_missing_in_dotdrop
+
         # create dirs that don't exist in dotdrop
         for toadd in diff.left_only:
             exist = os.path.join(left, toadd)
@@ -276,7 +305,8 @@ class Updater:
                 continue
             # match to dotdrop dotpath
             new = os.path.join(right, toadd)
-            if self._ignore([exist, new]):
+            if (ignore_missing_in_dotdrop and not os.path.exists(new)) or \
+                    self._ignore([exist, new]):
                 self.log.sub('\"{}\" ignored'.format(exist))
                 continue
             if self.dry:
@@ -329,14 +359,15 @@ class Updater:
         for f in fdiff:
             fleft = os.path.join(left, f)
             fright = os.path.join(right, f)
-            if self._ignore([fleft, fright]):
+            if (ignore_missing_in_dotdrop and not os.path.exists(fright)) or \
+                    self._ignore([fleft, fright]):
                 continue
             if self.dry:
                 self.log.dry('would cp {} {}'.format(fleft, fright))
                 continue
             if self.debug:
                 self.log.dbg('cp {} {}'.format(fleft, fright))
-            self._handle_file(fleft, fright, compare=False)
+            self._handle_file(fleft, fright, dotfile, compare=False)
 
         # copy files that don't exist in dotdrop
         for toadd in diff.left_only:
@@ -345,7 +376,8 @@ class Updater:
                 # ignore dirs, done above
                 continue
             new = os.path.join(right, toadd)
-            if self._ignore([exist, new]):
+            if (ignore_missing_in_dotdrop and not os.path.exists(new)) or \
+                    self._ignore([exist, new]):
                 continue
             if self.dry:
                 self.log.dry('would cp {} {}'.format(exist, new))
@@ -383,7 +415,7 @@ class Updater:
 
         # Recursively decent into common subdirectories.
         for subdir in diff.subdirs.values():
-            self._merge_dirs(subdir)
+            self._merge_dirs(subdir, dotfile)
 
         # Nothing more to do here.
         return True

tests-ng/compare-ignore-missing.sh

@@ -0,0 +1,158 @@
+#!/usr/bin/env bash
+# author: deadc0de6 (https://github.com/deadc0de6)
+# Copyright (c) 2017, deadc0de6
+#
+# test updates
+# returns 1 in case of error
+#
+
+# exit on first error
+set -e
+
+# all this crap to get current path
+rl="readlink -f"
+if ! ${rl} "${0}" >/dev/null 2>&1; then
+  rl="realpath"
+
+  if ! hash ${rl}; then
+    echo "\"${rl}\" not found !" && exit 1
+  fi
+fi
+cur=$(dirname "$(${rl} "${0}")")
+
+# dotdrop path can be pass as argument
+ddpath="${cur}/../"
+[ "${1}" != "" ] && ddpath="${1}"
+[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1
+
+export PYTHONPATH="${ddpath}:${PYTHONPATH}"
+bin="python3 -m dotdrop.dotdrop"
+hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true
+
+echo "dotdrop path: ${ddpath}"
+echo "pythonpath: ${PYTHONPATH}"
+
+# get the helpers
+source ${cur}/helpers
+
+echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)"
+
+################################################################
+# this is the test
+################################################################
+
+# dotdrop directory
+basedir=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d`
+echo "[+] dotdrop dir: ${basedir}"
+echo "[+] dotpath dir: ${basedir}/dotfiles"
+dt="${basedir}/dotfiles"
+mkdir -p ${dt}/folder
+touch ${dt}/folder/a
+
+# the dotfile to be imported
+tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d`
+
+# some files
+cp -r ${dt}/folder ${tmpd}/
+mkdir -p ${tmpd}/folder
+touch ${tmpd}/folder/b
+mkdir ${tmpd}/folder/c
+
+# create the config file
+cfg="${basedir}/config.yaml"
+cat > ${cfg} << _EOF
+config:
+  backup: false
+  create: true
+  dotpath: dotfiles
+dotfiles:
+  thedotfile:
+    dst: ${tmpd}/folder
+    src: folder
+profiles:
+  p1:
+    dotfiles:
+    - thedotfile
+_EOF
+
+#
+# Test with no ignore-missing setting
+#
+
+# Expect diff
+echo "[+] test with no ignore-missing setting"
+set +e
+cd ${ddpath} | ${bin} compare -c ${cfg} --verbose --profile=p1
+[ "$?" = "0" ] && exit 1
+set -e
+
+#
+# Test with command-line flga
+#
+
+# Expect no diff
+echo "[+] test with command-line flag"
+set +e
+cd ${ddpath} | ${bin} compare -c ${cfg} --verbose --profile=p1 --ignore-missing
+[ "$?" != "0" ] && exit 1
+set -e
+
+#
+# Test with global option
+#
+
+cat > ${cfg} << _EOF
+config:
+  backup: false
+  create: true
+  dotpath: dotfiles
+  ignore_missing_in_dotdrop: true
+dotfiles:
+  thedotfile:
+    dst: ${tmpd}/folder
+    src: folder
+profiles:
+  p1:
+    dotfiles:
+    - thedotfile
+_EOF
+
+# Expect no diff
+echo "[+] test global option"
+set +e
+cd ${ddpath} | ${bin} compare -c ${cfg} --verbose --profile=p1
+[ "$?" != "0" ] && exit 1
+set -e
+
+#
+# Test with dotfile option
+#
+
+cat > ${cfg} << _EOF
+config:
+  backup: false
+  create: true
+  dotpath: dotfiles
+dotfiles:
+  thedotfile:
+    dst: ${tmpd}/folder
+    src: folder
+    ignore_missing_in_dotdrop: true
+profiles:
+  p1:
+    dotfiles:
+    - thedotfile
+_EOF
+
+# Expect no diff
+echo "[+] test dotfile option"
+set +e
+cd ${ddpath} | ${bin} compare -c ${cfg} --verbose --profile=p1
+[ "$?" != "0" ] && exit 1
+set -e
+
+# CLEANING
+rm -rf ${basedir} ${tmpd}
+
+echo "OK"
+exit 0

tests-ng/update-ignore-missing.sh

@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+# author: deadc0de6 (https://github.com/deadc0de6)
+# Copyright (c) 2017, deadc0de6
+#
+# test missing files ignored as expected
+# returns 1 in case of error
+#
+
+# exit on first error
+set -e
+
+# all this crap to get current path
+rl="readlink -f"
+if ! ${rl} "${0}" >/dev/null 2>&1; then
+  rl="realpath"
+
+  if ! hash ${rl}; then
+    echo "\"${rl}\" not found !" && exit 1
+  fi
+fi
+cur=$(dirname "$(${rl} "${0}")")
+
+# dotdrop path can be pass as argument
+ddpath="${cur}/../"
+[ "${1}" != "" ] && ddpath="${1}"
+[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1
+
+export PYTHONPATH="${ddpath}:${PYTHONPATH}"
+bin="python3 -m dotdrop.dotdrop"
+hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true
+
+echo "dotdrop path: ${ddpath}"
+echo "pythonpath: ${PYTHONPATH}"
+
+# get the helpers
+source ${cur}/helpers
+
+echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)"
+
+################################################################
+# this is the test
+################################################################
+
+# $1 pattern
+# $2 path
+grep_or_fail()
+{
+  set +e
+  grep "${1}" "${2}" >/dev/null 2>&1 || (echo "pattern not found in ${2}" && exit 1)
+  set -e
+}
+
+# dotdrop directory
+tmps=`mktemp -d --suffix='-dotdrop-tests-source' || mktemp -d`
+dt="${tmps}/dotfiles"
+mkdir -p ${dt}/folder
+touch ${dt}/folder/a
+
+# fs dotfiles
+tmpd=`mktemp -d --suffix='-dotdrop-tests-dest' || mktemp -d`
+cp -r ${dt}/folder ${tmpd}/
+touch ${tmpd}/folder/b
+mkdir ${tmpd}/folder/c
+
+# create the config file
+cfg="${tmps}/config.yaml"
+cat > ${cfg} << _EOF
+config:
+  backup: false
+  create: true
+  dotpath: dotfiles
+dotfiles:
+  thedotfile:
+    dst: ${tmpd}/folder
+    src: folder
+profiles:
+  p1:
+    dotfiles:
+    - thedotfile
+_EOF
+#cat ${cfg}
+
+#tree ${dt}
+
+#
+# Test with no ignore-missing setting
+#
+
+# file b / folder c SHOULD be copied
+echo "[+] test with no ignore-missing setting"
+cd ${ddpath} | ${bin} update -f -c ${cfg} --verbose --profile=p1 --key thedotfile
+
+[ ! -e ${dt}/folder/b ] && echo "should have been updated" && exit 1
+[ ! -e ${dt}/folder/c ] && echo "should have been updated" && exit 1
+
+# Reset
+rm ${dt}/folder/b
+rmdir ${dt}/folder/c
+
+#
+# Test with command-line flag
+#
+
+# file b / folder c should NOT be copied
+echo "[+] test with command-line flag"
+cd ${ddpath} | ${bin} update -f -c ${cfg} --verbose --profile=p1 --key thedotfile --ignore-missing
+
+[ -e ${dt}/folder/b ] && echo "should not have been updated" && exit 1
+[ -e ${dt}/folder/c ] && echo "should not have been updated" && exit 1
+
+#
+# Test with global option
+#
+
+cat > ${cfg} << _EOF
+config:
+  backup: false
+  create: true
+  dotpath: dotfiles
+  ignore_missing_in_dotdrop: true
+dotfiles:
+  thedotfile:
+    dst: ${tmpd}/folder
+    src: folder
+profiles:
+  p1:
+    dotfiles:
+    - thedotfile
+_EOF
+
+# file b / folder c should NOT be copied
+echo "[+] test global option"
+cd ${ddpath} | ${bin} update -f -c ${cfg} --verbose --profile=p1 --key thedotfile
+
+[ -e ${dt}/folder/b ] && echo "should not have been updated" && exit 1
+[ -e ${dt}/folder/c ] && echo "should not have been updated" && exit 1
+
+#
+# Test with dotfile option
+#
+
+cat > ${cfg} << _EOF
+config:
+  backup: false
+  create: true
+  dotpath: dotfiles
+dotfiles:
+  thedotfile:
+    dst: ${tmpd}/folder
+    src: folder
+    ignore_missing_in_dotdrop: true
+profiles:
+  p1:
+    dotfiles:
+    - thedotfile
+_EOF
+# file b / folder c should NOT be copied
+echo "[+] test dotfile option"
+cd ${ddpath} | ${bin} update -f -c ${cfg} --verbose --profile=p1 --key thedotfile
+
+[ -e ${dt}/folder/b ] && echo "should not have been updated" && exit 1
+[ -e ${dt}/folder/c ] && echo "should not have been updated" && exit 1
+
+# CLEANING
+rm -rf ${tmps} ${tmpd}
+
+echo "OK"
+exit 0

tests/helpers.py

@@ -135,6 +135,7 @@ def _fake_args():
     args['--file-only'] = False
     args['--workers'] = 1
     args['--preserve-mode'] = False
+    args['--ignore-missing'] = False
     # cmds
     args['profiles'] = False
     args['files'] = False