refactor ignore patterns

CONTRIBUTING.md

@@ -177,7 +177,8 @@ dynvariables:
 * uses `shutil.copytree` with a callback
   that will match each path against the ignore pattern
 * the pattern (and negative pattern) will be matched
-  against the path(s) that are being imported
+  against the path that is being imported
+  (and not against its destination in the dotpath)
 
 **install**
 

docs/config/config-file.md

@@ -191,7 +191,8 @@ It is possible to ignore specific patterns when using dotdrop.
     * Using dotfiles block [upignore](config-dotfiles.md)
     * Using the command line switch `-i`/`--ignore`
 
-The ignore pattern must follow Unix shell-style wildcards, like, for example `*/path/to/file`.
+The ignore pattern must follow Unix shell-style wildcards, like, for example `*/path/to/file` for files or
+`*/path/to/directory/*` for directories.
 Make sure to quote these when using wildcards in the config file.
 
 ```yaml
@@ -210,10 +211,10 @@ dotfiles:
     dst: ~/.vim
     src: vim
     upignore:
-    - '*/undo-dir'
+    - '*/undo-dir/*'
-    - '*/plugged'
+    - '*/plugged/*'
     instignore:
-    - '*/internal'
+    - '*/internal/*'
     cmpignore:
     - '*/ignore-me'
 ...
@@ -235,7 +236,7 @@ dotfiles:
     src: vim
     cmpignore:
       - '*'
-      - '!*/colors/**'
+      - '!*/colors/*'
 ```
 
 To completely ignore comparison of a specific dotfile:
@@ -255,7 +256,7 @@ dotfiles:
     src: config/some_directory
     dst: ~/.config/some_directory
     upignore:
-      - '*sub_directory_to_ignore'
+      - '*/sub_directory_to_ignore/*'
 ```
 
 To ignore a specific file `testfile` and directory `testdir` when importing:
@@ -273,7 +274,7 @@ dotfiles:
     src: zsh
     dst: ~/.config/zsh
     upignore:
-      - "plugins/*"
+      - "*/plugins/*"
       - "!plugins/custom_plugin.zsh"
 ```
 

dotdrop/comparator.py

@@ -123,8 +123,14 @@ class Comparator:
         # handle files only in deployed dir
         self.log.dbg(f'files only in deployed dir: {comp.left_only}')
         for i in comp.left_only:
+            abspath1 = os.path.join(local_path, i)
+            if os.path.isdir(abspath1):
+                abspath1 += os.path.sep
+            abspath2 = os.path.join(deployed_path, i)
+            if os.path.isdir(abspath2):
+                abspath2 += os.path.sep
             if self.ignore_missing_in_dotdrop or \
-               must_ignore([os.path.join(local_path, i)],
+               must_ignore([abspath1, abspath2],
                            ignore, debug=self.debug):
                 continue
             ret.append(f'=> \"{i}\" does not exist on destination\n')
@@ -132,7 +138,13 @@ class Comparator:
         # handle files only in dotpath dir
         self.log.dbg(f'files only in dotpath dir: {comp.right_only}')
         for i in comp.right_only:
-            if must_ignore([os.path.join(deployed_path, i)],
+            abspath1 = os.path.join(local_path, i)
+            if os.path.isdir(abspath1):
+                abspath1 += os.path.sep
+            abspath2 = os.path.join(deployed_path, i)
+            if os.path.isdir(abspath2):
+                abspath2 += os.path.sep
+            if must_ignore([abspath1, abspath2],
                            ignore, debug=self.debug):
                 continue
 

dotdrop/dotdrop.py

@@ -952,6 +952,7 @@ def main():
     ret, command = _exec_command(opts)
     cmd_time = time.time() - time0
 
+    opts.debug_command()
     LOG.dbg(f'done executing command \"{command}\"')
     LOG.dbg(f'options loaded in {options_time}')
     LOG.dbg(f'command executed in {cmd_time}')

dotdrop/options.py

@@ -170,8 +170,8 @@ class Options(AttrMonitor):
         self.log.dbg('#################### DOTDROP ####################')
         self.log.dbg('#################################################')
         self.log.dbg(f'version: {VERSION}')
-        args = ' '.join(sys.argv)
+        self.argv = ' '.join(sys.argv)
-        self.log.dbg(f'command: {args}')
+        self.log.dbg(f'command: {self.argv}')
         self.log.dbg(f'config file: {self.confpath}')
 
         self._read_config()
@@ -185,6 +185,9 @@ class Options(AttrMonitor):
         # start monitoring for bad attribute
         self._set_attr_err = True
 
+    def debug_command(self):
+        self.log.dbg(f'command: {self.argv}')
+
     @classmethod
     def _get_config_from_env(cls, name):
         # look in XDG_CONFIG_HOME

dotdrop/updater.py

@@ -261,7 +261,7 @@ class Updater:
         local_path = os.path.expanduser(local_path)
 
         # find the differences
-        diff = filecmp.dircmp(deployed_path, local_path, ignore=None)
+        diff = filecmp.dircmp(deployed_path, local_path)
         # handle directories diff
         ret = self._merge_dirs(diff, dotfile,
                                ignores)
@@ -278,6 +278,7 @@ class Updater:
             if not os.path.isdir(exist):
                 # ignore files for now
                 continue
+            exist += os.path.sep
             # match to dotdrop dotpath
             new = os.path.join(right, toadd)
             if ignore_missing_in_dotdrop and not os.path.exists(new):
@@ -316,6 +317,7 @@ class Updater:
             if not os.path.isdir(old):
                 # ignore files for now
                 continue
+            old += os.path.sep
             if self._must_ignore([old], ignores):
                 continue
             if self.dry:
@@ -347,7 +349,7 @@ class Updater:
                               ignores,
                               compare=False)
 
-    def _merge_dirs_copy_left_only(self, diff, left, right,
+    def _merge_files_copy_left_only(self, diff, left, right,
                                    ignore_missing_in_dotdrop,
                                    ignores):
         """copy files that don't exist in dotdrop"""
@@ -375,7 +377,7 @@ class Updater:
             self._mirror_file_perms(exist, new)
             self.log.sub(f'\"{new}\" added')
 
-    def _merge_dirs_remove_right_only_2(self, diff, right, ignores):
+    def _merge_files_remove_right_only(self, diff, right, ignores):
         """remove files that don't exist in deployed version"""
         self.log.dbg(f'_merge_dirs_remove_right_only_2: {diff.right_only}')
         for toremove in diff.right_only:
@@ -402,16 +404,18 @@ class Updater:
         ignore_missing_in_dotdrop = self.ignore_missing_in_dotdrop or \
             dotfile.ignore_missing_in_dotdrop
 
+        # directories
         self._merge_dirs_create_left_only(diff, left, right,
                                           ignore_missing_in_dotdrop,
                                           ignores)
         self._merge_dirs_remove_right_only(diff, left, right,
                                            ignore_missing_in_dotdrop,
                                            ignores)
-        self._merge_dirs_copy_left_only(diff, left, right,
+        # files
+        self._merge_files_copy_left_only(diff, left, right,
                                         ignore_missing_in_dotdrop,
                                         ignores)
-        self._merge_dirs_remove_right_only_2(diff, right, ignores)
+        self._merge_files_remove_right_only(diff, right, ignores)
 
         # compare rights
         for common in diff.common_files:

dotdrop/utils.py

@@ -228,6 +228,9 @@ def _match_ignore_pattern(path, pattern, debug=False):
     """
     returns true if path matches the pattern
     """
+    if debug:
+        msg = f'fnmatch \"{path}\" against {pattern}'
+        LOG.dbg(msg, force=True)
     ret = fnmatch.fnmatch(path, pattern)
     if debug:
         LOG.dbg(f'ignore \"{pattern}\" match: {path}',
@@ -350,6 +353,8 @@ def copytree_with_ign(src, dst, ignore_func=None, debug=False):
         srcf = os.path.join(src, entry)
         dstf = os.path.join(dst, entry)
         if ignore_func:
+            if os.path.isdir(srcf):
+                srcf += os.path.sep
             if ignore_func(srcf):
                 continue
         if os.path.isdir(srcf):

tests-ng/ignore-patterns.sh

@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+# author: deadc0de6 (https://github.com/deadc0de6)
+# Copyright (c) 2017, deadc0de6
+#
+# test ignore patterns
+# returns 1 in case of error
+# see #418
+#
+
+## start-cookie
+set -eu -o errtrace -o pipefail
+cur=$(cd "$(dirname "${0}")" && pwd)
+ddpath="${cur}/../"
+PPATH="{PYTHONPATH:-}"
+export PYTHONPATH="${ddpath}:${PPATH}"
+altbin="python3 -m dotdrop.dotdrop"
+if hash coverage 2>/dev/null; then
+  mkdir -p coverages/
+  altbin="coverage run -p --data-file coverages/coverage --source=dotdrop -m dotdrop.dotdrop"
+fi
+bin="${DT_BIN:-${altbin}}"
+# shellcheck source=tests-ng/helpers
+source "${cur}"/helpers
+echo -e "$(tput setaf 6)==> RUNNING $(basename "${BASH_SOURCE[0]}") <==$(tput sgr0)"
+## end-cookie
+
+################################################################
+# this is the test
+################################################################
+
+# the dotfile source
+tmps=$(mktemp -d --suffix='-dotdrop-tests' || mktemp -d)
+dotpath="${tmps}"/dotfiles
+mkdir -p "${dotpath}"
+#echo "dotfile source: ${tmps}"
+# the dotfile destination
+tmpd=$(mktemp -d --suffix='-dotdrop-tests' || mktemp -d)
+#echo "dotfile destination: ${tmpd}"
+
+clear_on_exit "${tmps}"
+clear_on_exit "${tmpd}"
+
+# create the config file
+cfg1="${tmps}/config1.yaml"
+cfg2="${tmps}/config2.yaml"
+cfg3="${tmps}/config3.yaml"
+cfg4="${tmps}/config4.yaml"
+
+cat > "${cfg1}" << _EOF
+config:
+  backup: true
+  create: true
+  dotpath: dotfiles
+  ignoreempty: true
+dotfiles:
+  d_mpv:
+    src: mpv
+    dst: ${tmpd}/mpv
+    cmpignore:
+    - '*/watch_later/*'
+    upignore:
+    - '*/watch_later/*'
+    instignore:
+    - '*/watch_later/*'
+profiles:
+  p1:
+    dotfiles:
+    - d_mpv
+_EOF
+
+cat > "${cfg2}" << _EOF
+config:
+  backup: true
+  create: true
+  dotpath: dotfiles
+  ignoreempty: true
+dotfiles:
+  d_mpv:
+    src: mpv
+    dst: ${tmpd}/mpv
+    cmpignore:
+    - '*/watch_later/*'
+    upignore:
+    - '*/watch_later/*'
+    instignore:
+    - '*/watch_later/*'
+profiles:
+  p1:
+    dotfiles:
+    - d_mpv
+_EOF
+
+cat > "${cfg3}" << _EOF
+config:
+  backup: true
+  create: true
+  dotpath: dotfiles
+  ignoreempty: true
+  impignore:
+    - '*/watch_later/*'
+dotfiles:
+profiles:
+_EOF
+
+cat > "${cfg4}" << _EOF
+config:
+  backup: true
+  create: true
+  dotpath: dotfiles
+  ignoreempty: true
+  impignore:
+    - '*/watch_later/*'
+dotfiles:
+profiles:
+_EOF
+
+clean_both()
+{
+  rm -rf "${dotpath}/mpv"
+  rm -rf "${tmpd}/mpv"
+}
+
+# $1 parent
+create_hierarchy()
+{
+  mkdir -p "${1}"/mpv
+  echo "file" > "${1}"/mpv/file
+  mkdir -p "${1}"/mpv/dir1
+  echo "file2" > "${1}"/mpv/dir1/file
+  mkdir -p "${1}"/mpv/watch_later
+  echo "watch_later" > "${1}"/mpv/watch_later/watch_later_file
+}
+
+create_in_dotpath()
+{
+  create_hierarchy "${dotpath}"
+}
+
+create_in_dst()
+{
+  create_hierarchy "${tmpd}"
+}
+
+###################################################
+# test install
+###################################################
+clean_both
+create_in_dotpath
+cd "${ddpath}" | ${bin} install -f -c "${cfg1}" -p p1 -V
+
+[ -d "${tmpd}/mpv/watch_later" ] && echo "install (cfg1) failed" && exit 1
+
+clean_both
+create_in_dotpath
+cd "${ddpath}" | ${bin} install -f -c "${cfg2}" -p p1 -V
+
+[ -d "${tmpd}/mpv/watch_later" ] && echo "install (cfg2) failed" && exit 1
+
+###################################################
+# test update
+###################################################
+clean_both
+create_in_dotpath
+create_in_dst
+echo newfile "${tmpd}/mpv/watch_later/newfile"
+rm -rf "${dotpath}/mpv/watch_later"
+
+cd "${ddpath}" | ${bin} update -f -c "${cfg1}" -p p1 -V
+[ -d "${dotpath}/mpv/watch_later" ] && echo "update (cfg1) failed" && exit 1
+[ -e "${dotpath}/mpv/watch_later/newfile" ] && echo "update (cfg1) failed - new file" && exit 1
+
+clean_both
+create_in_dotpath
+create_in_dst
+echo newfile "${tmpd}/mpv/watch_later/newfile"
+rm -rf "${dotpath}/mpv/watch_later"
+
+cd "${ddpath}" | ${bin} update -f -c "${cfg2}" -p p1 -V
+[ -d "${dotpath}/mpv/watch_later" ] && echo "update (cfg2) failed" && exit 1
+[ -e "${dotpath}/mpv/watch_later/newfile" ] && echo "update (cfg2) failed - new file" && exit 1
+
+###################################################
+# test import
+###################################################
+clean_both
+create_in_dst
+
+cd "${ddpath}" | ${bin} import -f -c "${cfg3}" -p p1 -V "${tmpd}/mpv"
+[ -d "${dotpath}/${tmpd}/mpv/watch_later" ] && echo "import (cfg3) failed" && exit 1
+[ ! -e "${dotpath}/${tmpd}/mpv/file" ] && echo "import (cfg3) failed - file" && exit 1
+
+clean_both
+create_in_dst
+
+cd "${ddpath}" | ${bin} import -f -c "${cfg4}" -p p1 -V "${tmpd}/mpv"
+[ -d "${dotpath}/${tmpd}/mpv/watch_later" ] && echo "import (cfg4) failed" && exit 1
+[ ! -e "${dotpath}/${tmpd}/mpv/file" ] && echo "import (cfg4) failed - file" && exit 1
+
+###################################################
+# test compare
+###################################################
+clean_both
+create_in_dst
+create_in_dotpath
+
+rm -rf "${dotpath}/mpv/watch_later"
+cd "${ddpath}" | ${bin} compare -c "${cfg1}" -p p1 -V
+
+clean_both
+create_in_dst
+create_in_dotpath
+
+rm -rf "${tmpd}/mpv/watch_later"
+cd "${ddpath}" | ${bin} compare -c "${cfg1}" -p p1 -V
+
+clean_both
+create_in_dst
+create_in_dotpath
+
+rm -rf "${dotpath}/mpv/watch_later"
+cd "${ddpath}" | ${bin} compare -c "${cfg2}" -p p1 -V
+
+clean_both
+create_in_dst
+create_in_dotpath
+
+rm -rf "${tmpd}/mpv/watch_later"
+cd "${ddpath}" | ${bin} compare -c "${cfg2}" -p p1 -V
+
+###################################################
+
+echo "OK"
+exit 0