diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 29de141..b0bb3d9 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -20,15 +20,24 @@ jobs: run: | python -m pip install --upgrade pip pip install -r tests-requirements.txt + pip install --user --upgrade coverage pip install -r requirements.txt npm install -g remark-cli remark-validate-links npm install -g markdown-link-check - - name: Run tests + - name: Run sequential tests run: | ./tests.sh env: DOTDROP_FORCE_NODEBUG: yes DOTDROP_NOBANNER: yes + DOTDROP_WORKERS: 1 + - name: Run parallel tests + run: | + ./tests.sh + env: + DOTDROP_FORCE_NODEBUG: yes + DOTDROP_NOBANNER: yes + DOTDROP_WORKERS: 4 - name: Coveralls run: | coveralls diff --git a/docs/config-format.md b/docs/config-format.md index 666e5a3..3cc1649 100644 --- a/docs/config-format.md +++ b/docs/config-format.md @@ -46,6 +46,7 @@ Entry | Description `src` | dotfile path within the `dotpath` (dotfile with empty `src` are ignored and considered installed, can use `variables` and `dynvariables`, make sure to quote) `link` | define how this dotfile is installed. Possible values: *nolink*, *link*, *link_children* (see [Symlinking dotfiles](config.md#symlink-dotfiles)) (defaults to value of `link_dotfile_default`) `actions` | list of action keys that need to be defined in the **actions** entry below (see [actions](config-details.md#entry-actions)) +`chmod` | defines the file permission in octal notation to apply during installation (see [permissions](config.md#permissions)) `cmpignore` | list of patterns to ignore when comparing (enclose in quotes when using wildcards, see [ignore patterns](config.md#ignore-patterns)) `ignoreempty` | if true empty template will not be deployed (defaults to value of `ignoreempty`) `instignore` | list of patterns to ignore when installing (enclose in quotes when using wildcards, see [ignore patterns](config.md#ignore-patterns)) diff --git a/docs/config.md b/docs/config.md index 46b7a0f..b6863b1 100644 --- a/docs/config.md +++ b/docs/config.md @@ -59,6 +59,31 @@ Here are some rules on the use of variables in configs: * external/imported `(dyn)variables` take precedence over `(dyn)variables` defined inside the main config file +## Permissions + +Dotdrop allows to control the permission applied to a dotfile using the +config dotfile entry [chmod](config-format.md#dotfiles-entry). +A [chmod](config-format.md#dotfiles-entry) entry on a directory is applied to the +directory only, not recursively. + +On `import` the following rules are applied: + +* if the `-m --preserve-mode` switch is provided the imported file permissions are + stored in a `chmod` entry +* if imported file permissions differ from umask then its permissions are automatically + stored in the `chmod` entry +* otherwise no `chmod` entry is added + +On `install` the following rules are applied: + +* if `chmod` is specified in the dotfile, it will be applied to the installed dotfile +* otherwise the permissions of the dotfile in the `dotpath` are applied. + +On `update`: + +* if the permissions of the file in the filesystem differ from the dotfile in the `dotpath` + then the dotfile entry `chmod` is added/updated accordingly + ## Symlink dotfiles Dotdrop is able to install dotfiles in three different ways diff --git a/docs/howto/global-config-files.md b/docs/howto/global-config-files.md deleted file mode 100644 index 4b44f1f..0000000 --- a/docs/howto/global-config-files.md +++ /dev/null @@ -1,26 +0,0 @@ -# Manage system dotfiles - -Dotdrop doesn't allow to handle file rights and permissions (at least not directly). Every operations (`mkdir`, `cp`, `mv`, `ln`, file creation) are executed with the rights of the user calling dotdrop. The rights of the stored dotfile are mirrored on the deployed dotfile (`chmod` like). It works well for local/user dotfiles but doesn't allow to manage global/system config files (`/etc` or `/var` for example) directly. - -Using dotdrop with `sudo` to handle local **and** global dotfiles in the same *session* is a bad idea as the resulting files will all have messed up owners. - -It is therefore recommended to have two different config files (and thus two different *dotpath*) for handling these two uses cases: - -* one `config.yaml` for the local/user dotfiles (with its dedicated *dotpath*) -* another config file for the global/system dotfiles (with its dedicated *dotpath*) - -The default config file (`config.yaml`) is used when installing the user dotfiles as usual -```bash -# default config file is config.yaml -$ ./dotdrop.sh import -$ ./dotdrop.sh install -... -``` - -A different config file (for example `global-config.yaml` and its associated *dotpath*) is used when installing/managing global dotfiles and is to be used with `sudo` or directly by the root user -```bash -# specifying explicitly the config file with the --cfg switch -$ sudo ./dotdrop.sh import --cfg=global-config.yaml -$ sudo ./dotdrop.sh install --cfg=global-config.yaml -... -``` \ No newline at end of file diff --git a/docs/howto/howto.md b/docs/howto/howto.md index fbe35cb..41e6d43 100644 --- a/docs/howto/howto.md +++ b/docs/howto/howto.md @@ -28,7 +28,7 @@ ## Manage system dotfiles -[Manage system dotfiles](global-config-files.md) +[Manage system dotfiles](system-config-files.md) ## Merge files on install diff --git a/docs/howto/system-config-files.md b/docs/howto/system-config-files.md new file mode 100644 index 0000000..27ed6a9 --- /dev/null +++ b/docs/howto/system-config-files.md @@ -0,0 +1,29 @@ +# Manage system dotfiles + +Dotdrop doesn't allow to handle file owernership (at least not directly). Every file operations (create/copy file/directory, create symlinks, etc) are executed with the rights of the user calling dotdrop. + +Using dotdrop with `sudo` to unprivileged and privileged files in the same *session* is a bad idea as the resulting files will all have messed up owners. + +It is therefore recommended to have two different config files (and thus two different *dotpath*) +for handling these two uses cases: + +For example: + +* one `config-user.yaml` for the local/user dotfiles (with its dedicated *dotpath*, for example `dotfiles-user`) +* one `config-root.yaml` for the system/root dotfiles (with its dedicated *dotpath*, for example `dotfiles-root`) + +`config-user.yaml` is used when managing the user's dotfiles +```bash +## user config file is config-user.yaml +$ ./dotdrop.sh import --cfg config-user.yaml +$ ./dotdrop.sh install --cfg config-user.yaml +... +``` + +`config-root.yaml` is used when managing system's dotfiles and is to be used with `sudo` or directly by the root user +```bash +## root config file is config-root.yaml +$ sudo ./dotdrop.sh import --cfg=config-root.yaml +$ sudo ./dotdrop.sh install --cfg=config-root.yaml +... +``` diff --git a/docs/usage.md b/docs/usage.md index 2cbf1ec..84de6bb 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -226,6 +226,28 @@ dotdrop. It will: For more options, see the usage with `dotdrop --help` +## Concurrency + +The command line switch `-w --workers` if set to a value greater than one allows to use +multiple concurrent workers to execute an operation. It can be applied to the following +commands: + +* `install` +* `compare` +* `update` + +It should be set to a maximum of the number of cores available (usually returned +on linux by the command `nproc`). + +It may speed up the operation but cannot be used interractively (it needs `-f --force` to be set +except for `compare`) and cannot be used with `-d --dry`. Also information printed to stdout/stderr +will probably be messed up. + +**WARNING** this feature hasn't been extensively tested and is to be used at your own risk. +If you try it out and find any issue, please [report it](https://github.com/deadc0de6/dotdrop/issues). +Also if you find it useful and have been able to successfully speed up your operation when using +`-w --workers`, do please also report it [in an issue](https://github.com/deadc0de6/dotdrop/issues). + ## Environment variables Following environment variables can be used to specify different CLI options. @@ -255,3 +277,11 @@ export DOTDROP_FORCE_NODEBUG= ```bash export DOTDROP_TMPDIR="/tmp/dotdrop-tmp" ``` +* `DOTDROP_WORKDIR`: overwrite the `workdir` defined in the config +```bash +export DOTDROP_WORKDIR="/tmp/dotdrop-workdir" +``` +* `DOTDROP_WORKERS`: overwrite the `-w --workers` cli argument +```bash +export DOTDROP_WORKERS="10" +``` diff --git a/dotdrop/cfg_aggregator.py b/dotdrop/cfg_aggregator.py index 230e264..3853386 100644 --- a/dotdrop/cfg_aggregator.py +++ b/dotdrop/cfg_aggregator.py @@ -43,103 +43,9 @@ class CfgAggregator: self.log = Logger() self._load() - def _load(self): - """load lower level config""" - self.cfgyaml = CfgYaml(self.path, - self.profile_key, - debug=self.debug) - - # settings - self.settings = Settings.parse(None, self.cfgyaml.settings) - - # dotfiles - self.dotfiles = Dotfile.parse_dict(self.cfgyaml.dotfiles) - if self.debug: - self._debug_list('dotfiles', self.dotfiles) - - # profiles - self.profiles = Profile.parse_dict(self.cfgyaml.profiles) - if self.debug: - self._debug_list('profiles', self.profiles) - - # actions - self.actions = Action.parse_dict(self.cfgyaml.actions) - if self.debug: - self._debug_list('actions', self.actions) - - # trans_r - self.trans_r = Transform.parse_dict(self.cfgyaml.trans_r) - if self.debug: - self._debug_list('trans_r', self.trans_r) - - # trans_w - self.trans_w = Transform.parse_dict(self.cfgyaml.trans_w) - if self.debug: - self._debug_list('trans_w', self.trans_w) - - # variables - self.variables = self.cfgyaml.variables - if self.debug: - self._debug_dict('variables', self.variables) - - # patch dotfiles in profiles - self._patch_keys_to_objs(self.profiles, - "dotfiles", self.get_dotfile) - - # patch action in dotfiles actions - self._patch_keys_to_objs(self.dotfiles, - "actions", self._get_action_w_args) - # patch action in profiles actions - self._patch_keys_to_objs(self.profiles, - "actions", self._get_action_w_args) - - # patch actions in settings default_actions - self._patch_keys_to_objs([self.settings], - "default_actions", self._get_action_w_args) - if self.debug: - msg = 'default actions: {}'.format(self.settings.default_actions) - self.log.dbg(msg) - - # patch trans_w/trans_r in dotfiles - self._patch_keys_to_objs(self.dotfiles, - "trans_r", - self._get_trans_w_args(self._get_trans_r), - islist=False) - self._patch_keys_to_objs(self.dotfiles, - "trans_w", - self._get_trans_w_args(self._get_trans_w), - islist=False) - - def _patch_keys_to_objs(self, containers, keys, get_by_key, islist=True): - """ - map for each key in the attribute 'keys' in 'containers' - the returned object from the method 'get_by_key' - """ - if not containers: - return - if self.debug: - self.log.dbg('patching {} ...'.format(keys)) - for c in containers: - objects = [] - okeys = getattr(c, keys) - if not okeys: - continue - if not islist: - okeys = [okeys] - for k in okeys: - o = get_by_key(k) - if not o: - err = '{} does not contain'.format(c) - err += ' a {} entry named {}'.format(keys, k) - self.log.err(err) - raise Exception(err) - objects.append(o) - if not islist: - objects = objects[0] - # if self.debug: - # er = 'patching {}.{} with {}' - # self.log.dbg(er.format(c, keys, objects)) - setattr(c, keys, objects) + ######################################################## + # public methods + ######################################################## def del_dotfile(self, dotfile): """remove this dotfile from the config""" @@ -149,27 +55,21 @@ class CfgAggregator: """remove this dotfile from this profile""" return self.cfgyaml.del_dotfile_from_profile(dotfile.key, profile.key) - def _create_new_dotfile(self, src, dst, link): - """create a new dotfile""" - # get a new dotfile with a unique key - key = self._get_new_dotfile_key(dst) - if self.debug: - self.log.dbg('new dotfile key: {}'.format(key)) - # add the dotfile - self.cfgyaml.add_dotfile(key, src, dst, link) - return Dotfile(key, dst, src) - - def new(self, src, dst, link): + def new_dotfile(self, src, dst, link, chmod=None): """ import a new dotfile @src: path in dotpath @dst: path in FS @link: LinkType + @chmod: file permission """ dst = self.path_to_dotfile_dst(dst) dotfile = self.get_dotfile_by_src_dst(src, dst) if not dotfile: - dotfile = self._create_new_dotfile(src, dst, link) + dotfile = self._create_new_dotfile(src, dst, link, chmod=chmod) + + if not dotfile: + return False key = dotfile.key ret = self.cfgyaml.add_dotfile_to_profile(key, self.profile_key) @@ -177,82 +77,16 @@ class CfgAggregator: msg = 'new dotfile {} to profile {}' self.log.dbg(msg.format(key, self.profile_key)) - self.save() - if ret and not self.dry: - # reload - if self.debug: - self.log.dbg('reloading config') - olddebug = self.debug - self.debug = False - self._load() - self.debug = olddebug + if ret: + self._save_and_reload() return ret - def _get_new_dotfile_key(self, dst): - """return a new unique dotfile key""" - path = os.path.expanduser(dst) - existing_keys = self.cfgyaml.get_all_dotfile_keys() - if self.settings.longkey: - return self._get_long_key(path, existing_keys) - return self._get_short_key(path, existing_keys) - - def _norm_key_elem(self, elem): - """normalize path element for sanity""" - elem = elem.lstrip('.') - elem = elem.replace(' ', '-') - return elem.lower() - - def _split_path_for_key(self, path): - """return a list of path elements, excluded home path""" - p = strip_home(path) - dirs = [] - while True: - p, f = os.path.split(p) - dirs.append(f) - if not p or not f: - break - dirs.reverse() - # remove empty entries - dirs = filter(None, dirs) - # normalize entries - return list(map(self._norm_key_elem, dirs)) - - def _get_long_key(self, path, keys): - """ - return a unique long key representing the - absolute path of path - """ - dirs = self._split_path_for_key(path) - prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix - key = self.key_sep.join([prefix] + dirs) - return self._uniq_key(key, keys) - - def _get_short_key(self, path, keys): - """ - return a unique key where path - is known not to be an already existing dotfile - """ - dirs = self._split_path_for_key(path) - dirs.reverse() - prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix - entries = [] - for d in dirs: - entries.insert(0, d) - key = self.key_sep.join([prefix] + entries) - if key not in keys: - return key - return self._uniq_key(key, keys) - - def _uniq_key(self, key, keys): - """unique dotfile key""" - newkey = key - cnt = 1 - while newkey in keys: - # if unable to get a unique path - # get a random one - newkey = self.key_sep.join([key, str(cnt)]) - cnt += 1 - return newkey + def update_dotfile(self, key, chmod): + """update an existing dotfile""" + ret = self.cfgyaml.update_dotfile(key, chmod) + if ret: + self._save_and_reload() + return ret def path_to_dotfile_dst(self, path): """normalize the path to match dotfile dst""" @@ -353,6 +187,216 @@ class CfgAggregator: except StopIteration: return None + ######################################################## + # accessors for public methods + ######################################################## + + def _create_new_dotfile(self, src, dst, link, chmod=None): + """create a new dotfile""" + # get a new dotfile with a unique key + key = self._get_new_dotfile_key(dst) + if self.debug: + self.log.dbg('new dotfile key: {}'.format(key)) + # add the dotfile + if not self.cfgyaml.add_dotfile(key, src, dst, link, chmod=chmod): + return None + return Dotfile(key, dst, src) + + ######################################################## + # parsing + ######################################################## + + def _load(self): + """load lower level config""" + self.cfgyaml = CfgYaml(self.path, + self.profile_key, + debug=self.debug) + + # settings + self.settings = Settings.parse(None, self.cfgyaml.settings) + + # dotfiles + self.dotfiles = Dotfile.parse_dict(self.cfgyaml.dotfiles) + if self.debug: + self._debug_list('dotfiles', self.dotfiles) + + # profiles + self.profiles = Profile.parse_dict(self.cfgyaml.profiles) + if self.debug: + self._debug_list('profiles', self.profiles) + + # actions + self.actions = Action.parse_dict(self.cfgyaml.actions) + if self.debug: + self._debug_list('actions', self.actions) + + # trans_r + self.trans_r = Transform.parse_dict(self.cfgyaml.trans_r) + if self.debug: + self._debug_list('trans_r', self.trans_r) + + # trans_w + self.trans_w = Transform.parse_dict(self.cfgyaml.trans_w) + if self.debug: + self._debug_list('trans_w', self.trans_w) + + # variables + self.variables = self.cfgyaml.variables + if self.debug: + self._debug_dict('variables', self.variables) + + # patch dotfiles in profiles + self._patch_keys_to_objs(self.profiles, + "dotfiles", self.get_dotfile) + + # patch action in dotfiles actions + self._patch_keys_to_objs(self.dotfiles, + "actions", self._get_action_w_args) + # patch action in profiles actions + self._patch_keys_to_objs(self.profiles, + "actions", self._get_action_w_args) + + # patch actions in settings default_actions + self._patch_keys_to_objs([self.settings], + "default_actions", self._get_action_w_args) + if self.debug: + msg = 'default actions: {}'.format(self.settings.default_actions) + self.log.dbg(msg) + + # patch trans_w/trans_r in dotfiles + self._patch_keys_to_objs(self.dotfiles, + "trans_r", + self._get_trans_w_args(self._get_trans_r), + islist=False) + self._patch_keys_to_objs(self.dotfiles, + "trans_w", + self._get_trans_w_args(self._get_trans_w), + islist=False) + + def _patch_keys_to_objs(self, containers, keys, get_by_key, islist=True): + """ + map for each key in the attribute 'keys' in 'containers' + the returned object from the method 'get_by_key' + """ + if not containers: + return + if self.debug: + self.log.dbg('patching {} ...'.format(keys)) + for c in containers: + objects = [] + okeys = getattr(c, keys) + if not okeys: + continue + if not islist: + okeys = [okeys] + for k in okeys: + o = get_by_key(k) + if not o: + err = '{} does not contain'.format(c) + err += ' a {} entry named {}'.format(keys, k) + self.log.err(err) + raise Exception(err) + objects.append(o) + if not islist: + objects = objects[0] + # if self.debug: + # er = 'patching {}.{} with {}' + # self.log.dbg(er.format(c, keys, objects)) + setattr(c, keys, objects) + + ######################################################## + # dotfile key + ######################################################## + + def _get_new_dotfile_key(self, dst): + """return a new unique dotfile key""" + path = os.path.expanduser(dst) + existing_keys = self.cfgyaml.get_all_dotfile_keys() + if self.settings.longkey: + return self._get_long_key(path, existing_keys) + return self._get_short_key(path, existing_keys) + + def _norm_key_elem(self, elem): + """normalize path element for sanity""" + elem = elem.lstrip('.') + elem = elem.replace(' ', '-') + return elem.lower() + + def _get_long_key(self, path, keys): + """ + return a unique long key representing the + absolute path of path + """ + dirs = self._split_path_for_key(path) + prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix + key = self.key_sep.join([prefix] + dirs) + return self._uniq_key(key, keys) + + def _get_short_key(self, path, keys): + """ + return a unique key where path + is known not to be an already existing dotfile + """ + dirs = self._split_path_for_key(path) + dirs.reverse() + prefix = self.dir_prefix if os.path.isdir(path) else self.file_prefix + entries = [] + for d in dirs: + entries.insert(0, d) + key = self.key_sep.join([prefix] + entries) + if key not in keys: + return key + return self._uniq_key(key, keys) + + def _uniq_key(self, key, keys): + """unique dotfile key""" + newkey = key + cnt = 1 + while newkey in keys: + # if unable to get a unique path + # get a random one + newkey = self.key_sep.join([key, str(cnt)]) + cnt += 1 + return newkey + + ######################################################## + # helpers + ######################################################## + + def _save_and_reload(self): + if self.dry: + return + self.save() + if self.debug: + self.log.dbg('reloading config') + olddebug = self.debug + self.debug = False + self._load() + self.debug = olddebug + + def _norm_path(self, path): + if not path: + return path + path = os.path.expanduser(path) + path = os.path.expandvars(path) + path = os.path.abspath(path) + return path + + def _split_path_for_key(self, path): + """return a list of path elements, excluded home path""" + p = strip_home(path) + dirs = [] + while True: + p, f = os.path.split(p) + dirs.append(f) + if not p or not f: + break + dirs.reverse() + # remove empty entries + dirs = filter(None, dirs) + # normalize entries + return list(map(self._norm_key_elem, dirs)) + def _get_action(self, key): """return action by key""" try: @@ -404,14 +448,6 @@ class CfgAggregator: except StopIteration: return None - def _norm_path(self, path): - if not path: - return path - path = os.path.expanduser(path) - path = os.path.expandvars(path) - path = os.path.abspath(path) - return path - def _debug_list(self, title, elems): """pretty print list""" if not self.debug: diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index c5ccfab..16461c5 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -58,6 +58,7 @@ class CfgYaml: key_dotfile_actions = 'actions' key_dotfile_noempty = 'ignoreempty' key_dotfile_template = 'template' + key_dotfile_chmod = 'chmod' # profile key_profile_dotfiles = 'dotfiles' @@ -316,7 +317,29 @@ class CfgYaml: """return all existing dotfile keys""" return self.dotfiles.keys() - def add_dotfile(self, key, src, dst, link): + def update_dotfile(self, key, chmod): + """update an existing dotfile""" + if key not in self.dotfiles.keys(): + return False + df = self._yaml_dict[self.key_dotfiles][key] + old = None + if self.key_dotfile_chmod in df: + old = df[self.key_dotfile_chmod] + if old == chmod: + return False + if self._debug: + self._dbg('update dotfile: {}'.format(key)) + self._dbg('old chmod value: {}'.format(old)) + self._dbg('new chmod value: {}'.format(chmod)) + df = self._yaml_dict[self.key_dotfiles][key] + if not chmod: + del df[self.key_dotfile_chmod] + else: + df[self.key_dotfile_chmod] = str(format(chmod, 'o')) + self._dirty = True + return True + + def add_dotfile(self, key, src, dst, link, chmod=None): """add a new dotfile""" if key in self.dotfiles.keys(): return False @@ -324,16 +347,25 @@ class CfgYaml: self._dbg('adding new dotfile: {}'.format(key)) self._dbg('new dotfile src: {}'.format(src)) self._dbg('new dotfile dst: {}'.format(dst)) - + self._dbg('new dotfile link: {}'.format(link)) + self._dbg('new dotfile chmod: {}'.format(chmod)) df_dict = { self.key_dotfile_src: src, self.key_dotfile_dst: dst, } + # link dfl = self.settings[self.key_settings_link_dotfile_default] if str(link) != dfl: df_dict[self.key_dotfile_link] = str(link) + + # chmod + if chmod: + df_dict[self.key_dotfile_chmod] = str(format(chmod, 'o')) + + # add to global dict self._yaml_dict[self.key_dotfiles][key] = df_dict self._dirty = True + return True def del_dotfile(self, key): """remove this dotfile from config""" @@ -593,7 +625,7 @@ class CfgYaml: return new def _norm_dotfiles(self, dotfiles): - """normalize dotfiles entries""" + """normalize and check dotfiles entries""" if not dotfiles: return dotfiles new = {} @@ -623,6 +655,27 @@ class CfgYaml: if self.key_dotfile_template not in v: val = self.settings.get(self.key_settings_template, True) v[self.key_dotfile_template] = val + # validate value of chmod if defined + if self.key_dotfile_chmod in v: + val = str(v[self.key_dotfile_chmod]) + if len(val) < 3: + err = 'bad format for chmod: {}'.format(val) + self._log.err(err) + raise YamlException('config content error: {}'.format(err)) + try: + int(val) + except Exception: + err = 'bad format for chmod: {}'.format(val) + self._log.err(err) + raise YamlException('config content error: {}'.format(err)) + for x in list(val): + y = int(x) + if y >= 0 or y <= 7: + continue + err = 'bad format for chmod: {}'.format(val) + self._log.err(err) + raise YamlException('config content error: {}'.format(err)) + v[self.key_dotfile_chmod] = int(val, 8) return new diff --git a/dotdrop/comparator.py b/dotdrop/comparator.py index f29351f..634a489 100644 --- a/dotdrop/comparator.py +++ b/dotdrop/comparator.py @@ -10,7 +10,8 @@ import filecmp # local imports from dotdrop.logger import Logger -from dotdrop.utils import must_ignore, uniq_list, diff +from dotdrop.utils import must_ignore, uniq_list, diff, \ + get_file_perm class Comparator: @@ -31,6 +32,7 @@ class Comparator: if self.debug: self.log.dbg('comparing {} and {}'.format(left, right)) self.log.dbg('ignore pattern(s): {}'.format(ignore)) + # test type of file if os.path.isdir(left) and not os.path.isdir(right): return '\"{}\" is a dir while \"{}\" is a file\n'.format(left, @@ -38,14 +40,37 @@ class Comparator: if not os.path.isdir(left) and os.path.isdir(right): return '\"{}\" is a file while \"{}\" is a dir\n'.format(left, right) + # test content if not os.path.isdir(left): + if self.debug: + self.log.dbg('{} is a file'.format(left)) if self.debug: self.log.dbg('is file') - return self._comp_file(left, right, ignore) + ret = self._comp_file(left, right, ignore) + if not ret: + ret = self._comp_mode(left, right) + return ret + if self.debug: - self.log.dbg('is directory') - return self._comp_dir(left, right, ignore) + self.log.dbg('{} is a directory'.format(left)) + + ret = self._comp_dir(left, right, ignore) + if not ret: + ret = self._comp_mode(left, right) + return ret + + def _comp_mode(self, left, right): + """compare mode""" + left_mode = get_file_perm(left) + right_mode = get_file_perm(right) + if left_mode == right_mode: + return '' + if self.debug: + msg = 'mode differ {} ({:o}) and {} ({:o})' + self.log.dbg(msg.format(left, left_mode, right, right_mode)) + ret = 'modes differ for {} ({:o}) vs {:o}\n' + return ret.format(right, right_mode, left_mode) def _comp_file(self, left, right, ignore): """compare a file""" @@ -123,7 +148,7 @@ class Comparator: def _diff(self, left, right, header=False): """diff two files""" - out = diff(modified=left, original=right, raw=False, + out = diff(modified=left, original=right, diff_cmd=self.diff_cmd, debug=self.debug) if header: lshort = os.path.basename(left) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 6c06eee..f1ea6e6 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -9,7 +9,6 @@ import os import sys import time from concurrent import futures -import shutil # local imports from dotdrop.options import Options @@ -18,8 +17,10 @@ from dotdrop.templategen import Templategen from dotdrop.installer import Installer from dotdrop.updater import Updater from dotdrop.comparator import Comparator -from dotdrop.utils import get_tmpdir, removepath, strip_home, \ - uniq_list, patch_ignores, dependencies_met +from dotdrop.importer import Importer +from dotdrop.utils import get_tmpdir, removepath, \ + uniq_list, patch_ignores, dependencies_met, \ + adapt_workers from dotdrop.linktypes import LinkTypes from dotdrop.exceptions import YamlException, UndefinedException @@ -71,6 +72,115 @@ def action_executor(o, actions, defactions, templater, post=False): return execute +def _dotfile_update(o, path, key=False): + """ + update a dotfile pointed by path + if key is false or by key (in path) + """ + updater = Updater(o.dotpath, o.variables, o.conf, + dry=o.dry, safe=o.safe, debug=o.debug, + ignore=o.update_ignore, + showpatch=o.update_showpatch) + if key: + return updater.update_key(path) + return updater.update_path(path) + + +def _dotfile_compare(o, dotfile, tmp): + """ + compare a dotfile + returns True if same + """ + t = _get_templater(o) + inst = Installer(create=o.create, backup=o.backup, + dry=o.dry, base=o.dotpath, + workdir=o.workdir, debug=o.debug, + backup_suffix=o.install_backup_suffix, + diff_cmd=o.diff_command) + comp = Comparator(diff_cmd=o.diff_command, debug=o.debug) + + # add dotfile variables + newvars = dotfile.get_dotfile_variables() + t.add_tmp_vars(newvars=newvars) + + # dotfiles does not exist / not installed + if o.debug: + LOG.dbg('comparing {}'.format(dotfile)) + + src = dotfile.src + if not os.path.lexists(os.path.expanduser(dotfile.dst)): + line = '=> compare {}: \"{}\" does not exist on destination' + LOG.log(line.format(dotfile.key, dotfile.dst)) + return False + + # apply transformation + tmpsrc = None + if dotfile.trans_r: + if o.debug: + LOG.dbg('applying transformation before comparing') + tmpsrc = apply_trans(o.dotpath, dotfile, t, debug=o.debug) + if not tmpsrc: + # could not apply trans + return False + src = tmpsrc + + # is a symlink pointing to itself + asrc = os.path.join(o.dotpath, os.path.expanduser(src)) + adst = os.path.expanduser(dotfile.dst) + if os.path.samefile(asrc, adst): + if o.debug: + line = '=> compare {}: diffing with \"{}\"' + LOG.dbg(line.format(dotfile.key, dotfile.dst)) + LOG.dbg('points to itself') + return True + + insttmp = None + if dotfile.template and Templategen.is_template(src): + # install dotfile to temporary dir for compare + ret, err, insttmp = inst.install_to_temp(t, tmp, src, dotfile.dst, + is_template=True, + chmod=dotfile.chmod) + if not ret: + # failed to install to tmp + line = '=> compare {} error: {}' + LOG.log(line.format(dotfile.key, err)) + LOG.err(err) + return False + src = insttmp + + # compare + ignores = list(set(o.compare_ignore + dotfile.cmpignore)) + ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) + diff = comp.compare(src, dotfile.dst, ignore=ignores) + + # clean tmp transformed dotfile if any + if tmpsrc: + tmpsrc = os.path.join(o.dotpath, tmpsrc) + if os.path.exists(tmpsrc): + removepath(tmpsrc, LOG) + + # clean tmp template dotfile if any + if insttmp: + if os.path.exists(insttmp): + removepath(insttmp, LOG) + + if diff != '': + # print diff results + line = '=> compare {}: diffing with \"{}\"' + LOG.log(line.format(dotfile.key, dotfile.dst)) + if o.compare_fileonly: + LOG.raw('') + else: + LOG.emph(diff) + return False + # no difference + if o.debug: + line = '=> compare {}: diffing with \"{}\"' + LOG.dbg(line.format(dotfile.key, dotfile.dst)) + LOG.dbg('same file') + return True + + def _dotfile_install(o, dotfile, tmpdir=None): """ install a dotfile @@ -97,17 +207,22 @@ def _dotfile_install(o, dotfile, tmpdir=None): LOG.dbg('installing dotfile: \"{}\"'.format(dotfile.key)) LOG.dbg(dotfile.prt()) + is_template = dotfile.template and Templategen.is_template(dotfile.src) if hasattr(dotfile, 'link') and dotfile.link == LinkTypes.LINK: # link - r, err = inst.link(t, dotfile.src, dotfile.dst, - actionexec=pre_actions_exec, - template=dotfile.template) + r, err = inst.install(t, dotfile.src, dotfile.dst, + dotfile.link, + actionexec=pre_actions_exec, + is_template=is_template, + chmod=dotfile.chmod) elif hasattr(dotfile, 'link') and \ dotfile.link == LinkTypes.LINK_CHILDREN: # link_children - r, err = inst.link_children(t, dotfile.src, dotfile.dst, - actionexec=pre_actions_exec, - template=dotfile.template) + r, err = inst.install(t, dotfile.src, dotfile.dst, + dotfile.link, + actionexec=pre_actions_exec, + is_template=is_template, + chmod=dotfile.chmod) else: # nolink src = dotfile.src @@ -120,10 +235,12 @@ def _dotfile_install(o, dotfile, tmpdir=None): ignores = list(set(o.install_ignore + dotfile.instignore)) ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) r, err = inst.install(t, src, dotfile.dst, + LinkTypes.NOLINK, actionexec=pre_actions_exec, noempty=dotfile.noempty, ignore=ignores, - template=dotfile.template) + is_template=is_template, + chmod=dotfile.chmod) if tmp: tmp = os.path.join(o.dotpath, tmp) if os.path.exists(tmp): @@ -161,6 +278,9 @@ def cmd_install(o): """install dotfiles for this profile""" dotfiles = o.dotfiles prof = o.conf.get_profile() + + adapt_workers(o, LOG) + pro_pre_actions = prof.get_pre_actions() if prof else [] pro_post_actions = prof.get_post_actions() if prof else [] @@ -189,14 +309,23 @@ def cmd_install(o): return False # install each dotfile - if o.install_parallel > 1: + if o.workers > 1: # in parallel - ex = futures.ThreadPoolExecutor(max_workers=o.install_parallel) + if o.debug: + LOG.dbg('run with {} workers'.format(o.workers)) + ex = futures.ThreadPoolExecutor(max_workers=o.workers) - wait_for = [ - ex.submit(_dotfile_install, o, dotfile, tmpdir=tmpdir) - for dotfile in dotfiles - ] + wait_for = [] + for dotfile in dotfiles: + if not dotfile.src or not dotfile.dst: + # fake dotfile are always considered installed + if o.debug: + LOG.dbg('fake dotfile installed') + installed += 1 + else: + j = ex.submit(_dotfile_install, o, dotfile, tmpdir=tmpdir) + wait_for.append(j) + # check result for f in futures.as_completed(wait_for): r, key, err = f.result() if r: @@ -207,7 +336,16 @@ def cmd_install(o): else: # sequentially for dotfile in dotfiles: - r, key, err = _dotfile_install(o, dotfile, tmpdir=tmpdir) + if not dotfile.src or not dotfile.dst: + # fake dotfile are always considered installed + if o.debug: + LOG.dbg('fake dotfile installed') + key = dotfile.key + r = True + err = None + else: + r, key, err = _dotfile_install(o, dotfile, tmpdir=tmpdir) + # check result if r: installed += 1 elif err: @@ -239,153 +377,101 @@ def cmd_compare(o, tmp): msg = 'no dotfile defined for this profile (\"{}\")' LOG.warn(msg.format(o.profile)) return True + # compare only specific files - same = True selected = dotfiles if o.compare_focus: selected = _select(o.compare_focus, dotfiles) if len(selected) < 1: + LOG.log('\nno dotfile to compare') return False - t = _get_templater(o) - tvars = t.add_tmp_vars() - inst = Installer(create=o.create, backup=o.backup, - dry=o.dry, base=o.dotpath, - workdir=o.workdir, debug=o.debug, - backup_suffix=o.install_backup_suffix, - diff_cmd=o.diff_command) - comp = Comparator(diff_cmd=o.diff_command, debug=o.debug) - - for dotfile in selected: - if not dotfile.src and not dotfile.dst: - # ignore fake dotfile - continue - # add dotfile variables - t.restore_vars(tvars) - newvars = dotfile.get_dotfile_variables() - t.add_tmp_vars(newvars=newvars) - - # dotfiles does not exist / not installed + same = True + cnt = 0 + if o.workers > 1: + # in parallel if o.debug: - LOG.dbg('comparing {}'.format(dotfile)) - src = dotfile.src - if not os.path.lexists(os.path.expanduser(dotfile.dst)): - line = '=> compare {}: \"{}\" does not exist on destination' - LOG.log(line.format(dotfile.key, dotfile.dst)) - same = False - continue - - # apply transformation - tmpsrc = None - if dotfile.trans_r: - if o.debug: - LOG.dbg('applying transformation before comparing') - tmpsrc = apply_trans(o.dotpath, dotfile, t, debug=o.debug) - if not tmpsrc: - # could not apply trans - same = False + LOG.dbg('run with {} workers'.format(o.workers)) + ex = futures.ThreadPoolExecutor(max_workers=o.workers) + wait_for = [] + for dotfile in selected: + j = ex.submit(_dotfile_compare, o, dotfile, tmp) + wait_for.append(j) + # check result + for f in futures.as_completed(wait_for): + if not dotfile.src and not dotfile.dst: + # ignore fake dotfile continue - src = tmpsrc - - # is a symlink pointing to itself - asrc = os.path.join(o.dotpath, os.path.expanduser(src)) - adst = os.path.expanduser(dotfile.dst) - if os.path.samefile(asrc, adst): - if o.debug: - line = '=> compare {}: diffing with \"{}\"' - LOG.dbg(line.format(dotfile.key, dotfile.dst)) - LOG.dbg('points to itself') - continue - - # install dotfile to temporary dir and compare - ret, err, insttmp = inst.install_to_temp(t, tmp, src, dotfile.dst, - template=dotfile.template) - if not ret: - # failed to install to tmp - line = '=> compare {}: error' - LOG.log(line.format(dotfile.key, err)) - LOG.err(err) - same = False - continue - ignores = list(set(o.compare_ignore + dotfile.cmpignore)) - ignores = patch_ignores(ignores, dotfile.dst, debug=o.debug) - diff = comp.compare(insttmp, dotfile.dst, ignore=ignores) - - # clean tmp transformed dotfile if any - if tmpsrc: - tmpsrc = os.path.join(o.dotpath, tmpsrc) - if os.path.exists(tmpsrc): - removepath(tmpsrc, LOG) - - if diff == '': - # no difference - if o.debug: - line = '=> compare {}: diffing with \"{}\"' - LOG.dbg(line.format(dotfile.key, dotfile.dst)) - LOG.dbg('same file') - else: - # print diff results - line = '=> compare {}: diffing with \"{}\"' - LOG.log(line.format(dotfile.key, dotfile.dst)) - if o.compare_fileonly: - LOG.raw('') - else: - LOG.emph(diff) - same = False + if not f.result(): + same = False + cnt += 1 + else: + # sequentially + for dotfile in selected: + if not dotfile.src and not dotfile.dst: + # ignore fake dotfile + continue + if not _dotfile_compare(o, dotfile, tmp): + same = False + cnt += 1 + LOG.log('\n{} dotfile(s) compared.'.format(cnt)) return same def cmd_update(o): """update the dotfile(s) from path(s) or key(s)""" - ret = True + cnt = 0 paths = o.update_path iskey = o.update_iskey - ignore = o.update_ignore - showpatch = o.update_showpatch + + adapt_workers(o, LOG) if not paths: # update the entire profile if iskey: + if o.debug: + LOG.dbg('update by keys: {}'.format(paths)) paths = [d.key for d in o.dotfiles] else: + if o.debug: + LOG.dbg('update by paths: {}'.format(paths)) paths = [d.dst for d in o.dotfiles] msg = 'Update all dotfiles for profile \"{}\"'.format(o.profile) if o.safe and not LOG.ask(msg): + LOG.log('\n{} file(s) updated.'.format(cnt)) return False if not paths: - LOG.log('no dotfile to update') + LOG.log('\nno dotfile to update') return True + if o.debug: LOG.dbg('dotfile to update: {}'.format(paths)) - updater = Updater(o.dotpath, o.variables, - o.conf.get_dotfile, - o.conf.get_dotfile_by_dst, - o.conf.path_to_dotfile_dst, - dry=o.dry, safe=o.safe, debug=o.debug, - ignore=ignore, showpatch=showpatch) - if not iskey: - # update paths + # update each dotfile + if o.workers > 1: + # in parallel if o.debug: - LOG.dbg('update by paths: {}'.format(paths)) + LOG.dbg('run with {} workers'.format(o.workers)) + ex = futures.ThreadPoolExecutor(max_workers=o.workers) + wait_for = [] for path in paths: - if not updater.update_path(path): - ret = False + j = ex.submit(_dotfile_update, o, path, key=iskey) + wait_for.append(j) + # check result + for f in futures.as_completed(wait_for): + if f.result(): + cnt += 1 else: - # update keys - keys = paths - if not keys: - # if not provided, take all keys - keys = [d.key for d in o.dotfiles] - if o.debug: - LOG.dbg('update by keys: {}'.format(keys)) - for key in keys: - if not updater.update_key(key): - ret = False - return ret + # sequentially + for path in paths: + if _dotfile_update(o, path, key=iskey): + cnt += 1 + + LOG.log('\n{} file(s) updated.'.format(cnt)) + return cnt == len(paths) def cmd_importer(o): @@ -393,119 +479,26 @@ def cmd_importer(o): ret = True cnt = 0 paths = o.import_path + importer = Importer(o.profile, o.conf, o.dotpath, o.diff_command, + dry=o.dry, safe=o.safe, debug=o.debug, + keepdot=o.keepdot) + for path in paths: - if o.debug: - LOG.dbg('trying to import {}'.format(path)) - if not os.path.exists(path): - LOG.err('\"{}\" does not exist, ignored!'.format(path)) + r = importer.import_path(path, import_as=o.import_as, + import_link=o.import_link, + import_mode=o.import_mode) + if r < 0: ret = False - continue - dst = path.rstrip(os.sep) - dst = os.path.abspath(dst) - - if o.safe: - # ask for symlinks - realdst = os.path.realpath(dst) - if dst != realdst: - msg = '\"{}\" is a symlink, dereference it and continue?' - if not LOG.ask(msg.format(dst)): - continue - - src = strip_home(dst) - if o.import_as: - # handle import as - src = os.path.expanduser(o.import_as) - src = src.rstrip(os.sep) - src = os.path.abspath(src) - src = strip_home(src) - if o.debug: - LOG.dbg('import src for {} as {}'.format(dst, src)) - - strip = '.' + os.sep - if o.keepdot: - strip = os.sep - src = src.lstrip(strip) - - # set the link attribute - linktype = o.import_link - if linktype == LinkTypes.LINK_CHILDREN and \ - not os.path.isdir(path): - LOG.err('importing \"{}\" failed!'.format(path)) - ret = False - continue - - if o.debug: - LOG.dbg('import dotfile: src:{} dst:{}'.format(src, dst)) - - # test no other dotfile exists with same - # dst for this profile but different src - dfs = o.conf.get_dotfile_by_dst(dst) - if dfs: - invalid = False - for df in dfs: - profiles = o.conf.get_profiles_by_dotfile_key(df.key) - profiles = [x.key for x in profiles] - if o.profile in profiles and \ - not o.conf.get_dotfile_by_src_dst(src, dst): - # same profile - # different src - LOG.err('duplicate dotfile for this profile') - ret = False - invalid = True - break - if invalid: - continue - - # prepare hierarchy for dotfile - srcf = os.path.join(o.dotpath, src) - overwrite = not os.path.exists(srcf) - if os.path.exists(srcf): - overwrite = True - if o.safe: - c = Comparator(debug=o.debug, diff_cmd=o.diff_command) - diff = c.compare(srcf, dst) - if diff != '': - # files are different, dunno what to do - LOG.log('diff \"{}\" VS \"{}\"'.format(dst, srcf)) - LOG.emph(diff) - # ask user - msg = 'Dotfile \"{}\" already exists, overwrite?' - overwrite = LOG.ask(msg.format(srcf)) - - if o.debug: - LOG.dbg('will overwrite: {}'.format(overwrite)) - if overwrite: - cmd = 'mkdir -p {}'.format(os.path.dirname(srcf)) - if o.dry: - LOG.dry('would run: {}'.format(cmd)) - else: - try: - os.makedirs(os.path.dirname(srcf), exist_ok=True) - except Exception: - LOG.err('importing \"{}\" failed!'.format(path)) - ret = False - continue - if o.dry: - LOG.dry('would copy {} to {}'.format(dst, srcf)) - else: - if os.path.isdir(dst): - if os.path.exists(srcf): - shutil.rmtree(srcf) - shutil.copytree(dst, srcf) - else: - shutil.copy2(dst, srcf) - retconf = o.conf.new(src, dst, linktype) - if retconf: - LOG.sub('\"{}\" imported'.format(path)) + elif r > 0: cnt += 1 - else: - LOG.warn('\"{}\" ignored'.format(path)) + if o.dry: LOG.dry('new config file would be:') LOG.raw(o.conf.dump()) else: o.conf.save() LOG.log('\n{} file(s) imported.'.format(cnt)) + return ret @@ -522,7 +515,7 @@ def cmd_list_profiles(o): LOG.log('') -def cmd_list_files(o): +def cmd_files(o): """list all dotfiles for a specific profile""" if o.profile not in [p.key for p in o.profiles]: LOG.warn('unknown profile \"{}\"'.format(o.profile)) @@ -540,12 +533,18 @@ def cmd_list_files(o): fmt = '{},dst:{},src:{},link:{}' fmt = fmt.format(dotfile.key, dotfile.dst, dotfile.src, dotfile.link.name.lower()) + if dotfile.chmod: + fmt += ',chmod:{:o}' + else: + fmt += ',chmod:None' LOG.raw(fmt) else: LOG.log('{}'.format(dotfile.key), bold=True) LOG.sub('dst: {}'.format(dotfile.dst)) LOG.sub('src: {}'.format(dotfile.src)) LOG.sub('link: {}'.format(dotfile.link.name.lower())) + if dotfile.chmod: + LOG.sub('chmod: {:o}'.format(dotfile.chmod)) LOG.log('') @@ -596,7 +595,8 @@ def cmd_remove(o): k = dotfile.key # ignore if uses any type of link if dotfile.link != LinkTypes.NOLINK: - LOG.warn('dotfile uses link, remove manually') + msg = '{} uses link/link_children, remove manually' + LOG.warn(msg.format(k)) continue if o.debug: @@ -679,12 +679,16 @@ def _get_templater(o): def _detail(dotpath, dotfile): """display details on all files under a dotfile entry""" - LOG.log('{} (dst: \"{}\", link: {})'.format(dotfile.key, dotfile.dst, - dotfile.link.name.lower())) + entry = '{}'.format(dotfile.key) + attribs = [] + attribs.append('dst: \"{}\"'.format(dotfile.dst)) + attribs.append('link: \"{}\"'.format(dotfile.link.name.lower())) + attribs.append('chmod: \"{}\"'.format(dotfile.chmod)) + LOG.log('{} ({})'.format(entry, ', '.join(attribs))) path = os.path.join(dotpath, os.path.expanduser(dotfile.src)) if not os.path.isdir(path): template = 'no' - if Templategen.is_template(path): + if dotfile.template and Templategen.is_template(path): template = 'yes' LOG.sub('{} (template:{})'.format(path, template)) else: @@ -692,7 +696,7 @@ def _detail(dotpath, dotfile): for f in files: p = os.path.join(root, f) template = 'no' - if Templategen.is_template(p): + if dotfile.template and Templategen.is_template(p): template = 'yes' LOG.sub('{} (template:{})'.format(p, template)) @@ -778,7 +782,7 @@ def main(): command = 'files' if o.debug: LOG.dbg('running cmd: {}'.format(command)) - cmd_list_files(o) + cmd_files(o) elif o.cmd_install: # install the dotfiles stored in dotdrop diff --git a/dotdrop/dotfile.py b/dotdrop/dotfile.py index da7890d..9c96434 100644 --- a/dotdrop/dotfile.py +++ b/dotdrop/dotfile.py @@ -22,7 +22,7 @@ class Dotfile(DictParser): actions=[], trans_r=None, trans_w=None, link=LinkTypes.NOLINK, noempty=False, cmpignore=[], upignore=[], - instignore=[], template=True): + instignore=[], template=True, chmod=None): """ constructor @key: dotfile key @@ -37,6 +37,7 @@ class Dotfile(DictParser): @cmpignore: patterns to ignore when comparing @instignore: patterns to ignore when installing @template: template this dotfile + @chmod: file permission """ self.actions = actions self.dst = dst @@ -50,6 +51,7 @@ class Dotfile(DictParser): self.cmpignore = cmpignore self.instignore = instignore self.template = template + self.chmod = chmod if self.link != LinkTypes.NOLINK and \ ( @@ -113,6 +115,8 @@ class Dotfile(DictParser): msg += ', dst:\"{}\"'.format(self.dst) msg += ', link:\"{}\"'.format(str(self.link)) msg += ', template:{}'.format(self.template) + if self.chmod: + msg += ', chmod:{:o}'.format(self.chmod) return msg def prt(self): @@ -123,6 +127,8 @@ class Dotfile(DictParser): out += '\n{}dst: \"{}\"'.format(indent, self.dst) out += '\n{}link: \"{}\"'.format(indent, str(self.link)) out += '\n{}template: \"{}\"'.format(indent, str(self.template)) + if self.chmod: + out += '\n{}chmod: \"{:o}\"'.format(indent, self.chmod) out += '\n{}pre-action:'.format(indent) some = self.get_pre_actions() diff --git a/dotdrop/importer.py b/dotdrop/importer.py new file mode 100644 index 0000000..5abaf3d --- /dev/null +++ b/dotdrop/importer.py @@ -0,0 +1,203 @@ +""" +author: deadc0de6 (https://github.com/deadc0de6) +Copyright (c) 2020, deadc0de6 + +handle import of dotfiles +""" + +import os +import shutil + +# local imports +from dotdrop.logger import Logger +from dotdrop.utils import strip_home, get_default_file_perms, \ + get_file_perm, get_umask +from dotdrop.linktypes import LinkTypes +from dotdrop.comparator import Comparator + + +class Importer: + + def __init__(self, profile, conf, dotpath, diff_cmd, + dry=False, safe=True, debug=False, + keepdot=True): + """constructor + @profile: the selected profile + @conf: configuration manager + @dotpath: dotfiles dotpath + @diff_cmd: diff command to use + @dry: simulate + @safe: ask for overwrite if True + @debug: enable debug + @keepdot: keep dot prefix + """ + self.profile = profile + self.conf = conf + self.dotpath = dotpath + self.diff_cmd = diff_cmd + self.dry = dry + self.safe = safe + self.debug = debug + self.keepdot = keepdot + + self.umask = get_umask() + self.log = Logger() + + def import_path(self, path, import_as=None, + import_link=LinkTypes.NOLINK, import_mode=False): + """ + import a dotfile pointed by path + returns: + 1: 1 dotfile imported + 0: ignored + -1: error + """ + if self.debug: + self.log.dbg('import {}'.format(path)) + if not os.path.exists(path): + self.log.err('\"{}\" does not exist, ignored!'.format(path)) + return -1 + + return self._import(path, import_as=import_as, + import_link=import_link, import_mode=import_mode) + + def _import(self, path, import_as=None, + import_link=LinkTypes.NOLINK, import_mode=False): + """ + import path + returns: + 1: 1 dotfile imported + 0: ignored + -1: error + """ + + # normalize path + dst = path.rstrip(os.sep) + dst = os.path.abspath(dst) + + # ask confirmation for symlinks + if self.safe: + realdst = os.path.realpath(dst) + if dst != realdst: + msg = '\"{}\" is a symlink, dereference it and continue?' + if not self.log.ask(msg.format(dst)): + return 0 + + # create src path + src = strip_home(dst) + if import_as: + # handle import as + src = os.path.expanduser(import_as) + src = src.rstrip(os.sep) + src = os.path.abspath(src) + src = strip_home(src) + if self.debug: + self.log.dbg('import src for {} as {}'.format(dst, src)) + # with or without dot prefix + strip = '.' + os.sep + if self.keepdot: + strip = os.sep + src = src.lstrip(strip) + + # get the permission + perm = get_file_perm(dst) + + # get the link attribute + linktype = import_link + if linktype == LinkTypes.LINK_CHILDREN and \ + not os.path.isdir(path): + self.log.err('importing \"{}\" failed!'.format(path)) + return -1 + + if self._already_exists(src, dst): + return -1 + + if self.debug: + self.log.dbg('import dotfile: src:{} dst:{}'.format(src, dst)) + + if not self._prepare_hierarchy(src, dst): + return -1 + + # handle file mode + chmod = None + dflperm = get_default_file_perms(dst, self.umask) + if self.debug: + self.log.dbg('import mode: {}'.format(import_mode)) + if import_mode or perm != dflperm: + if self.debug: + msg = 'adopt mode {:o} (umask {:o})' + self.log.dbg(msg.format(perm, dflperm)) + chmod = perm + + # add file to config file + retconf = self.conf.new_dotfile(src, dst, linktype, chmod=chmod) + if not retconf: + self.log.warn('\"{}\" ignored'.format(path)) + return 0 + + self.log.sub('\"{}\" imported'.format(path)) + return 1 + + def _prepare_hierarchy(self, src, dst): + """prepare hierarchy for dotfile""" + srcf = os.path.join(self.dotpath, src) + + # a dotfile in dotpath already exists at that spot + if os.path.exists(srcf): + if self.safe: + c = Comparator(debug=self.debug, + diff_cmd=self.diff_cmd) + diff = c.compare(srcf, dst) + if diff != '': + # files are different, dunno what to do + self.log.log('diff \"{}\" VS \"{}\"'.format(dst, srcf)) + self.log.emph(diff) + # ask user + msg = 'Dotfile \"{}\" already exists, overwrite?' + if not self.log.ask(msg.format(srcf)): + return False + if self.debug: + self.log.dbg('will overwrite existing file') + + # create directory hierarchy + cmd = 'mkdir -p {}'.format(os.path.dirname(srcf)) + if self.dry: + self.log.dry('would run: {}'.format(cmd)) + else: + try: + os.makedirs(os.path.dirname(srcf), exist_ok=True) + except Exception: + self.log.err('importing \"{}\" failed!'.format(dst)) + return False + + if self.dry: + self.log.dry('would copy {} to {}'.format(dst, srcf)) + else: + # copy the file to the dotpath + if os.path.isdir(dst): + if os.path.exists(srcf): + shutil.rmtree(srcf) + shutil.copytree(dst, srcf) + else: + shutil.copy2(dst, srcf) + + return True + + def _already_exists(self, src, dst): + """ + test no other dotfile exists with same + dst for this profile but different src + """ + dfs = self.conf.get_dotfile_by_dst(dst) + if not dfs: + return False + for df in dfs: + profiles = self.conf.get_profiles_by_dotfile_key(df.key) + profiles = [x.key for x in profiles] + if self.profile in profiles and \ + not self.conf.get_dotfile_by_src_dst(src, dst): + # same profile + # different src + self.log.err('duplicate dotfile for this profile') + return True + return False diff --git a/dotdrop/installer.py b/dotdrop/installer.py index 663ac6f..5d6c9c3 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -11,7 +11,7 @@ import shutil # local imports from dotdrop.logger import Logger -from dotdrop.templategen import Templategen +from dotdrop.linktypes import LinkTypes import dotdrop.utils as utils from dotdrop.exceptions import UndefinedException @@ -22,7 +22,7 @@ class Installer: dry=False, safe=False, workdir='~/.config/dotdrop', debug=False, diff=True, totemp=None, showdiff=False, backup_suffix='.dotdropbak', diff_cmd=''): - """constructor + """ @base: directory path where to search for templates @create: create directory hierarchy if missing when installing @backup: backup existing dotfile when installing @@ -40,7 +40,11 @@ class Installer: self.backup = backup self.dry = dry self.safe = safe - self.workdir = os.path.expanduser(workdir) + workdir = os.path.expanduser(workdir) + workdir = os.path.normpath(workdir) + self.workdir = workdir + base = os.path.expanduser(base) + base = os.path.normpath(base) self.base = base self.debug = debug self.diff = diff @@ -48,34 +52,33 @@ class Installer: self.showdiff = showdiff self.backup_suffix = backup_suffix self.diff_cmd = diff_cmd - self.comparing = False self.action_executed = False + # avoids printing file copied logs + # when using install_to_tmp for comparing + self.comparing = False + self.log = Logger() - def _log_install(self, boolean, err): - if not self.debug: - return boolean, err - if boolean: - self.log.dbg('install: SUCCESS') - else: - if err: - self.log.dbg('install: ERROR: {}'.format(err)) - else: - self.log.dbg('install: IGNORED') - return boolean, err + ######################################################## + # public methods + ######################################################## - def install(self, templater, src, dst, + def install(self, templater, src, dst, linktype, actionexec=None, noempty=False, - ignore=[], template=True): + ignore=[], is_template=True, + chmod=None): """ - install src to dst using a template + install src to dst + @templater: the templater object @src: dotfile source path in dotpath @dst: dotfile destination path in the FS + @linktype: linktypes.LinkTypes @actionexec: action executor callback @noempty: render empty template flag @ignore: pattern to ignore when installing - @template: template this dotfile + @is_template: this dotfile is a template + @chmod: rights to apply if any return - True, None : success @@ -83,126 +86,200 @@ class Installer: - False, None : ignored """ if self.debug: - self.log.dbg('installing \"{}\" to \"{}\"'.format(src, dst)) - if not dst or not src: - if self.debug: - self.log.dbg('empty dst for {}'.format(src)) - return self._log_install(True, None) - self.action_executed = False - src = os.path.join(self.base, os.path.expanduser(src)) + msg = 'installing \"{}\" to \"{}\" (link: {})' + self.log.dbg(msg.format(src, dst, str(linktype))) + src, dst, cont, err = self._check_paths(src, dst, chmod) + if not cont: + return self._log_install(cont, err) + + # check source file exists + src = os.path.join(self.base, src) if not os.path.exists(src): err = 'source dotfile does not exist: {}'.format(src) return self._log_install(False, err) - dst = os.path.expanduser(dst) + + self.action_executed = False + + # install to temporary dir + # and ignore any actions if self.totemp: - dst = self._pivot_path(dst, self.totemp) - if utils.samefile(src, dst): - # symlink loop - err = 'dotfile points to itself: {}'.format(dst) - return self._log_install(False, err) + r, err, _ = self.install_to_temp(templater, self.totemp, + src, dst, is_template=is_template, + chmod=chmod) + return self._log_install(r, err) + isdir = os.path.isdir(src) if self.debug: self.log.dbg('install {} to {}'.format(src, dst)) - self.log.dbg('is a directory \"{}\": {}'.format(src, isdir)) - if isdir: - b, e = self._install_dir(templater, src, dst, - actionexec=actionexec, - noempty=noempty, ignore=ignore, - template=template) - return self._log_install(b, e) - b, e = self._install_file(templater, src, dst, - actionexec=actionexec, - noempty=noempty, ignore=ignore, - template=template) - return self._log_install(b, e) + self.log.dbg('\"{}\" is a directory: {}'.format(src, isdir)) - def link(self, templater, src, dst, actionexec=None, template=True): + if linktype == LinkTypes.NOLINK: + # normal file + if isdir: + r, err = self._copy_dir(templater, src, dst, + actionexec=actionexec, + noempty=noempty, ignore=ignore, + is_template=is_template, + chmod=chmod) + else: + r, err = self._copy_file(templater, src, dst, + actionexec=actionexec, + noempty=noempty, ignore=ignore, + is_template=is_template, + chmod=chmod) + elif linktype == LinkTypes.LINK: + # symlink + r, err = self._link(templater, src, dst, + actionexec=actionexec, + is_template=is_template) + elif linktype == LinkTypes.LINK_CHILDREN: + # symlink direct children + if not isdir: + if self.debug: + msg = 'symlink children of {} to {}' + self.log.dbg(msg.format(src, dst)) + err = 'source dotfile is not a directory: {}'.format(src) + r = False + else: + r, err = self._link_children(templater, src, dst, + actionexec=actionexec, + is_template=is_template) + + if self.debug: + self.log.dbg('before chmod: {} err:{}'.format(r, err)) + + if self.dry: + return self._log_install(r, err) + + # handle chmod + # - on success (r, not err) + # - no change (not r, not err) + # but not when + # - error (not r, err) + # - aborted (not r, err) + if (r or (not r and not err)): + if not chmod: + chmod = utils.get_file_perm(src) + dstperms = utils.get_file_perm(dst) + if dstperms != chmod: + # apply mode + msg = 'chmod {} to {:o}'.format(dst, chmod) + if self.safe and not self.log.ask(msg): + r = False + err = 'aborted' + else: + if not self.comparing: + self.log.sub('chmod {} to {:o}'.format(dst, chmod)) + if utils.chmod(dst, chmod, debug=self.debug): + r = True + else: + r = False + err = 'chmod failed' + + return self._log_install(r, err) + + def install_to_temp(self, templater, tmpdir, src, dst, + is_template=True, chmod=None): """ - set src as the link target of dst - @templater: the templater + install a dotfile to a tempdir + + @templater: the templater object + @tmpdir: where to install @src: dotfile source path in dotpath @dst: dotfile destination path in the FS - @actionexec: action executor callback - @template: template this dotfile + @is_template: this dotfile is a template + @chmod: rights to apply if any + + return + - success, error-if-any, dotfile-installed-path + """ + if self.debug: + self.log.dbg('tmp install {} (defined dst: {})'.format(src, dst)) + src, dst, cont, err = self._check_paths(src, dst, chmod) + if not cont: + return self._log_install(cont, err) + + ret = False + tmpdst = '' + + # save flags + self.comparing = True + drysaved = self.dry + self.dry = False + diffsaved = self.diff + self.diff = False + createsaved = self.create + self.create = True + totemp = self.totemp + self.totemp = None + + # install the dotfile to a temp directory + tmpdst = self._pivot_path(dst, tmpdir) + ret, err = self.install(templater, src, tmpdst, + LinkTypes.NOLINK, + is_template=is_template, + chmod=chmod) + if self.debug: + if ret: + self.log.dbg('tmp installed in {}'.format(tmpdst)) + + # restore flags + self.dry = drysaved + self.diff = diffsaved + self.create = createsaved + self.comparing = False + self.totemp = totemp + + return ret, err, tmpdst + + ######################################################## + # low level accessors for public methods + ######################################################## + + def _link(self, templater, src, dst, actionexec=None, is_template=True): + """ + install link:link return - True, None : success - False, error_msg : error - False, None : ignored + - False, 'aborted' : user aborted """ - if self.debug: - self.log.dbg('link \"{}\" to \"{}\"'.format(src, dst)) - if not dst or not src: + if is_template: if self.debug: - self.log.dbg('empty dst for {}'.format(src)) - return self._log_install(True, None) - self.action_executed = False - src = os.path.normpath(os.path.join(self.base, - os.path.expanduser(src))) - if not os.path.exists(src): - err = 'source dotfile does not exist: {}'.format(src) - return self._log_install(False, err) - dst = os.path.normpath(os.path.expanduser(dst)) - if self.totemp: - # ignore actions - b, e = self.install(templater, src, dst, actionexec=None, - template=template) - return self._log_install(b, e) - - if template and Templategen.is_template(src): - if self.debug: - self.log.dbg('dotfile is a template') - self.log.dbg('install to {} and symlink'.format(self.workdir)) + self.log.dbg('is a template') + self.log.dbg('install to {}'.format(self.workdir)) tmp = self._pivot_path(dst, self.workdir, striphome=True) - i, err = self.install(templater, src, tmp, actionexec=actionexec, - template=template) - if not i and not os.path.exists(tmp): - return self._log_install(i, err) + r, err = self.install(templater, src, tmp, + LinkTypes.NOLINK, + actionexec=actionexec, + is_template=is_template) + if not r and not os.path.exists(tmp): + return r, err src = tmp - b, e = self._link(src, dst, actionexec=actionexec) - return self._log_install(b, e) + r, err = self._symlink(src, dst, actionexec=actionexec) + return r, err - def link_children(self, templater, src, dst, actionexec=None, - template=True): + def _link_children(self, templater, src, dst, + actionexec=None, is_template=True): """ - link all files under a given directory - @templater: the templater - @src: dotfile source path in dotpath - @dst: dotfile destination path in the FS - @actionexec: action executor callback - @template: template this dotfile + install link:link_children return - - True, None: success - - False, error_msg: error - - False, None, ignored + - True, None : success + - False, error_msg : error + - False, None : ignored + - False, 'aborted' : user aborted """ - if self.debug: - self.log.dbg('link_children \"{}\" to \"{}\"'.format(src, dst)) - if not dst or not src: - if self.debug: - self.log.dbg('empty dst for {}'.format(src)) - return self._log_install(True, None) - self.action_executed = False - parent = os.path.join(self.base, os.path.expanduser(src)) - - # Fail if source doesn't exist - if not os.path.exists(parent): - err = 'source dotfile does not exist: {}'.format(parent) - return self._log_install(False, err) - - # Fail if source not a directory - if not os.path.isdir(parent): - if self.debug: - self.log.dbg('symlink children of {} to {}'.format(src, dst)) - - err = 'source dotfile is not a directory: {}'.format(parent) - return self._log_install(False, err) - - dst = os.path.normpath(os.path.expanduser(dst)) + parent = os.path.join(self.base, src) if not os.path.lexists(dst): - self.log.sub('creating directory "{}"'.format(dst)) - os.makedirs(dst) + if self.dry: + self.log.dry('would create directory "{}"'.format(dst)) + else: + if not self.comparing: + self.log.sub('creating directory "{}"'.format(dst)) + self._create_dirs(dst) if os.path.isfile(dst): msg = ''.join([ @@ -211,10 +288,9 @@ class Installer: ]).format(dst) if self.safe and not self.log.ask(msg): - err = 'ignoring "{}", nothing installed'.format(dst) - return self._log_install(False, err) + return False, 'aborted' os.unlink(dst) - os.mkdir(dst) + self._create_dirs(dst) children = os.listdir(parent) srcs = [os.path.normpath(os.path.join(parent, child)) @@ -224,25 +300,27 @@ class Installer: installed = 0 for i in range(len(children)): - src = srcs[i] - dst = dsts[i] + subsrc = srcs[i] + subdst = dsts[i] if self.debug: - self.log.dbg('symlink child {} to {}'.format(src, dst)) + self.log.dbg('symlink child {} to {}'.format(subsrc, subdst)) - if template and Templategen.is_template(src): + if is_template: if self.debug: - self.log.dbg('dotfile is a template') + self.log.dbg('child is a template') self.log.dbg('install to {} and symlink' .format(self.workdir)) - tmp = self._pivot_path(dst, self.workdir, striphome=True) - r, e = self.install(templater, src, tmp, actionexec=actionexec, - template=template) + tmp = self._pivot_path(subdst, self.workdir, striphome=True) + r, e = self.install(templater, subsrc, tmp, + LinkTypes.NOLINK, + actionexec=actionexec, + is_template=is_template) if not r and e and not os.path.exists(tmp): continue - src = tmp + subsrc = tmp - ret, err = self._link(src, dst, actionexec=actionexec) + ret, err = self._symlink(subsrc, subdst, actionexec=actionexec) if ret: installed += 1 # void actionexec if dotfile installed @@ -250,18 +328,23 @@ class Installer: actionexec = None else: if err: - return self._log_install(ret, err) + return ret, err - return self._log_install(installed > 0, None) + return installed > 0, None - def _link(self, src, dst, actionexec=None): + ######################################################## + # file operations + ######################################################## + + def _symlink(self, src, dst, actionexec=None): """ set src as a link target of dst return - - True, None: success - - False, error_msg: error - - False, None, ignored + - True, None : success + - False, error_msg : error + - False, None : ignored + - False, 'aborted' : user aborted """ overwrite = not self.safe if os.path.lexists(dst): @@ -274,11 +357,10 @@ class Installer: self.log.dry('would remove {} and link to {}'.format(dst, src)) return True, None if self.showdiff: - self._diff_before_write(src, dst, quiet=False) + self._show_diff_before_write(src, dst) msg = 'Remove "{}" for link creation?'.format(dst) if self.safe and not self.log.ask(msg): - err = 'ignoring "{}", link was not created'.format(dst) - return False, err + return False, 'aborted' overwrite = True try: utils.removepath(dst) @@ -299,41 +381,49 @@ class Installer: if os.path.lexists(dst): msg = 'Remove "{}" for link creation?'.format(dst) if self.safe and not overwrite and not self.log.ask(msg): - err = 'ignoring "{}", link was not created'.format(dst) - return False, err + return False, 'aborted' try: utils.removepath(dst) except OSError as e: err = 'something went wrong with {}: {}'.format(src, e) return False, err os.symlink(src, dst) - self.log.sub('linked {} to {}'.format(dst, src)) + if not self.comparing: + self.log.sub('linked {} to {}'.format(dst, src)) return True, None - def _get_tmp_file_vars(self, src, dst): - tmp = {} - tmp['_dotfile_sub_abs_src'] = src - tmp['_dotfile_sub_abs_dst'] = dst - return tmp + def _copy_file(self, templater, src, dst, + actionexec=None, noempty=False, + ignore=[], is_template=True, + chmod=None): + """ + install src to dst when is a file - def _install_file(self, templater, src, dst, - actionexec=None, noempty=False, - ignore=[], template=True): - """install src to dst when is a file""" + return + - True, None : success + - False, error_msg : error + - False, None : ignored + - False, 'aborted' : user aborted + """ if self.debug: self.log.dbg('deploy file: {}'.format(src)) self.log.dbg('ignore empty: {}'.format(noempty)) self.log.dbg('ignore pattern: {}'.format(ignore)) - self.log.dbg('template: {}'.format(template)) + self.log.dbg('is_template: {}'.format(is_template)) self.log.dbg('no empty: {}'.format(noempty)) + # check no loop + if utils.samefile(src, dst): + err = 'dotfile points to itself: {}'.format(dst) + return False, err + if utils.must_ignore([src, dst], ignore, debug=self.debug): if self.debug: self.log.dbg('ignoring install of {} to {}'.format(src, dst)) return False, None if utils.samefile(src, dst): - # symlink loop + # loop err = 'dotfile points to itself: {}'.format(dst) return False, err @@ -343,7 +433,7 @@ class Installer: # handle the file content = None - if template: + if is_template: # template the file saved = templater.add_tmp_vars(self._get_tmp_file_vars(src, dst)) try: @@ -352,6 +442,7 @@ class Installer: return False, str(e) finally: templater.restore_vars(saved) + # test is empty if noempty and utils.content_empty(content): if self.debug: self.log.dbg('ignoring empty template: {}'.format(src)) @@ -359,52 +450,53 @@ class Installer: if content is None: err = 'empty template {}'.format(src) return False, err + + # write the file ret, err = self._write(src, dst, content=content, actionexec=actionexec, - template=template) - - # build return values - if ret < 0: - # error - return False, err - if ret > 0: - # already exists - if self.debug: - self.log.dbg('ignoring {}'.format(dst)) - return False, None - if ret == 0: - # success + chmod=chmod) + if ret and not err: if not self.dry and not self.comparing: - self.log.sub('copied {} to {}'.format(src, dst)) - return True, None - # error - err = 'installing {} to {}'.format(src, dst) - return False, err + self.log.sub('install {} to {}'.format(src, dst)) + return ret, err - def _install_dir(self, templater, src, dst, - actionexec=None, noempty=False, - ignore=[], template=True): - """install src to dst when is a directory""" + def _copy_dir(self, templater, src, dst, + actionexec=None, noempty=False, + ignore=[], is_template=True, chmod=None): + """ + install src to dst when is a directory + + return + - True, None : success + - False, error_msg : error + - False, None : ignored + - False, 'aborted' : user aborted + """ if self.debug: - self.log.dbg('install dir {}'.format(src)) - self.log.dbg('ignore empty: {}'.format(noempty)) + self.log.dbg('deploy dir {}'.format(src)) # default to nothing installed and no error ret = False, None + + # create the directory anyway if not self._create_dirs(dst): err = 'creating directory for {}'.format(dst) return False, err + # handle all files in dir for entry in os.listdir(src): f = os.path.join(src, entry) + if self.debug: + self.log.dbg('deploy sub from {}: {}'.format(dst, entry)) if not os.path.isdir(f): # is file - res, err = self._install_file(templater, f, - os.path.join(dst, entry), - actionexec=actionexec, - noempty=noempty, - ignore=ignore, - template=template) + res, err = self._copy_file(templater, f, + os.path.join(dst, entry), + actionexec=actionexec, + noempty=noempty, + ignore=ignore, + is_template=is_template, + chmod=None) if not res and err: # error occured ret = res, err @@ -414,12 +506,13 @@ class Installer: ret = True, None else: # is directory - res, err = self._install_dir(templater, f, - os.path.join(dst, entry), - actionexec=actionexec, - noempty=noempty, - ignore=ignore, - template=template) + res, err = self._copy_dir(templater, f, + os.path.join(dst, entry), + actionexec=actionexec, + noempty=noempty, + ignore=ignore, + is_template=is_template, + chmod=None) if not res and err: # error occured ret = res, err @@ -429,82 +522,67 @@ class Installer: ret = True, None return ret - def _fake_diff(self, dst, content): - """ - fake diff by comparing file content with content - returns True if same - """ - cur = '' - with open(dst, 'br') as f: - cur = f.read() - return cur == content - def _write(self, src, dst, content=None, - actionexec=None, template=True): + actionexec=None, chmod=None): """ copy dotfile / write content to file - return 0, None: for success, - 1, None: when already exists - -1, err: when error - content is always empty if template is False - and is to be ignored + + return + - True, None : success + - False, error_msg : error + - False, None : ignored + - False, 'aborted' : user aborted """ overwrite = not self.safe if self.dry: self.log.dry('would install {}'.format(dst)) - return 0, None + return True, None + if os.path.lexists(dst): - rights = os.stat(src).st_mode - samerights = False try: - samerights = os.stat(dst).st_mode == rights + os.stat(dst) except OSError as e: if e.errno == errno.ENOENT: # broken symlink err = 'broken symlink {}'.format(dst) - return -1, err - diff = None + return False, err + + src_mode = chmod + if not src_mode: + src_mode = utils.get_file_perm(src) if self.diff: - diff = self._diff_before_write(src, dst, - content=content, - quiet=True) - if not diff and samerights: + if not self._is_different(src, dst, content=content): if self.debug: self.log.dbg('{} is the same'.format(dst)) - return 1, None + return False, None if self.safe: if self.debug: self.log.dbg('change detected for {}'.format(dst)) if self.showdiff: - if diff is None: - # get diff - diff = self._diff_before_write(src, dst, - content=content, - quiet=True) - if diff: - self._print_diff(src, dst, diff) + # get diff + self._show_diff_before_write(src, dst, + content=content) if not self.log.ask('Overwrite \"{}\"'.format(dst)): - self.log.warn('ignoring {}'.format(dst)) - return 1, None + return False, 'aborted' overwrite = True if self.backup and os.path.lexists(dst): self._backup(dst) base = os.path.dirname(dst) if not self._create_dirs(base): err = 'creating directory for {}'.format(dst) - return -1, err + return False, err r, e = self._exec_pre_actions(actionexec) if not r: - return -1, e + return False, e if self.debug: - self.log.dbg('install dotfile to \"{}\"'.format(dst)) + self.log.dbg('install file to \"{}\"'.format(dst)) # re-check in case action created the file if self.safe and not overwrite and os.path.lexists(dst): if not self.log.ask('Overwrite \"{}\"'.format(dst)): self.log.warn('ignoring {}'.format(dst)) - return 1, None + return False, 'aborted' - if template: + if content: # write content the file try: with open(dst, 'wb') as f: @@ -512,19 +590,44 @@ class Installer: shutil.copymode(src, dst) except NotADirectoryError as e: err = 'opening dest file: {}'.format(e) - return -1, err + return False, err except Exception as e: - return -1, str(e) + return False, str(e) else: # copy file try: shutil.copyfile(src, dst) shutil.copymode(src, dst) except Exception as e: - return -1, str(e) - return 0, None + return False, str(e) + return True, None - def _diff_before_write(self, src, dst, content=None, quiet=False): + ######################################################## + # helpers + ######################################################## + + def _get_tmp_file_vars(self, src, dst): + tmp = {} + tmp['_dotfile_sub_abs_src'] = src + tmp['_dotfile_sub_abs_dst'] = dst + return tmp + + def _is_different(self, src, dst, content=None): + """ + returns True if file is different and + needs to be installed + """ + # check file content + if content: + tmp = utils.write_to_tmpfile(content) + src = tmp + r = utils.fastdiff(src, dst) + if r: + if self.debug: + self.log.dbg('content differ') + return r + + def _show_diff_before_write(self, src, dst, content=None): """ diff before writing using a temp file if content is not None @@ -534,12 +637,12 @@ class Installer: if content: tmp = utils.write_to_tmpfile(content) src = tmp - diff = utils.diff(modified=src, original=dst, raw=False, + diff = utils.diff(modified=src, original=dst, diff_cmd=self.diff_cmd) if tmp: utils.removepath(tmp, logger=self.log) - if not quiet and diff: + if diff: self._print_diff(src, dst, diff) return diff @@ -561,7 +664,10 @@ class Installer: return True if self.debug: self.log.dbg('mkdir -p {}'.format(directory)) - os.makedirs(directory) + if not self.comparing: + self.log.sub('create directory {}'.format(directory)) + + os.makedirs(directory, exist_ok=True) return os.path.exists(directory) def _backup(self, path): @@ -595,38 +701,36 @@ class Installer: self.action_executed = True return ret, err - def _install_to_temp(self, templater, src, dst, tmpdir, template=True): - """install a dotfile to a tempdir""" - tmpdst = self._pivot_path(dst, tmpdir) - r = self.install(templater, src, tmpdst, template=template) - return r, tmpdst + def _log_install(self, boolean, err): + """log installation process""" + if not self.debug: + return boolean, err + if boolean: + self.log.dbg('install: SUCCESS') + else: + if err: + self.log.dbg('install: ERROR: {}'.format(err)) + else: + self.log.dbg('install: IGNORED') + return boolean, err + + def _check_paths(self, src, dst, chmod): + """ + check and normalize param + returns , , , + """ + # check both path are valid + if not dst or not src: + err = 'empty dst or src for {}'.format(src) + if self.debug: + self.log.dbg(err) + return None, None, False, err - def install_to_temp(self, templater, tmpdir, src, dst, template=True): - """install a dotfile to a tempdir""" - ret = False - tmpdst = '' - # save some flags while comparing - self.comparing = True - drysaved = self.dry - self.dry = False - diffsaved = self.diff - self.diff = False - createsaved = self.create - self.create = True # normalize src and dst src = os.path.expanduser(src) + src = os.path.normpath(src) + dst = os.path.expanduser(dst) - if self.debug: - self.log.dbg('tmp install {} (defined dst: {})'.format(src, dst)) - # install the dotfile to a temp directory for comparing - r, tmpdst = self._install_to_temp(templater, src, dst, tmpdir, - template=template) - ret, err = r - if self.debug: - self.log.dbg('tmp installed in {}'.format(tmpdst)) - # reset flags - self.dry = drysaved - self.diff = diffsaved - self.comparing = False - self.create = createsaved - return ret, err, tmpdst + dst = os.path.normpath(dst) + + return src, dst, True, None diff --git a/dotdrop/options.py b/dotdrop/options.py index 8c549a1..7fc7680 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -25,6 +25,7 @@ ENV_NOBANNER = 'DOTDROP_NOBANNER' ENV_DEBUG = 'DOTDROP_DEBUG' ENV_NODEBUG = 'DOTDROP_FORCE_NODEBUG' ENV_XDG = 'XDG_CONFIG_HOME' +ENV_WORKERS = 'DOTDROP_WORKERS' BACKUP_SUFFIX = '.dotdropbak' PROFILE = socket.gethostname() @@ -54,12 +55,12 @@ USAGE = """ Usage: dotdrop install [-VbtfndDa] [-c ] [-p ] [-w ] [...] - dotdrop import [-Vbdf] [-c ] [-p ] [-s ] + dotdrop import [-Vbdfm] [-c ] [-p ] [-s ] [-l ] ... dotdrop compare [-LVb] [-c ] [-p ] - [-C ...] [-i ...] + [-w ] [-C ...] [-i ...] dotdrop update [-VbfdkP] [-c ] [-p ] - [-i ...] [...] + [-w ] [-i ...] [...] dotdrop remove [-Vbfdk] [-c ] [-p ] [...] dotdrop files [-VbTG] [-c ] [-p ] dotdrop detail [-Vb] [-c ] [-p ] [...] @@ -73,15 +74,16 @@ Options: -c --cfg= Path to the config. -C --file= Path of dotfile to compare. -d --dry Dry run. - -l --link= Link option (nolink|link|link_children). - -L --file-only Do not show diff but only the files that differ. - -p --profile= Specify the profile to use [default: {}]. -D --showdiff Show a diff before overwriting. -f --force Do not ask user confirmation for anything. -G --grepable Grepable output. -i --ignore= Pattern to ignore. -k --key Treat as a dotfile key. + -l --link= Link option (nolink|link|link_children). + -L --file-only Do not show diff but only the files that differ. + -m --preserve-mode Insert a chmod entry in the dotfile with its mode. -n --nodiff Do not diff when installing. + -p --profile= Specify the profile to use [default: {}]. -P --show-patch Provide a one-liner to manually patch template. -s --as= Import as a different path from actual path. -t --temp Install to a temporary directory for review. @@ -129,6 +131,9 @@ class Options(AttrMonitor): if not self.confpath: raise YamlException('no config file found') if self.debug: + self.log.dbg('#################################################') + self.log.dbg('#################### DOTDROP ####################') + self.log.dbg('#################################################') self.log.dbg('version: {}'.format(VERSION)) self.log.dbg('command: {}'.format(' '.join(sys.argv))) self.log.dbg('config file: {}'.format(self.confpath)) @@ -212,6 +217,16 @@ class Options(AttrMonitor): # adapt attributes based on arguments self.safe = not self.args['--force'] + try: + if ENV_WORKERS in os.environ: + workers = int(os.environ[ENV_WORKERS]) + else: + workers = int(self.args['--workers']) + self.workers = workers + except ValueError: + self.log.err('bad option for --workers') + sys.exit(USAGE) + # import link default value self.import_link = self.link_on_import if self.args['--link']: @@ -241,14 +256,6 @@ class Options(AttrMonitor): self.install_default_actions_post = [a for a in self.default_actions if a.kind == Action.post] self.install_ignore = self.instignore - try: - self.install_parallel = int(self.args['--workers']) - except ValueError: - self.log.err('bad option for --workers') - sys.exit(USAGE) - if self.safe and self.install_parallel > 1: - self.log.err('\"-w --workers\" must be used with \"-f --force\"') - sys.exit(USAGE) # "compare" specifics self.compare_focus = self.args['--file'] @@ -261,6 +268,7 @@ class Options(AttrMonitor): # "import" specifics self.import_path = self.args[''] self.import_as = self.args['--as'] + self.import_mode = self.args['--preserve-mode'] # "update" specifics self.update_path = self.args[''] diff --git a/dotdrop/settings.py b/dotdrop/settings.py index c29e97d..f1062dc 100644 --- a/dotdrop/settings.py +++ b/dotdrop/settings.py @@ -5,11 +5,16 @@ Copyright (c) 2019, deadc0de6 settings block """ +import os + # local imports from dotdrop.linktypes import LinkTypes from dotdrop.dictparser import DictParser +ENV_WORKDIR = 'DOTDROP_WORKDIR' + + class Settings(DictParser): # key in yaml file key_yaml = 'config' @@ -68,6 +73,8 @@ class Settings(DictParser): self.cmpignore = cmpignore self.instignore = instignore self.workdir = workdir + if ENV_WORKDIR in os.environ: + self.workdir = os.environ[ENV_WORKDIR] self.link_dotfile_default = LinkTypes.get(link_dotfile_default) self.link_on_import = LinkTypes.get(link_on_import) self.minversion = minversion diff --git a/dotdrop/templategen.py b/dotdrop/templategen.py index 3324131..fb164cb 100644 --- a/dotdrop/templategen.py +++ b/dotdrop/templategen.py @@ -6,6 +6,9 @@ jinja2 template generator """ import os +import io +import re +import mmap from jinja2 import Environment, FileSystemLoader, \ ChoiceLoader, FunctionLoader, TemplateNotFound, \ StrictUndefined @@ -154,7 +157,7 @@ class Templategen: except ImportError: # fallback _, filetype = utils.run(['file', '-b', '--mime-type', src], - raw=False, debug=self.debug) + debug=self.debug) if self.debug: self.log.dbg('using \"file\" for filetype identification') filetype = filetype.strip() @@ -245,16 +248,19 @@ class Templategen: """test if file pointed by path is a template""" if not os.path.isfile(path): return False + if os.stat(path).st_size == 0: + return False + markers = [BLOCK_START, VAR_START, COMMENT_START] + patterns = [re.compile(marker.encode()) for marker in markers] try: - with open(path, 'r') as f: - data = f.read() + with io.open(path, "r", encoding="utf-8") as f: + m = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + for pattern in patterns: + if pattern.search(m): + return True except UnicodeDecodeError: # is binary so surely no template return False - markers = [BLOCK_START, VAR_START, COMMENT_START] - for marker in markers: - if marker in data: - return True return False def _debug_dict(self, title, elems): diff --git a/dotdrop/updater.py b/dotdrop/updater.py index 3f92faf..15c8830 100644 --- a/dotdrop/updater.py +++ b/dotdrop/updater.py @@ -13,7 +13,7 @@ import filecmp from dotdrop.logger import Logger from dotdrop.templategen import Templategen from dotdrop.utils import patch_ignores, removepath, get_unique_tmp_name, \ - write_to_tmpfile, must_ignore, mirror_file_rights + write_to_tmpfile, must_ignore, mirror_file_rights, get_file_perm from dotdrop.exceptions import UndefinedException @@ -22,17 +22,13 @@ TILD = '~' class Updater: - def __init__(self, dotpath, variables, - dotfile_key_getter, dotfile_dst_getter, - dotfile_path_normalizer, - dry=False, safe=True, - debug=False, ignore=[], showpatch=False): + def __init__(self, dotpath, variables, conf, + dry=False, safe=True, debug=False, + ignore=[], showpatch=False): """constructor @dotpath: path where dotfiles are stored @variables: dictionary of variables for the templates - @dotfile_key_getter: func to get a dotfile by key - @dotfile_dst_getter: func to get a dotfile by dst - @dotfile_path_normalizer: func to normalize dotfile dst + @conf: configuration manager @dry: simulate @safe: ask for overwrite if True @debug: enable debug @@ -41,9 +37,7 @@ class Updater: """ self.dotpath = dotpath self.variables = variables - self.dotfile_key_getter = dotfile_key_getter - self.dotfile_dst_getter = dotfile_dst_getter - self.dotfile_path_normalizer = dotfile_path_normalizer + self.conf = conf self.dry = dry self.safe = safe self.debug = debug @@ -62,7 +56,7 @@ class Updater: if not os.path.lexists(path): self.log.err('\"{}\" does not exist!'.format(path)) return False - dotfiles = self.dotfile_dst_getter(path) + dotfiles = self.conf.get_dotfile_by_dst(path) if not dotfiles: return False for dotfile in dotfiles: @@ -80,12 +74,12 @@ class Updater: def update_key(self, key): """update the dotfile referenced by key""" - dotfile = self.dotfile_key_getter(key) + dotfile = self.conf.get_dotfile(key) if not dotfile: return False if self.debug: self.log.dbg('updating {} from key \"{}\"'.format(dotfile, key)) - path = self.dotfile_path_normalizer(dotfile.dst) + path = self.conf.path_to_dotfile_dst(dotfile.dst) return self._update(path, dotfile) def _update(self, path, dotfile): @@ -108,10 +102,26 @@ class Updater: new_path = self._apply_trans_w(path, dotfile) if not new_path: return False + + # save current rights + fsmode = get_file_perm(path) + dfmode = get_file_perm(dtpath) + + # handle the pointed file if os.path.isdir(new_path): ret = self._handle_dir(new_path, dtpath) else: ret = self._handle_file(new_path, dtpath) + + if fsmode != dfmode: + # mirror rights + if self.debug: + m = 'adopt mode {:o} for {}' + self.log.dbg(m.format(fsmode, dotfile.key)) + r = self.conf.update_dotfile(dotfile.key, fsmode) + if r: + ret = True + # clean temporary files if new_path != path and os.path.exists(new_path): removepath(new_path, logger=self.log) @@ -162,14 +172,21 @@ class Updater: def _same_rights(self, left, right): """return True if files have the same modes""" try: - lefts = os.stat(left) - rights = os.stat(right) - return lefts.st_mode == rights.st_mode + lefts = get_file_perm(left) + rights = get_file_perm(right) + return lefts == rights except OSError as e: self.log.err(e) return False def _mirror_rights(self, src, dst): + srcr = get_file_perm(src) + dstr = get_file_perm(dst) + if srcr == dstr: + return + if self.debug: + msg = 'copy rights from {} ({:o}) to {} ({:o})' + self.log.dbg(msg.format(src, srcr, dst, dstr)) try: mirror_file_rights(src, dst) except OSError as e: @@ -228,7 +245,9 @@ class Updater: # find the differences diff = filecmp.dircmp(path, dtpath, ignore=None) # handle directories diff - return self._merge_dirs(diff) + ret = self._merge_dirs(diff) + self._mirror_rights(path, dtpath) + return ret def _merge_dirs(self, diff): """Synchronize directories recursively.""" diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 16e5cd2..6e2fdab 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -12,6 +12,7 @@ import uuid import fnmatch import inspect import importlib +import filecmp from shutil import rmtree, which # local import @@ -32,7 +33,7 @@ DONOTDELETE = [ NOREMOVE = [os.path.normpath(p) for p in DONOTDELETE] -def run(cmd, raw=True, debug=False, checkerr=False): +def run(cmd, debug=False): """run a command (expects a list)""" if debug: LOG.dbg('exec: {}'.format(' '.join(cmd))) @@ -42,13 +43,6 @@ def run(cmd, raw=True, debug=False, checkerr=False): ret = p.returncode out = out.splitlines(keepends=True) lines = ''.join([x.decode('utf-8', 'replace') for x in out]) - if checkerr and ret != 0: - c = ' '.join(cmd) - errl = lines.rstrip() - m = '\"{}\" returned non zero ({}): {}'.format(c, ret, errl) - LOG.err(m) - if raw: - return ret == 0, out return ret == 0, lines @@ -73,7 +67,12 @@ def shell(cmd, debug=False): return ret == 0, out -def diff(original, modified, raw=True, +def fastdiff(left, right): + """fast compare files and returns True if different""" + return not filecmp.cmp(left, right, shallow=False) + + +def diff(original, modified, diff_cmd='', debug=False): """compare two files, returns '' if same""" if not diff_cmd: @@ -86,7 +85,7 @@ def diff(original, modified, raw=True, "{modified}": modified, } cmd = [replacements.get(x, x) for x in diff_cmd.split()] - _, out = run(cmd, raw=raw, debug=debug) + _, out = run(cmd, debug=debug) return out @@ -310,5 +309,44 @@ def dependencies_met(): def mirror_file_rights(src, dst): """mirror file rights of src to dst (can rise exc)""" - rights = os.stat(src).st_mode + if not os.path.exists(src) or not os.path.exists(dst): + return + rights = get_file_perm(src) os.chmod(dst, rights) + + +def get_umask(): + """return current umask value""" + cur = os.umask(0) + os.umask(cur) + # return 0o777 - cur + return cur + + +def get_default_file_perms(path, umask): + """get default rights for a file""" + base = 0o666 + if os.path.isdir(path): + base = 0o777 + return base - umask + + +def get_file_perm(path): + """return file permission""" + return os.stat(path).st_mode & 0o777 + + +def chmod(path, mode, debug=False): + if debug: + LOG.dbg('chmod {} {}'.format(oct(mode), path)) + os.chmod(path, mode) + return get_file_perm(path) == mode + + +def adapt_workers(options, logger): + if options.safe and options.workers > 1: + logger.warn('workers set to 1 when --force is not used') + options.workers = 1 + if options.dry and options.workers > 1: + logger.warn('workers set to 1 when --dry is used') + options.workers = 1 diff --git a/tests-ng/actions-pre.sh b/tests-ng/actions-pre.sh index 24ae575..5f8949b 100755 --- a/tests-ng/actions-pre.sh +++ b/tests-ng/actions-pre.sh @@ -46,6 +46,15 @@ echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" # this is the test ################################################################ +# $1 pattern +# $2 path +grep_or_fail() +{ + set +e + grep "${1}" "${2}" >/dev/null 2>&1 || (echo "pattern not found in ${2}" && exit 1) + set -e +} + # the action temp tmpa=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` # the dotfile source @@ -136,38 +145,36 @@ cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V # checks [ ! -e ${tmpa}/pre ] && echo 'pre action not executed' && exit 1 -grep pre ${tmpa}/pre >/dev/null +grep_or_fail pre ${tmpa}/pre [ ! -e ${tmpa}/naked ] && echo 'naked action not executed' && exit 1 -grep naked ${tmpa}/naked >/dev/null +grep_or_fail naked ${tmpa}/naked [ ! -e ${tmpa}/multiple ] && echo 'pre action multiple not executed' && exit 1 -grep multiple ${tmpa}/multiple >/dev/null +grep_or_fail multiple ${tmpa}/multiple [ "`wc -l ${tmpa}/multiple | awk '{print $1}'`" -gt "1" ] && echo 'pre action multiple executed twice' && exit 1 [ ! -e ${tmpa}/pre2 ] && echo 'pre action 2 not executed' && exit 1 -grep pre2 ${tmpa}/pre2 >/dev/null +grep_or_fail pre2 ${tmpa}/pre2 [ ! -e ${tmpa}/naked2 ] && echo 'naked action 2 not executed' && exit 1 -grep naked2 ${tmpa}/naked2 >/dev/null +grep_or_fail naked2 ${tmpa}/naked2 [ ! -e ${tmpa}/multiple2 ] && echo 'pre action multiple 2 not executed' && exit 1 -grep multiple2 ${tmpa}/multiple2 >/dev/null +grep_or_fail multiple2 ${tmpa}/multiple2 [ "`wc -l ${tmpa}/multiple2 | awk '{print $1}'`" -gt "1" ] && echo 'pre action multiple 2 executed twice' && exit 1 [ ! -e ${tmpa}/naked3 ] && echo 'naked action 3 not executed' && exit 1 -grep naked3 ${tmpa}/naked3 >/dev/null +grep_or_fail naked3 ${tmpa}/naked3 - -# remove the pre action result and re-run +# remove the pre action result and re-install rm ${tmpa}/pre - -cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -[ -e ${tmpa}/pre ] && exit 1 +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -V +[ -e ${tmpa}/pre ] && echo "pre exists" && exit 1 # ensure failing actions make the installation fail # install set +e cd ${ddpath} | ${bin} install -f -c ${cfg} -p p2 -V set -e -[ -e ${tmpd}/fail ] && exit 1 +[ -e ${tmpd}/fail ] && echo "fail exists" && exit 1 ## CLEANING rm -rf ${tmps} ${tmpd} ${tmpa} diff --git a/tests-ng/chmod-compare.sh b/tests-ng/chmod-compare.sh new file mode 100755 index 0000000..6a2b909 --- /dev/null +++ b/tests-ng/chmod-compare.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2020, deadc0de6 +# +# test chmod on compare +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +#echo "dotfile destination: ${tmpd}" + +# create the dotfile +dnormal="${tmpd}/dir_normal" +mkdir -p ${dnormal} +echo "dir_normal/f1" > ${dnormal}/file1 +echo "dir_normal/f2" > ${dnormal}/file2 +chmod 777 ${dnormal} + +dlink="${tmpd}/dir_link" +mkdir -p ${dlink} +echo "dir_link/f1" > ${dlink}/file1 +echo "dir_link/f2" > ${dlink}/file2 +chmod 777 ${dlink} + +dlinkchildren="${tmpd}/dir_link_children" +mkdir -p ${dlinkchildren} +echo "dir_linkchildren/f1" > ${dlinkchildren}/file1 +echo "dir_linkchildren/f2" > ${dlinkchildren}/file2 +chmod 777 ${dlinkchildren} + +fnormal="${tmpd}/filenormal" +echo "filenormal" > ${fnormal} +chmod 777 ${fnormal} + +flink="${tmpd}/filelink" +echo "filelink" > ${flink} +chmod 777 ${flink} + +toimport="${dnormal} ${dlink} ${dlinkchildren} ${fnormal} ${flink}" + +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: +profiles: +_EOF +#cat ${cfg} + +# import +for i in ${toimport}; do + cd ${ddpath} | ${bin} import -c ${cfg} -f -p p1 ${i} +done + +#cat ${cfg} + +# patch rights +chmod 700 ${dnormal} +chmod 700 ${dlink} +chmod 700 ${dlinkchildren} +chmod 700 ${fnormal} +chmod 700 ${flink} + +set +e +cnt=`cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 2>&1 | grep 'modes differ' | wc -l` +set -e +[ "${cnt}" != "5" ] && echo "compare modes failed (${cnt})" && exit 1 + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 diff --git a/tests-ng/chmod-import.sh b/tests-ng/chmod-import.sh new file mode 100755 index 0000000..6d1a5da --- /dev/null +++ b/tests-ng/chmod-import.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2020, deadc0de6 +# +# test chmod on import +# with files and directories +# with different link +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# $1 file +chmod_to_umask() +{ + u=`umask` + u=`echo ${u} | sed 's/^0*//'` + if [ -d ${1} ]; then + v=$((777 - u)) + else + v=$((666 - u)) + fi + chmod ${v} ${1} +} + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +#echo "dotfile destination: ${tmpd}" + +# create the dotfiles +dnormal="${tmpd}/dir_normal" +mkdir -p ${dnormal} +echo "dir_normal/f1" > ${dnormal}/file1 +echo "dir_normal/f2" > ${dnormal}/file2 +chmod 777 ${dnormal} + +dlink="${tmpd}/dir_link" +mkdir -p ${dlink} +echo "dir_link/f1" > ${dlink}/file1 +echo "dir_link/f2" > ${dlink}/file2 +chmod 777 ${dlink} + +dlinkchildren="${tmpd}/dir_link_children" +mkdir -p ${dlinkchildren} +echo "dir_linkchildren/f1" > ${dlinkchildren}/file1 +echo "dir_linkchildren/f2" > ${dlinkchildren}/file2 +chmod 777 ${dlinkchildren} + +fnormal="${tmpd}/filenormal" +echo "filenormal" > ${fnormal} +chmod 777 ${fnormal} + +flink="${tmpd}/filelink" +echo "filelink" > ${flink} +chmod 777 ${flink} + +toimport="${dnormal} ${dlink} ${dlinkchildren} ${fnormal} ${flink}" + +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: +profiles: +_EOF +#cat ${cfg} + +# import without --preserve-mode +for i in ${toimport}; do + cd ${ddpath} | ${bin} import -c ${cfg} -f -p p1 -V ${i} +done + +cat ${cfg} + +# list files +cd ${ddpath} | ${bin} detail -c ${cfg} -p p1 -V + +tot=`echo ${toimport} | wc -w` +cnt=`cat ${cfg} | grep "chmod: '777'" | wc -l` +[ "${cnt}" != "${tot}" ] && echo "not all chmod inserted (1)" && exit 1 + +## with link +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: +profiles: +_EOF + +# clean +rm -rf ${tmps}/dotfiles +mkdir -p ${tmps}/dotfiles + +# import without --preserve-mode and link +for i in ${toimport}; do + cd ${ddpath} | ${bin} import -c ${cfg} -l link -f -p p1 -V ${i} +done + +cat ${cfg} + +# list files +cd ${ddpath} | ${bin} detail -c ${cfg} -p p1 -V + +tot=`echo ${toimport} | wc -w` +cnt=`cat ${cfg} | grep "chmod: '777'" | wc -l` +[ "${cnt}" != "${tot}" ] && echo "not all chmod inserted (2)" && exit 1 + +tot=`echo ${toimport} | wc -w` +cnt=`cat ${cfg} | grep 'link: link' | wc -l` +[ "${cnt}" != "${tot}" ] && echo "not all link inserted" && exit 1 + +## --preserve-mode +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: +profiles: +_EOF + +# clean +rm -rf ${tmps}/dotfiles +mkdir -p ${tmps}/dotfiles + +# import with --preserve-mode +for i in ${toimport}; do + chmod_to_umask ${i} + cd ${ddpath} | ${bin} import -c ${cfg} -m -f -p p1 -V ${i} +done + +cat ${cfg} + +# list files +cd ${ddpath} | ${bin} detail -c ${cfg} -p p1 -V + +tot=`echo ${toimport} | wc -w` +cnt=`cat ${cfg} | grep "chmod: " | wc -l` +[ "${cnt}" != "${tot}" ] && echo "not all chmod inserted (3)" && exit 1 + +## import normal +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: +profiles: +_EOF + +# clean +rm -rf ${tmps}/dotfiles +mkdir -p ${tmps}/dotfiles + +# import without --preserve-mode +for i in ${toimport}; do + chmod_to_umask ${i} + cd ${ddpath} | ${bin} import -c ${cfg} -f -p p1 -V ${i} +done + +cat ${cfg} + +# list files +cd ${ddpath} | ${bin} detail -c ${cfg} -p p1 -V + +cnt=`cat ${cfg} | grep chmod | wc -l` +[ "${cnt}" != "0" ] && echo "chmod inserted but not needed" && exit 1 + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 diff --git a/tests-ng/chmod-install.sh b/tests-ng/chmod-install.sh new file mode 100755 index 0000000..e59562b --- /dev/null +++ b/tests-ng/chmod-install.sh @@ -0,0 +1,275 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2020, deadc0de6 +# +# test chmod on install +# with files and directories +# with different link +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# $1 path +# $2 rights +has_rights() +{ + echo "testing ${1} is ${2}" + [ ! -e "$1" ] && echo "`basename $1` does not exist" && exit 1 + local mode=`stat -L -c '%a' "$1"` + [ "${mode}" != "$2" ] && echo "bad mode for `basename $1` (${mode} VS expected ${2})" && exit 1 + true +} + +get_file_mode() +{ + u=`umask` + u=`echo ${u} | sed 's/^0*//'` + v=$((666 - u)) + echo "${v}" +} + +get_dir_mode() +{ + u=`umask` + u=`echo ${u} | sed 's/^0*//'` + v=$((777 - u)) + echo "${v}" +} + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +#echo "dotfile destination: ${tmpd}" + +# create the config file +cfg="${tmps}/config.yaml" + +echo 'f777' > ${tmps}/dotfiles/f777 +echo 'link' > ${tmps}/dotfiles/link +mkdir -p ${tmps}/dotfiles/dir +echo "f1" > ${tmps}/dotfiles/dir/f1 + +echo "exists" > ${tmps}/dotfiles/exists +chmod 644 ${tmps}/dotfiles/exists +echo "exists" > ${tmpd}/exists +chmod 644 ${tmpd}/exists + +echo "existslink" > ${tmps}/dotfiles/existslink +chmod 644 ${tmpd}/exists + +mkdir -p ${tmps}/dotfiles/direxists +echo "f1" > ${tmps}/dotfiles/direxists/f1 +mkdir -p ${tmpd}/direxists +echo "f1" > ${tmpd}/direxists/f1 +chmod 644 ${tmpd}/direxists/f1 +chmod 744 ${tmpd}/direxists + +mkdir -p ${tmps}/dotfiles/linkchildren +echo "f1" > ${tmps}/dotfiles/linkchildren/f1 +mkdir -p ${tmps}/dotfiles/linkchildren/d1 +echo "f2" > ${tmps}/dotfiles/linkchildren/d1/f2 + +echo '{{@@ profile @@}}' > ${tmps}/dotfiles/symlinktemplate + +mkdir -p ${tmps}/dotfiles/symlinktemplatedir +echo "{{@@ profile @@}}" > ${tmps}/dotfiles/symlinktemplatedir/t + +echo 'nomode' > ${tmps}/dotfiles/nomode + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: + f_f777: + src: f777 + dst: ${tmpd}/f777 + chmod: 777 + f_link: + src: link + dst: ${tmpd}/link + chmod: 777 + link: link + d_dir: + src: dir + dst: ${tmpd}/dir + chmod: 777 + f_exists: + src: exists + dst: ${tmpd}/exists + chmod: 777 + f_existslink: + src: existslink + dst: ${tmpd}/existslink + chmod: 777 + link: link + d_direxists: + src: direxists + dst: ${tmpd}/direxists + chmod: 777 + d_linkchildren: + src: linkchildren + dst: ${tmpd}/linkchildren + chmod: 777 + link: link_children + f_symlinktemplate: + src: symlinktemplate + dst: ${tmpd}/symlinktemplate + chmod: 777 + link: link + d_symlinktemplatedir: + src: symlinktemplatedir + dst: ${tmpd}/symlinktemplatedir + chmod: 777 + link: link + f_nomode: + src: nomode + dst: ${tmpd}/nomode +profiles: + p1: + dotfiles: + - f_f777 + - f_link + - d_dir + - f_exists + - f_existslink + - d_direxists + - d_linkchildren + - f_symlinktemplate + - d_symlinktemplatedir + - f_nomode + p2: + dotfiles: + - f_exists + - f_existslink + - d_linkchildren + - f_symlinktemplate + - f_nomode +_EOF +#cat ${cfg} + +# install +echo "first install round" +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 -V + +has_rights "${tmpd}/f777" "777" +has_rights "${tmpd}/link" "777" +has_rights "${tmpd}/dir" "777" +has_rights "${tmpd}/exists" "777" +has_rights "${tmpd}/existslink" "777" +has_rights "${tmpd}/direxists" "777" +has_rights "${tmpd}/direxists/f1" "644" +has_rights "${tmpd}/linkchildren" "777" +has_rights "${tmpd}/linkchildren/f1" "644" +has_rights "${tmpd}/linkchildren/d1" "755" +has_rights "${tmpd}/linkchildren/d1/f2" "644" +has_rights "${tmpd}/symlinktemplate" "777" +m=`get_file_mode` +has_rights "${tmpd}/nomode" "${m}" + +grep 'p1' ${tmpd}/symlinktemplate +grep 'p1' ${tmpd}/symlinktemplatedir/t + +## second round +echo "exists" > ${tmps}/dotfiles/exists +chmod 600 ${tmps}/dotfiles/exists +echo "exists" > ${tmpd}/exists +chmod 600 ${tmpd}/exists + +chmod 600 ${tmpd}/existslink + +chmod 700 ${tmpd}/linkchildren + +chmod 600 ${tmpd}/symlinktemplate + +echo "second install round" +cd ${ddpath} | ${bin} install -c ${cfg} -p p2 -f -V + +has_rights "${tmpd}/exists" "777" +has_rights "${tmpd}/existslink" "777" +has_rights "${tmpd}/linkchildren/f1" "644" +has_rights "${tmpd}/linkchildren/d1" "755" +has_rights "${tmpd}/linkchildren/d1/f2" "644" +has_rights "${tmpd}/symlinktemplate" "777" +m=`get_file_mode` +has_rights "${tmpd}/nomode" "${m}" + +## no user confirmation expected +## same mode +echo "same mode" +echo "nomode" > ${tmps}/dotfiles/nomode +chmod 600 ${tmps}/dotfiles/nomode +echo "nomode" > ${tmpd}/nomode +chmod 600 ${tmpd}/nomode +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p2 -V f_nomode +echo "same mode" +has_rights "${tmpd}/nomode" "600" + +## no user confirmation with force +## different mode +echo "different mode" +echo "nomode" > ${tmps}/dotfiles/nomode +chmod 600 ${tmps}/dotfiles/nomode +echo "nomode" > ${tmpd}/nomode +chmod 700 ${tmpd}/nomode +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p2 -V f_nomode +echo "different mode (1)" +has_rights "${tmpd}/nomode" "600" + +## user confirmation expected +## different mode +echo "different mode" +echo "nomode" > ${tmps}/dotfiles/nomode +chmod 600 ${tmps}/dotfiles/nomode +echo "nomode" > ${tmpd}/nomode +chmod 700 ${tmpd}/nomode +cd ${ddpath} | printf 'y\ny\n' | ${bin} install -f -c ${cfg} -p p2 -V f_nomode +echo "different mode (2)" +has_rights "${tmpd}/nomode" "600" + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 diff --git a/tests-ng/chmod-more.sh b/tests-ng/chmod-more.sh new file mode 100755 index 0000000..f78a719 --- /dev/null +++ b/tests-ng/chmod-more.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2020, deadc0de6 +# +# test chmod on import +# with files and directories +# with different link +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# $1 path +# $2 rights +has_rights() +{ + echo "testing ${1} is ${2}" + [ ! -e "$1" ] && echo "`basename $1` does not exist" && exit 1 + local mode=`stat -L -c '%a' "$1"` + [ "${mode}" != "$2" ] && echo "bad mode for `basename $1` (${mode} instead of ${2})" && exit 1 + true +} + +# $1 file +chmod_to_umask() +{ + u=`umask` + u=`echo ${u} | sed 's/^0*//'` + if [ -d ${1} ]; then + v=$((777 - u)) + else + v=$((666 - u)) + fi + chmod ${v} ${1} +} + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +#echo "dotfile destination: ${tmpd}" + +# create the dotfiles +f1="${tmpd}/f1" +touch ${f1} +chmod 777 ${f1} +stat -c '%a' ${f1} + +f2="${tmpd}/f2" +touch ${f2} +chmod 644 ${f2} +stat -c '%a' ${f2} + +toimport="${f1} ${f2}" + +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: +profiles: +_EOF +#cat ${cfg} + +# import without --preserve-mode +for i in ${toimport}; do + stat -c '%a' ${i} + cd ${ddpath} | ${bin} import -c ${cfg} -f -p p1 -V ${i} +done + +cat ${cfg} + +has_rights "${tmpd}/f1" "777" +has_rights "${tmps}/dotfiles/${tmpd}/f1" "777" +has_rights "${tmpd}/f2" "644" +has_rights "${tmps}/dotfiles/${tmpd}/f2" "644" + +# install +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 -V | grep '0 dotfile(s) installed' || (echo "should not install" && exit 1) + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 diff --git a/tests-ng/chmod-update.sh b/tests-ng/chmod-update.sh new file mode 100755 index 0000000..6e96a66 --- /dev/null +++ b/tests-ng/chmod-update.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2020, deadc0de6 +# +# test chmod on update +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +#echo "dotfile destination: ${tmpd}" + +# create the dotfile +dnormal="${tmpd}/dir_normal" +mkdir -p ${dnormal} +echo "dir_normal/f1" > ${dnormal}/file1 +echo "dir_normal/f2" > ${dnormal}/file2 + +dlink="${tmpd}/dir_link" +mkdir -p ${dlink} +echo "dir_link/f1" > ${dlink}/file1 +echo "dir_link/f2" > ${dlink}/file2 + +dlinkchildren="${tmpd}/dir_link_children" +mkdir -p ${dlinkchildren} +echo "dir_linkchildren/f1" > ${dlinkchildren}/file1 +echo "dir_linkchildren/f2" > ${dlinkchildren}/file2 + +fnormal="${tmpd}/filenormal" +echo "filenormal" > ${fnormal} + +flink="${tmpd}/filelink" +echo "filelink" > ${flink} + +toimport="${dnormal} ${dlink} ${dlinkchildren} ${fnormal} ${flink}" + +# create the config file +cfg="${tmps}/config.yaml" + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: +profiles: +_EOF + +# import +for i in ${toimport}; do + cd ${ddpath} | ${bin} import -c ${cfg} -f -p p1 -V ${i} +done + +cat ${cfg} + +# test no chmod +cnt=`cat ${cfg} | grep chmod | wc -l` +[ "${cnt}" != "0" ] && echo "chmod wrongly inserted" && exit 1 + +###################### +# update dnormal +chmod 777 ${dnormal} +cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V ${dnormal} + +# check rights updated +[ "`stat -c '%a' ${tmps}/dotfiles/${tmpd}/$(basename ${dnormal})`" != "777" ] && echo "rights not updated (1)" && exit 1 + +cnt=`cat ${cfg} | grep "chmod: '777'" | wc -l` +[ "${cnt}" != "1" ] && echo "chmod not updated (1)" && exit 1 + +###################### +# update dlink +chmod 777 ${dlink} +cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V ${dlink} + +# check rights updated +[ "`stat -c '%a' ${tmps}/dotfiles/${tmpd}/$(basename ${dlink})`" != "777" ] && echo "rights not updated (2)" && exit 1 +cnt=`cat ${cfg} | grep "chmod: '777'" | wc -l` +[ "${cnt}" != "2" ] && echo "chmod not updated (2)" && exit 1 + +###################### +# update dlinkchildren +chmod 777 ${dlinkchildren} +cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V ${dlinkchildren} + +# check rights updated +[ "`stat -c '%a' ${tmps}/dotfiles/${tmpd}/$(basename ${dlinkchildren})`" != "777" ] && echo "rights not updated (3)" && exit 1 +cnt=`cat ${cfg} | grep "chmod: '777'" | wc -l` +[ "${cnt}" != "3" ] && echo "chmod not updated (3)" && exit 1 + +###################### +# update fnormal +chmod 777 ${fnormal} +cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V ${fnormal} + +# check rights updated +[ "`stat -c '%a' ${tmps}/dotfiles/${tmpd}/$(basename ${fnormal})`" != "777" ] && echo "rights not updated (4)" && exit 1 +cnt=`cat ${cfg} | grep "chmod: '777'" | wc -l` +[ "${cnt}" != "4" ] && echo "chmod not updated (4)" && exit 1 + +###################### +# update flink +chmod 777 ${flink} +cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V ${flink} + +# check rights updated +[ "`stat -c '%a' ${tmps}/dotfiles/${tmpd}/$(basename ${flink})`" != "777" ] && echo "rights not updated (5)" && exit 1 +cnt=`cat ${cfg} | grep "chmod: '777'" | wc -l` +[ "${cnt}" != "5" ] && echo "chmod not updated (5)" && exit 1 + +## CLEANING +rm -rf ${tmps} ${tmpd} + +echo "OK" +exit 0 diff --git a/tests-ng/corner-case.sh b/tests-ng/corner-case.sh index bea39c1..e5fb665 100755 --- a/tests-ng/corner-case.sh +++ b/tests-ng/corner-case.sh @@ -56,6 +56,8 @@ basedir=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` echo "[+] dotdrop dir: ${basedir}" echo "[+] dotpath dir: ${basedir}/dotfiles" +export DOTDROP_WORKERS=1 + # create the config file cfg="${basedir}/config.yaml" cat > ${cfg} << _EOF @@ -89,7 +91,7 @@ cd ${ddpath} | ${bin} install -D -c ${cfg} -p p1 --verbose f_x [ "$?" != "0" ] && exit 1 echo "[+] test install not existing src" -cd ${ddpath} | ${bin} install -c ${cfg} --dry -p p1 --verbose f_y +cd ${ddpath} | ${bin} install -c ${cfg} -f --dry -p p1 --verbose f_y echo "[+] test install to temp" cd ${ddpath} | ${bin} install -t -c ${cfg} -p p1 --verbose f_x diff --git a/tests-ng/deprecated-link.sh b/tests-ng/deprecated-link.sh index b63238b..be96783 100755 --- a/tests-ng/deprecated-link.sh +++ b/tests-ng/deprecated-link.sh @@ -126,12 +126,12 @@ set -e # test values have been correctly updated echo "========> test for updated entries" -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_link' | head -1 | grep ',link:link$' -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_nolink' | head -1 | grep ',link:nolink$' -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_nolink1' | head -1 | grep ',link:nolink$' -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_children' | head -1 | grep ',link:link_children$' -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_children2' | head -1 | grep ',link:link_children$' -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_children3' | head -1 | grep ',link:nolink$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_link' | head -1 | grep ',link:link,' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_nolink' | head -1 | grep ',link:nolink,' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_nolink1' | head -1 | grep ',link:nolink,' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_children' | head -1 | grep ',link:link_children,' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_children2' | head -1 | grep ',link:link_children,' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -G | grep '^f_children3' | head -1 | grep ',link:nolink,' ## CLEANING rm -rf ${tmps} ${tmpd} diff --git a/tests-ng/diff-cmd.sh b/tests-ng/diff-cmd.sh index 166135d..639c6ae 100755 --- a/tests-ng/diff-cmd.sh +++ b/tests-ng/diff-cmd.sh @@ -81,7 +81,7 @@ echo "modified" > ${tmpd}/singlefile # default diff (unified) echo "[+] comparing with default diff (unified)" set +e -cd ${ddpath} | ${bin} compare -c ${cfg} 2>&1 | grep -v '=>' | grep -v '^+++\|^---' > ${tmpd}/normal +cd ${ddpath} | ${bin} compare -c ${cfg} 2>&1 | grep -v '=>' | grep -v '\->' | grep -v 'dotfile(s) compared' | sed '$d' | grep -v '^+++\|^---' > ${tmpd}/normal diff -u -r ${tmpd}/singlefile ${basedir}/dotfiles/${tmpd}/singlefile | grep -v '^+++\|^---' > ${tmpd}/real set -e @@ -96,7 +96,7 @@ sed '/dotpath: dotfiles/a \ \ diff_command: "diff -r {0} {1}"' ${cfg} > ${cfg2} # normal diff echo "[+] comparing with normal diff" set +e -cd ${ddpath} | ${bin} compare -c ${cfg2} 2>&1 | grep -v '=>' > ${tmpd}/unified +cd ${ddpath} | ${bin} compare -c ${cfg2} 2>&1 | grep -v '=>' | grep -v '\->' | grep -v 'dotfile(s) compared' | sed '$d' > ${tmpd}/unified diff -r ${tmpd}/singlefile ${basedir}/dotfiles/${tmpd}/singlefile > ${tmpd}/real set -e @@ -113,7 +113,7 @@ sed '/dotpath: dotfiles/a \ \ diff_command: "echo fakediff"' ${cfg} > ${cfg3} # fake diff echo "[+] comparing with fake diff" set +e -cd ${ddpath} | ${bin} compare -c ${cfg3} 2>&1 | grep -v '=>' > ${tmpd}/fake +cd ${ddpath} | ${bin} compare -c ${cfg3} 2>&1 | grep -v '=>' | grep -v '\->' | grep -v 'dotfile(s) compared' | sed '$d' > ${tmpd}/fake set -e # verify diff --git a/tests-ng/dry.sh b/tests-ng/dry.sh new file mode 100755 index 0000000..5707f08 --- /dev/null +++ b/tests-ng/dry.sh @@ -0,0 +1,319 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2020, deadc0de6 +# +# test dry +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ +# the dotfile source +tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${tmps}/dotfiles +# the dotfile destination +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +#echo "dotfile destination: ${tmpd}" +# workdir +tmpw=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +# temp +tmpa=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` + +# ----------------------------- +# test install +# ----------------------------- +# cleaning +rm -rf ${tmps}/* +mkdir -p ${tmps}/dotfiles +rm -rf ${tmpw}/* +rm -rf ${tmpd}/* +rm -rf ${tmpa}/* +# create the config file +cfg="${tmps}/config.yaml" + +echo '{{@@ profile @@}}' > ${tmps}/dotfiles/file +echo '{{@@ profile @@}}' > ${tmps}/dotfiles/link +mkdir -p ${tmps}/dotfiles/dir +echo "{{@@ profile @@}}" > ${tmps}/dotfiles/dir/f1 +mkdir -p ${tmps}/dotfiles/dirchildren +echo "{{@@ profile @@}}" > ${tmps}/dotfiles/dirchildren/f1 +echo "{{@@ profile @@}}" > ${tmps}/dotfiles/dirchildren/f2 + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + workdir: ${tmpw} +actions: + pre: + preaction: echo 'pre' > ${tmpa}/pre + post: + postaction: echo 'post' > ${tmpa}/post +dotfiles: + f_file: + src: file + dst: ${tmpd}/file + actions: + - preaction + - postaction + f_link: + src: link + dst: ${tmpd}/link + link: link + actions: + - preaction + - postaction + d_dir: + src: dir + dst: ${tmpd}/dir + actions: + - preaction + - postaction + d_dirchildren: + src: dirchildren + dst: ${tmpd}/dirchildren + link: link_children + actions: + - preaction + - postaction +profiles: + p1: + dotfiles: + - f_file + - f_link + - d_dir + - d_dirchildren +_EOF + +# install +echo "dry install" +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 -V --dry + +cnt=`ls -1 ${tmpd} | wc -l` +ls -1 ${tmpd} +[ "${cnt}" != "0" ] && echo "dry install failed (1)" && exit 1 + +cnt=`ls -1 ${tmpw} | wc -l` +ls -1 ${tmpw} +[ "${cnt}" != "0" ] && echo "dry install failed (2)" && exit 1 + +cnt=`ls -1 ${tmpa} | wc -l` +ls -1 ${tmpa} +[ "${cnt}" != "0" ] && echo "dry install failed (3)" && exit 1 + +# ----------------------------- +# test import +# ----------------------------- +# cleaning +rm -rf ${tmps}/* +mkdir -p ${tmps}/dotfiles +rm -rf ${tmpw}/* +rm -rf ${tmpd}/* +rm -rf ${tmpa}/* + +# create the config file +cfg="${tmps}/config.yaml" +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + workdir: ${tmpw} +dotfiles: +profiles: +_EOF +cp ${cfg} ${tmpa}/config.yaml + +echo 'content' > ${tmpd}/file +echo 'content' > ${tmpd}/link +mkdir -p ${tmpd}/dir +echo "content" > ${tmpd}/dir/f1 +mkdir -p ${tmpd}/dirchildren +echo "content" > ${tmpd}/dirchildren/f1 +echo "content" > ${tmpd}/dirchildren/f2 + +dotfiles="${tmpd}/file ${tmpd}/link ${tmpd}/dir ${tmpd}/dirchildren" + +echo "dry import" +cd ${ddpath} | ${bin} import -c ${cfg} -f -p p1 -V --dry ${dotfiles} + +cnt=`ls -1 ${tmps}/dotfiles | wc -l` +ls -1 ${tmps}/dotfiles +[ "${cnt}" != "0" ] && echo "dry import failed (1)" && exit 1 + +diff ${cfg} ${tmpa}/config.yaml || (echo "dry import failed (2)" && exit 1) + +# ----------------------------- +# test update +# ----------------------------- +# cleaning +rm -rf ${tmps}/* +mkdir -p ${tmps}/dotfiles +rm -rf ${tmpw}/* +rm -rf ${tmpd}/* +rm -rf ${tmpa}/* + +echo 'original' > ${tmps}/dotfiles/file +echo 'original' > ${tmps}/dotfiles/link +mkdir -p ${tmps}/dotfiles/dir +echo "original" > ${tmps}/dotfiles/dir/f1 +mkdir -p ${tmps}/dotfiles/dirchildren +echo "original" > ${tmps}/dotfiles/dirchildren/f1 +echo "original" > ${tmps}/dotfiles/dirchildren/f2 + +echo 'modified' > ${tmpd}/file +echo 'modified' > ${tmpd}/link +mkdir -p ${tmpd}/dir +echo "modified" > ${tmpd}/dir/f1 +mkdir -p ${tmpd}/dirchildren +echo "modified" > ${tmpd}/dirchildren/f1 +echo "modified" > ${tmpd}/dirchildren/f2 + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + workdir: ${tmpw} +dotfiles: + f_file: + src: file + dst: ${tmpd}/file + f_link: + src: link + dst: ${tmpd}/link + link: link + d_dir: + src: dir + dst: ${tmpd}/dir + d_dirchildren: + src: dirchildren + dst: ${tmpd}/dirchildren + link: link_children +profiles: + p1: + dotfiles: + - f_file + - f_link + - d_dir + - d_dirchildren +_EOF +cp ${cfg} ${tmpa}/config.yaml + +echo "dry update" +dotfiles="${tmpd}/file ${tmpd}/link ${tmpd}/dir ${tmpd}/dirchildren" +cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 -V --dry ${dotfiles} + +grep 'modified' ${tmps}/dotfiles/file && echo "dry update failed (1)" && exit 1 +grep 'modified' ${tmps}/dotfiles/link && echo "dry update failed (2)" && exit 1 +grep "modified" ${tmps}/dotfiles/dir/f1 && echo "dry update failed (3)" && exit 1 +grep "modified" ${tmps}/dotfiles/dirchildren/f1 && echo "dry update failed (4)" && exit 1 +grep "modified" ${tmps}/dotfiles/dirchildren/f2 && echo "dry update failed (5)" && exit 1 + +diff ${cfg} ${tmpa}/config.yaml || (echo "dry update failed (6)" && exit 1) + +# ----------------------------- +# test remove +# ----------------------------- +# cleaning +rm -rf ${tmps}/* +mkdir -p ${tmps}/dotfiles +rm -rf ${tmpw}/* +rm -rf ${tmpd}/* +rm -rf ${tmpa}/* + +echo '{{@@ profile @@}}' > ${tmps}/dotfiles/file +echo '{{@@ profile @@}}' > ${tmps}/dotfiles/link +mkdir -p ${tmps}/dotfiles/dir +echo "{{@@ profile @@}}" > ${tmps}/dotfiles/dir/f1 +mkdir -p ${tmps}/dotfiles/dirchildren +echo "{{@@ profile @@}}" > ${tmps}/dotfiles/dirchildren/f1 +echo "{{@@ profile @@}}" > ${tmps}/dotfiles/dirchildren/f2 + +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles + workdir: ${tmpw} +dotfiles: + f_file: + src: file + dst: ${tmpd}/file + f_link: + src: link + dst: ${tmpd}/link + link: link + d_dir: + src: dir + dst: ${tmpd}/dir + d_dirchildren: + src: dirchildren + dst: ${tmpd}/dirchildren + link: link_children +profiles: + p1: + dotfiles: + - f_file + - f_link + - d_dir + - d_dirchildren +_EOF +cp ${cfg} ${tmpa}/config.yaml + +echo "dry remove" +dotfiles="${tmpd}/file ${tmpd}/link ${tmpd}/dir ${tmpd}/dirchildren" +cd ${ddpath} | ${bin} remove -c ${cfg} -f -p p1 -V --dry ${dotfiles} + +[ ! -e ${tmps}/dotfiles/file ] && echo "dry remove failed (1)" && exit 1 +[ ! -e ${tmps}/dotfiles/link ] && echo "dry remove failed (2)" && exit 1 +[ ! -d ${tmps}/dotfiles/dir ] && echo "dry remove failed (3)" && exit 1 +[ ! -e ${tmps}/dotfiles/dir/f1 ] && echo "dry remove failed (4)" && exit 1 +[ ! -d ${tmps}/dotfiles/dirchildren ] && echo "dry remove failed (5)" && exit 1 +[ ! -e ${tmps}/dotfiles/dirchildren/f1 ] && echo "dry remove failed (6)" && exit 1 +[ ! -e ${tmps}/dotfiles/dirchildren/f2 ] && echo "dry remove failed (7)" && exit 1 + +diff ${cfg} ${tmpa}/config.yaml || (echo "dry remove failed (8)" && exit 1) + +## CLEANING +rm -rf ${tmps} ${tmpd} ${tmpw} ${tmpa} + +echo "OK" +exit 0 diff --git a/tests-ng/global-update-ignore.sh b/tests-ng/global-update-ignore.sh index 5b4c302..93c6971 100755 --- a/tests-ng/global-update-ignore.sh +++ b/tests-ng/global-update-ignore.sh @@ -99,8 +99,9 @@ cd ${ddpath} | ${bin} update -f -c ${cfg} --verbose --profile=p1 --key f_abc # check files haven't been updated [ ! -e ${dt}/a/c/acfile ] && echo "acfile not found" && exit 1 -cat ${dt}/a/c/acfile -grep 'b' ${dt}/a/c/acfile >/dev/null +set +e +grep 'b' ${dt}/a/c/acfile || (echo "acfile not updated" && exit 1) +set -e [ -e ${dt}/a/newfile ] && echo "newfile found" && exit 1 ## CLEANING diff --git a/tests-ng/globs.sh b/tests-ng/globs.sh index a682961..d38915f 100755 --- a/tests-ng/globs.sh +++ b/tests-ng/globs.sh @@ -98,7 +98,7 @@ mkdir -p ${tmps}/dotfiles/ echo "abc" > ${tmps}/dotfiles/abc # install -cd ${ddpath} | ${bin} install -c ${cfg} -p p1 -V +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 -V # checks [ ! -e ${tmpd}/abc ] && echo "dotfile not installed" && exit 1 diff --git a/tests-ng/import-configs.sh b/tests-ng/import-configs.sh index 4c75d4c..41a45e8 100755 --- a/tests-ng/import-configs.sh +++ b/tests-ng/import-configs.sh @@ -143,7 +143,7 @@ cd ${ddpath} | ${bin} files -c ${cfg1} -p pup -V | grep f_sub cd ${ddpath} | ${bin} files -c ${cfg1} -p psubsub -V | grep f_sub # test compare too -cd ${ddpath} | ${bin} install -c ${cfg1} -p p2 -V +cd ${ddpath} | ${bin} install -c ${cfg1} -p p2 -V -f cd ${ddpath} | ${bin} compare -c ${cfg1} -p p2 -V # test with non-existing dotpath this time @@ -172,7 +172,7 @@ profiles: dotfiles: - f_asub _EOF -cd ${ddpath} | ${bin} install -c ${cfg1} -p p2 -V +cd ${ddpath} | ${bin} install -c ${cfg1} -p p2 -V -f cd ${ddpath} | ${bin} compare -c ${cfg1} -p p2 -V ## CLEANING diff --git a/tests-ng/import-link-children.sh b/tests-ng/import-link-children.sh index 036e667..309fd54 100755 --- a/tests-ng/import-link-children.sh +++ b/tests-ng/import-link-children.sh @@ -89,7 +89,7 @@ _EOF cd ${ddpath} | ${bin} import -c ${cfg} -p p1 -V --link=link_children ${dt} # check is set to link_children -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "d_`basename ${dt}`" | grep ',link:link_children$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "d_`basename ${dt}`" | grep ',link:link_children,' # checks file exists in dotpath [ ! -e ${dotpath}/${dt} ] && echo "dotfile not imported" && exit 1 diff --git a/tests-ng/import-with-empty.sh b/tests-ng/import-with-empty.sh index b43fb93..8937313 100755 --- a/tests-ng/import-with-empty.sh +++ b/tests-ng/import-with-empty.sh @@ -97,9 +97,9 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 --verbose ${dftoimport} [ "$?" != "0" ] && exit 1 echo "[+] install" -cd ${ddpath} | ${bin} install -c ${cfg} -p p1 --verbose | grep '^5 dotfile(s) installed.$' +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 --verbose | grep '^5 dotfile(s) installed.$' rm -f ${dftoimport} -cd ${ddpath} | ${bin} install -c ${cfg} -p p1 --verbose | grep '^6 dotfile(s) installed.$' +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 --verbose | grep '^6 dotfile(s) installed.$' nb=`cd ${ddpath} | ${bin} files -c ${cfg} -p p1 --verbose | grep '^[a-zA-Z]' | wc -l` [ "${nb}" != "6" ] && echo 'error in dotfile list' && exit 1 diff --git a/tests-ng/include-order.sh b/tests-ng/include-order.sh index 01c295a..3c261e4 100755 --- a/tests-ng/include-order.sh +++ b/tests-ng/include-order.sh @@ -55,6 +55,7 @@ tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` # temporary tmpa=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +export DOTDROP_WORKERS=1 # create the config file cfg="${tmps}/config.yaml" @@ -66,8 +67,8 @@ config: actions: pre: first: 'echo first > ${tmpa}/cookie' - second: 'echo second >> ${tmpa}/cookie' - third: 'echo third >> ${tmpa}/cookie' + second: 'sleep 1; echo second >> ${tmpa}/cookie' + third: 'sleep 1; echo third >> ${tmpa}/cookie' dotfiles: f_first: dst: ${tmpd}/first @@ -115,9 +116,9 @@ for ((i=0;i<${attempts};i++)); do echo "second timestamp: `stat -c %y ${tmpd}/second`" echo "third timestamp: `stat -c %y ${tmpd}/third`" - ts_first=`date "+%S%N" -d "$(stat -c %y ${tmpd}/first)"` - ts_second=`date "+%S%N" -d "$(stat -c %y ${tmpd}/second)"` - ts_third=`date "+%S%N" -d "$(stat -c %y ${tmpd}/third)"` + ts_first=`date "+%s" -d "$(stat -c %y ${tmpd}/first)"` + ts_second=`date "+%s" -d "$(stat -c %y ${tmpd}/second)"` + ts_third=`date "+%s" -d "$(stat -c %y ${tmpd}/third)"` #echo "first ts: ${ts_first}" #echo "second ts: ${ts_second}" diff --git a/tests-ng/install-empty.sh b/tests-ng/install-empty.sh index 6773ca7..8ca2ff3 100755 --- a/tests-ng/install-empty.sh +++ b/tests-ng/install-empty.sh @@ -87,7 +87,7 @@ profiles: _EOF echo "[+] install" -cd ${ddpath} | ${bin} install -c ${cfg} -p p1 --verbose | grep '^5 dotfile(s) installed.$' +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 --verbose | grep '^5 dotfile(s) installed.$' [ "$?" != "0" ] && exit 1 ## CLEANING diff --git a/tests-ng/install-ignore.sh b/tests-ng/install-ignore.sh index fd62890..76258ec 100755 --- a/tests-ng/install-ignore.sh +++ b/tests-ng/install-ignore.sh @@ -82,7 +82,7 @@ echo "new data" > ${basedir}/dotfiles/${tmpd}/readmes/README.md # install rm -rf ${tmpd} echo "[+] install normal" -cd ${ddpath} | ${bin} install --showdiff -c ${cfg} --verbose +cd ${ddpath} | ${bin} install --showdiff -c ${cfg} --verbose -f [ "$?" != "0" ] && exit 1 nb=`find ${tmpd} -iname 'README.md' | wc -l` echo "(1) found ${nb} README.md file(s)" @@ -96,7 +96,7 @@ cat ${cfg2} # install rm -rf ${tmpd} echo "[+] install with ignore in dotfile" -cd ${ddpath} | ${bin} install -c ${cfg2} --verbose +cd ${ddpath} | ${bin} install -c ${cfg2} --verbose -f [ "$?" != "0" ] && exit 1 nb=`find ${tmpd} -iname 'README.md' | wc -l` echo "(2) found ${nb} README.md file(s)" @@ -110,7 +110,7 @@ cat ${cfg2} # install rm -rf ${tmpd} echo "[+] install with ignore in config" -cd ${ddpath} | ${bin} install -c ${cfg2} --verbose +cd ${ddpath} | ${bin} install -c ${cfg2} --verbose -f [ "$?" != "0" ] && exit 1 nb=`find ${tmpd} -iname 'README.md' | wc -l` echo "(3) found ${nb} README.md file(s)" @@ -118,7 +118,7 @@ echo "(3) found ${nb} README.md file(s)" ## reinstall to trigger showdiff echo "showdiff" > ${tmpd}/program/a -cd ${ddpath} | echo "y" | ${bin} install --showdiff -c ${cfg} --verbose +cd ${ddpath} | echo "y" | ${bin} install --showdiff -c ${cfg} --verbose -f [ "$?" != "0" ] && exit 1 ## CLEANING diff --git a/tests-ng/install-to-temp.sh b/tests-ng/install-to-temp.sh index d8e95f2..7ffa49c 100755 --- a/tests-ng/install-to-temp.sh +++ b/tests-ng/install-to-temp.sh @@ -84,7 +84,7 @@ echo 'test_y' > ${basedir}/dotfiles/y echo "00000000 01 02 03 04 05" | xxd -r - ${basedir}/dotfiles/z echo "[+] install" -cd ${ddpath} | ${bin} install -c ${cfg} -p p1 --showdiff --verbose --temp | grep '^3 dotfile(s) installed.$' +cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 --showdiff --verbose --temp | grep '^3 dotfile(s) installed.$' [ "$?" != "0" ] && exit 1 ## CLEANING diff --git a/tests-ng/install.sh b/tests-ng/install.sh new file mode 100755 index 0000000..16afa60 --- /dev/null +++ b/tests-ng/install.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2020, deadc0de6 +# +# test install +# returns 1 in case of error +# + +# exit on first error +set -e + +# all this crap to get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +#hash dotdrop >/dev/null 2>&1 +#[ "$?" != "0" ] && echo "install dotdrop to run tests" && exit 1 + +#echo "called with ${1}" + +# dotdrop path can be pass as argument +ddpath="${cur}/../" +[ "${1}" != "" ] && ddpath="${1}" +[ ! -d ${ddpath} ] && echo "ddpath \"${ddpath}\" is not a directory" && exit 1 + +export PYTHONPATH="${ddpath}:${PYTHONPATH}" +bin="python3 -m dotdrop.dotdrop" +hash coverage 2>/dev/null && bin="coverage run -a --source=dotdrop -m dotdrop.dotdrop" || true + +echo "dotdrop path: ${ddpath}" +echo "pythonpath: ${PYTHONPATH}" + +# get the helpers +source ${cur}/helpers + +echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" + +################################################################ +# this is the test +################################################################ + +get_file_mode() +{ + u=`umask` + u=`echo ${u} | sed 's/^0*//'` + v=$((666 - u)) + echo "${v}" +} + +# $1 path +# $2 rights +has_rights() +{ + echo "testing ${1} is ${2}" + [ ! -e "$1" ] && echo "`basename $1` does not exist" && exit 1 + local mode=`stat -L -c '%a' "$1"` + [ "${mode}" != "$2" ] && echo "bad mode for `basename $1` (${mode} VS expected ${2})" && exit 1 + true +} + +# dotdrop directory +basedir=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` +mkdir -p ${basedir}/dotfiles +echo "[+] dotdrop dir: ${basedir}" +echo "[+] dotpath dir: ${basedir}/dotfiles" +tmpd=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` + +echo "content" > ${basedir}/dotfiles/x + +# create the config file +cfg="${basedir}/config.yaml" +cat > ${cfg} << _EOF +config: + backup: true + create: true + dotpath: dotfiles +dotfiles: + f_x: + src: x + dst: ${tmpd}/x +profiles: + p1: + dotfiles: + - f_x +_EOF + +echo "[+] install" +cd ${ddpath} | ${bin} install -c ${cfg} -f -p p1 --verbose | grep '^1 dotfile(s) installed.$' +[ "$?" != "0" ] && exit 1 + +[ ! -e ${tmpd}/x ] && echo "f_x not installed" && exit 1 + +# update chmod +chmod 666 ${tmpd}/x +cd ${ddpath} | ${bin} update -c ${cfg} -f -p p1 --verbose ${tmpd}/x + +# chmod updated +cat ${cfg} | grep "chmod: '666'" + +chmod 644 ${tmpd}/x + +mode=`get_file_mode ${tmpd}/x` +echo "[+] re-install with no" +cd ${ddpath} | printf "N\n" | ${bin} install -c ${cfg} -p p1 --verbose +[ "$?" != "0" ] && exit 1 + +# if user answers N, chmod should not be done +has_rights "${tmpd}/x" "${mode}" + +echo "[+] re-install with yes" +cd ${ddpath} | printf "y\n" | ${bin} install -c ${cfg} -p p1 --verbose +[ "$?" != "0" ] && exit 1 + +has_rights "${tmpd}/x" "666" + +## CLEANING +rm -rf ${basedir} ${tmpd} + +echo "OK" +exit 0 diff --git a/tests-ng/link-value-tests.sh b/tests-ng/link-value-tests.sh index 4988e6f..853fade 100755 --- a/tests-ng/link-value-tests.sh +++ b/tests-ng/link-value-tests.sh @@ -80,7 +80,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink,' # try to install rm -rf ${tmpd}/qwert @@ -114,7 +114,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink,' # try to install rm -rf ${tmpd}/qwert @@ -148,7 +148,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V --link=nolink # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink,' # try to install rm -rf ${tmpd}/qwert @@ -182,7 +182,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V --link=link # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:link$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:link,' # try to install rm -rf ${tmpd}/qwert @@ -216,7 +216,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:link$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:link,' # try to install rm -rf ${tmpd}/qwert @@ -250,7 +250,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V --link=nolink # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink,' # try to install rm -rf ${tmpd}/qwert @@ -284,7 +284,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V --link=nolink # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink,' # try to install rm -rf ${tmpd}/qwert @@ -318,7 +318,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V --link=nolink # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:nolink,' # try to install rm -rf ${tmpd}/qwert @@ -350,7 +350,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} --link=link -p p1 ${df} -V # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:link$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "f_`basename ${df}`" | head -1 | grep ',link:link,' # try to install rm -rf ${tmpd}/qwert @@ -411,7 +411,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} --link=link_children -p p1 ${df} -V # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "d_`basename ${df}`" | head -1 | grep ',link:link_children$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "d_`basename ${df}`" | head -1 | grep ',link:link_children,' # try to install rm -rf ${tmpd}/qwert @@ -451,7 +451,7 @@ cd ${ddpath} | ${bin} import -c ${cfg} -p p1 ${df} -V # checks cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "d_`basename ${df}`" | head -1 | grep ',link:link_children$' +cd ${ddpath} | ${bin} files -c ${cfg} -p p1 -V -G | grep "d_`basename ${df}`" | head -1 | grep ',link:link_children,' # try to install rm -rf ${tmpd}/qwert diff --git a/tests-ng/macro-with-globals.sh b/tests-ng/macro-with-globals.sh index 48bb1d3..f29b6f3 100755 --- a/tests-ng/macro-with-globals.sh +++ b/tests-ng/macro-with-globals.sh @@ -88,7 +88,7 @@ cat > ${tmps}/dotfiles/abc << _EOF _EOF # install -cd ${ddpath} | ${bin} install -c ${cfg} -p p0 -V +cd ${ddpath} | ${bin} install -c ${cfg} -p p0 -V -f # test file content cat ${tmpd}/abc diff --git a/tests-ng/profile-actions.sh b/tests-ng/profile-actions.sh index 5840021..307f5cb 100755 --- a/tests-ng/profile-actions.sh +++ b/tests-ng/profile-actions.sh @@ -94,6 +94,9 @@ profiles: _EOF #cat ${cfg} +# list profiles +cd ${ddpath} | ${bin} profiles -c ${cfg} -V + # create the dotfile echo "test" > ${tmps}/dotfiles/abc echo "test" > ${tmps}/dotfiles/def diff --git a/tests-ng/tests-launcher.py b/tests-ng/tests-launcher.py new file mode 100755 index 0000000..aa36e8e --- /dev/null +++ b/tests-ng/tests-launcher.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2020, deadc0de6 +# +# tests launcher +# + + +import os +import sys +import subprocess +from concurrent import futures + + +MAX_JOBS = 10 + + +def run_test(path): + cur = os.path.dirname(sys.argv[0]) + name = os.path.basename(path) + path = os.path.join(cur, name) + + p = subprocess.Popen(path, shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + out, _ = p.communicate() + out = out.decode() + r = p.returncode == 0 + reason = 'returncode' + if 'Traceback' in out: + r = False + reason = 'traceback' + return r, reason, path, out + + +def get_tests(): + tests = [] + cur = os.path.dirname(sys.argv[0]) + for (_, _, filenames) in os.walk(cur): + for path in filenames: + if not path.endswith('.sh'): + continue + tests.append(path) + break + return tests + + +def main(): + global MAX_JOBS + if len(sys.argv) > 1: + MAX_JOBS = int(sys.argv[1]) + + tests = get_tests() + + with futures.ThreadPoolExecutor(max_workers=MAX_JOBS) as ex: + wait_for = [] + for test in tests: + j = ex.submit(run_test, test) + wait_for.append(j) + + for f in futures.as_completed(wait_for): + r, reason, p, log = f.result() + if not r: + ex.shutdown(wait=False) + for x in wait_for: + x.cancel() + print() + print(log) + print('test {} failed ({})'.format(p, reason)) + return False + else: + sys.stdout.write('.') + sys.stdout.flush() + sys.stdout.write('\n') + return True + + +if __name__ == '__main__': + if not main(): + sys.exit(1) + sys.exit(0) diff --git a/tests-ng/transformations.sh b/tests-ng/transformations.sh index 0bd329f..470e63e 100755 --- a/tests-ng/transformations.sh +++ b/tests-ng/transformations.sh @@ -89,6 +89,7 @@ dotfiles: src: ghi trans: uncompress trans_write: compress + chmod: 700 profiles: p1: dotfiles: @@ -125,40 +126,43 @@ tar -tf ${tmps}/dotfiles/ghi # test install and compare ########################### +echo "[+] run install" # install cd ${ddpath} | ${bin} install -f -c ${cfg} -p p1 -b -V # check canary dotfile -[ ! -e ${tmpd}/def ] && exit 1 +[ ! -e ${tmpd}/def ] && echo "def does not exist" && exit 1 # check base64 dotfile -[ ! -e ${tmpd}/abc ] && exit 1 +[ ! -e ${tmpd}/abc ] && echo "abc does not exist" && exit 1 content=`cat ${tmpd}/abc` -[ "${content}" != "${token}" ] && exit 1 +[ "${content}" != "${token}" ] && echo "bad content for abc" && exit 1 # check directory dotfile -[ ! -e ${tmpd}/ghi/a/dir1/otherfile ] && exit 1 +[ ! -e ${tmpd}/ghi/a/dir1/otherfile ] && echo "otherfile does not exist" && exit 1 content=`cat ${tmpd}/ghi/a/somefile` -[ "${content}" != "${tokend}" ] && exit 1 +[ "${content}" != "${tokend}" ] && echo "bad content for somefile" && exit 1 content=`cat ${tmpd}/ghi/a/dir1/otherfile` -[ "${content}" != "${tokend}" ] && exit 1 +[ "${content}" != "${tokend}" ] && echo "bad content for otherfile" && exit 1 # compare +set +e cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 -b -V -[ "$?" != "0" ] && exit 1 +[ "$?" != "0" ] && echo "compare failed (0)" && exit 1 +set -e # change base64 deployed file echo ${touched} > ${tmpd}/abc set +e cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 -b -V -[ "$?" != "1" ] && exit 1 +[ "$?" != "1" ] && echo "compare failed (1)" && exit 1 set -e # change uncompressed deployed dotfile echo ${touched} > ${tmpd}/ghi/a/somefile set +e cd ${ddpath} | ${bin} compare -c ${cfg} -p p1 -b -V -[ "$?" != "1" ] && exit 1 +[ "$?" != "1" ] && echo "compare failed (2)" && exit 1 set -e ########################### @@ -167,38 +171,44 @@ set -e # update single file echo 'update' > ${tmpd}/def +set +e cd ${ddpath} | ${bin} update -f -k -c ${cfg} -p p1 -b -V f_def -[ "$?" != "0" ] && exit 1 +[ "$?" != "0" ] && echo "update failed (1)" && exit 1 +set -e [ ! -e ${tmpd}/def ] && echo 'dotfile in FS removed' && exit 1 [ ! -e ${tmps}/dotfiles/def ] && echo 'dotfile in dotpath removed' && exit 1 # update single file +set +e cd ${ddpath} | ${bin} update -f -k -c ${cfg} -p p1 -b -V f_abc -[ "$?" != "0" ] && exit 1 +[ "$?" != "0" ] && echo "update failed (2)" && exit 1 +set -e # test updated file -[ ! -e ${tmps}/dotfiles/abc ] && exit 1 +[ ! -e ${tmps}/dotfiles/abc ] && echo "abc does not exist" && exit 1 content=`cat ${tmps}/dotfiles/abc` bcontent=`echo ${touched} | base64` -[ "${content}" != "${bcontent}" ] && exit 1 +[ "${content}" != "${bcontent}" ] && echo "bad content for abc" && exit 1 # update directory echo ${touched} > ${tmpd}/ghi/b/newfile rm -r ${tmpd}/ghi/c cd ${ddpath} | ${bin} update -f -k -c ${cfg} -p p1 -b -V d_ghi -[ "$?" != "0" ] && exit 1 +[ "$?" != "0" ] && echo "update failed" && exit 1 # test updated directory -tar -tf ${tmps}/dotfiles/ghi | grep './b/newfile' -tar -tf ${tmps}/dotfiles/ghi | grep './a/dir1/otherfile' +set +e +tar -tf ${tmps}/dotfiles/ghi | grep './b/newfile' || (echo "newfile not found in tar" && exit 1) +tar -tf ${tmps}/dotfiles/ghi | grep './a/dir1/otherfile' || (echo "otherfile not found in tar" && exit 1) +set -e tmpy=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` tar -xf ${tmps}/dotfiles/ghi -C ${tmpy} content=`cat ${tmpy}/a/somefile` -[ "${content}" != "${touched}" ] && exit 1 +[ "${content}" != "${touched}" ] && echo "bad content" && exit 1 # check canary dotfile -[ ! -e ${tmps}/dotfiles/def ] && exit 1 +[ ! -e ${tmps}/dotfiles/def ] && echo "def not found" && exit 1 ## CLEANING rm -rf ${tmps} ${tmpd} ${tmpx} ${tmpy} diff --git a/tests-ng/update-ignore.sh b/tests-ng/update-ignore.sh index 9794e36..cf928ce 100755 --- a/tests-ng/update-ignore.sh +++ b/tests-ng/update-ignore.sh @@ -46,6 +46,15 @@ echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" # this is the test ################################################################ +# $1 pattern +# $2 path +grep_or_fail() +{ + set +e + grep "${1}" "${2}" >/dev/null 2>&1 || (echo "pattern not found in ${2}" && exit 1) + set -e +} + # dotdrop directory tmps=`mktemp -d --suffix='-dotdrop-tests' || mktemp -d` dt="${tmps}/dotfiles" @@ -98,7 +107,7 @@ cd ${ddpath} | ${bin} update -f -c ${cfg} --verbose --profile=p1 --key f_abc #tree ${dt} # check files haven't been updated -grep 'b' ${dt}/a/c/acfile >/dev/null +grep_or_fail 'b' "${dt}/a/c/acfile" [ -e ${dt}/a/newfile ] && echo "should not have been updated" && exit 1 ## CLEANING diff --git a/tests-ng/workdir.sh b/tests-ng/workdir.sh index dae5751..bdc2d31 100755 --- a/tests-ng/workdir.sh +++ b/tests-ng/workdir.sh @@ -45,6 +45,7 @@ echo -e "$(tput setaf 6)==> RUNNING $(basename $BASH_SOURCE) <==$(tput sgr0)" ################################################################ # this is the test ################################################################ +unset DOTDROP_WORKDIR string="blabla" # the dotfile source diff --git a/tests.sh b/tests.sh index 1a18386..722f973 100755 --- a/tests.sh +++ b/tests.sh @@ -3,7 +3,8 @@ # Copyright (c) 2017, deadc0de6 # stop on first error -set -ev +#set -ev +set -e # PEP8 tests which pycodestyle >/dev/null 2>&1 @@ -30,10 +31,23 @@ export DOTDROP_FORCE_NODEBUG=yes # coverage file location cur=`dirname $(readlink -f "${0}")` -export COVERAGE_FILE="${cur}/.coverage" + +workers=${DOTDROP_WORKERS} +if [ ! -z ${workers} ]; then + unset DOTDROP_WORKERS + echo "DISABLE workers" +fi # execute tests with coverage -PYTHONPATH="dotdrop" ${nosebin} -s --with-coverage --cover-package=dotdrop +if [ -z ${GITHUB_WORKFLOW} ]; then + ## local + export COVERAGE_FILE= + PYTHONPATH="dotdrop" ${nosebin} -s --processes=-1 --with-coverage --cover-package=dotdrop +else + ## CI/CD + export COVERAGE_FILE="${cur}/.coverage" + PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop +fi #PYTHONPATH="dotdrop" python3 -m pytest tests # enable debug logs @@ -41,32 +55,23 @@ export DOTDROP_DEBUG=yes unset DOTDROP_FORCE_NODEBUG # do not print debugs when running tests (faster) #export DOTDROP_FORCE_NODEBUG=yes +export DOTDROP_WORKDIR=/tmp/dotdrop-tests-workdir -## execute bash script tests -[ "$1" = '--python-only' ] || { - echo "doing extended tests" - logdir=`mktemp -d` - for scr in tests-ng/*.sh; do - logfile="${logdir}/`basename ${scr}`.log" - echo "-> running test ${scr} (logfile:${logfile})" - set +e - ${scr} > "${logfile}" 2>&1 - if [ "$?" -ne 0 ]; then - cat ${logfile} - echo "test ${scr} finished with error" - rm -rf ${logdir} - exit 1 - elif grep Traceback ${logfile}; then - cat ${logfile} - echo "test ${scr} crashed" - rm -rf ${logdir} - exit 1 - fi - set -e - echo "test ${scr} ok" - done - rm -rf ${logdir} -} +if [ ! -z ${workers} ]; then + DOTDROP_WORKERS=${workers} + echo "ENABLE workers: ${workers}" +fi + +# run bash tests +if [ -z ${GITHUB_WORKFLOW} ]; then + ## local + export COVERAGE_FILE= + tests-ng/tests-launcher.py +else + ## CI/CD + export COVERAGE_FILE="${cur}/.coverage" + tests-ng/tests-launcher.py 1 +fi ## test the doc with remark ## https://github.com/remarkjs/remark-validate-links @@ -81,18 +86,18 @@ else remark -f -u validate-links *.md fi -## test the doc with markdown-link-check -## https://github.com/tcort/markdown-link-check -set +e -which markdown-link-check >/dev/null 2>&1 -r="$?" -set -e -if [ "$r" != "0" ]; then - echo "[WARNING] install \"markdown-link-check\" to test the doc" -else - for i in `find docs -iname '*.md'`; do markdown-link-check $i; done - markdown-link-check README.md -fi +### test the doc with markdown-link-check +### https://github.com/tcort/markdown-link-check +#set +e +#which markdown-link-check >/dev/null 2>&1 +#r="$?" +#set -e +#if [ "$r" != "0" ]; then +# echo "[WARNING] install \"markdown-link-check\" to test the doc" +#else +# for i in `find docs -iname '*.md'`; do markdown-link-check $i; done +# markdown-link-check README.md +#fi ## done echo "All test finished successfully in ${SECONDS}s" diff --git a/tests/helpers.py b/tests/helpers.py index 8abb198..d080e9e 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -65,7 +65,9 @@ def get_string(length): def get_tempdir(): """Get a temporary directory""" - return tempfile.mkdtemp(suffix=TMPSUFFIX) + tmpdir = tempfile.mkdtemp(suffix=TMPSUFFIX) + os.chmod(tmpdir, 0o755) + return tmpdir def create_random_file(directory, content=None, @@ -132,6 +134,7 @@ def _fake_args(): args['--as'] = None args['--file-only'] = False args['--workers'] = 1 + args['--preserve-mode'] = False # cmds args['profiles'] = False args['files'] = False diff --git a/tests/test_import.py b/tests/test_import.py index 08df58c..186382a 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -10,7 +10,7 @@ import os from dotdrop.dotdrop import cmd_importer from dotdrop.dotdrop import cmd_list_profiles -from dotdrop.dotdrop import cmd_list_files +from dotdrop.dotdrop import cmd_files from dotdrop.dotdrop import cmd_update from dotdrop.linktypes import LinkTypes @@ -184,7 +184,7 @@ class TestImport(unittest.TestCase): self.assertTrue(os.path.exists(s4)) cmd_list_profiles(o) - cmd_list_files(o) + cmd_files(o) # fake test update editcontent = 'edited' diff --git a/tests/test_install.py b/tests/test_install.py index 2da99be..c20cd4e 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -6,7 +6,7 @@ basic unittest for the install function import os import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import filecmp from dotdrop.cfg_aggregator import CfgAggregator as Cfg @@ -349,8 +349,9 @@ exec bspwm srcs = [create_random_file(src_dir)[0] for _ in range(3)] installer = Installer() - installer.link_children(templater=MagicMock(), src=src_dir, - dst=dst_dir, actionexec=None) + installer.install(templater=MagicMock(), src=src_dir, + dst=dst_dir, linktype=LinkTypes.LINK_CHILDREN, + actionexec=None) # Ensure all destination files point to source for src in srcs: @@ -365,8 +366,10 @@ exec bspwm # logger = MagicMock() # installer.log.err = logger - res, err = installer.link_children(templater=MagicMock(), src=src, - dst='/dev/null', actionexec=None) + res, err = installer.install(templater=MagicMock(), src=src, + dst='/dev/null', + linktype=LinkTypes.LINK_CHILDREN, + actionexec=None) self.assertFalse(res) e = 'source dotfile does not exist: {}'.format(src) @@ -387,8 +390,10 @@ exec bspwm # installer.log.err = logger # pass src file not src dir - res, err = installer.link_children(templater=templater, src=src, - dst='/dev/null', actionexec=None) + res, err = installer.install(templater=templater, src=src, + dst='/dev/null', + linktype=LinkTypes.LINK_CHILDREN, + actionexec=None) # ensure nothing performed self.assertFalse(res) @@ -410,8 +415,9 @@ exec bspwm self.assertFalse(os.path.exists(dst_dir)) installer = Installer() - installer.link_children(templater=MagicMock(), src=src_dir, - dst=dst_dir, actionexec=None) + installer.install(templater=MagicMock(), src=src_dir, + dst=dst_dir, linktype=LinkTypes.LINK_CHILDREN, + actionexec=None) # ensure dst dir created self.assertTrue(os.path.exists(dst_dir)) @@ -442,8 +448,9 @@ exec bspwm installer.safe = True installer.log.ask = ask - installer.link_children(templater=MagicMock(), src=src_dir, dst=dst, - actionexec=None) + installer.install(templater=MagicMock(), src=src_dir, + dst=dst, linktype=LinkTypes.LINK_CHILDREN, + actionexec=None) # ensure destination now a directory self.assertTrue(os.path.isdir(dst)) @@ -453,8 +460,7 @@ exec bspwm 'Remove regular file {} and replace with empty directory?' .format(dst)) - @patch('dotdrop.installer.Templategen') - def test_runs_templater(self, mocked_templategen): + def test_runs_templater(self): """test runs templater""" # create source dir src_dir = get_tempdir() @@ -473,11 +479,9 @@ exec bspwm installer = Installer() templater = MagicMock() templater.generate.return_value = b'content' - # make templategen treat everything as a template - mocked_templategen.is_template.return_value = True - installer.link_children(templater=templater, src=src_dir, dst=dst_dir, - actionexec=None) + installer.install(templater=templater, src=src_dir, dst=dst_dir, + linktype=LinkTypes.LINK_CHILDREN, actionexec=None) for src in srcs: dst = os.path.join(dst_dir, os.path.basename(src)) diff --git a/tests/test_listings.py b/tests/test_listings.py index 39905ec..72a82e6 100644 --- a/tests/test_listings.py +++ b/tests/test_listings.py @@ -9,7 +9,7 @@ import unittest import os from dotdrop.dotdrop import cmd_list_profiles -from dotdrop.dotdrop import cmd_list_files +from dotdrop.dotdrop import cmd_files from dotdrop.dotdrop import cmd_detail from dotdrop.dotdrop import cmd_importer @@ -87,9 +87,9 @@ class TestListings(unittest.TestCase): # list files o.files_templateonly = False - cmd_list_files(o) + cmd_files(o) o.files_templateonly = True - cmd_list_files(o) + cmd_files(o) # details o.detail_keys = None