From d008e6a895aa670cbbedc53362d03709a6a92d1b Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 6 Jun 2019 17:11:13 +0200 Subject: [PATCH] migrate from PyYAML to ruamel.yaml --- dotdrop/cfg_yaml.py | 57 ++++++++++++++++------------------ dotdrop/dotdrop.py | 5 ++- packages/arch-dotdrop/.SRCINFO | 2 +- packages/arch-dotdrop/PKGBUILD | 2 +- requirements.txt | 2 +- scripts/change-link.py | 4 +-- setup.py | 2 +- tests/helpers.py | 32 ++++++++++++------- tests/test_import.py | 9 ++---- tests/test_remove.py | 15 +++------ tests/test_yamlcfg.py | 17 +++------- 11 files changed, 71 insertions(+), 76 deletions(-) diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index a07d224..48949bb 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -6,7 +6,7 @@ handle lower level of the config file """ import os -import yaml +from ruamel.yaml import YAML as yaml import glob from copy import deepcopy @@ -572,12 +572,11 @@ class CfgYaml: content = {} if not os.path.exists(path): raise YamlException('config path not found: {}'.format(path)) - with open(path, 'r') as f: - try: - content = yaml.safe_load(f) - except Exception as e: - self.log.err(e) - raise YamlException('invalid config: {}'.format(path)) + try: + content = self._yaml_load(path) + except Exception as e: + self.log.err(e) + raise YamlException('invalid config: {}'.format(path)) return content def _new_profile(self, key): @@ -733,18 +732,6 @@ class CfgYaml: new[k] = newv return new - def _yaml_version(self): - """returns True if version >= 5.1""" - minv = [5, 1] - try: - cur = list(map(int, yaml.__version__.split('.'))) - if cur.pop(0) >= minv.pop(0) and \ - cur.pop(1) >= minv.pop(1): - return True - except Exception: - return False - return False - def save(self): """save this instance and return True if saved""" if not self.dirty: @@ -760,19 +747,14 @@ class CfgYaml: if self.key_profiles not in content: content[self.key_profiles] = None - opts = {'default_flow_style': False, 'indent': 2} - if self._yaml_version(): - opts['sort_keys'] = False - data = yaml.safe_dump(content, **opts) - - # ensure no null are displayed - data = data.replace('null', '') - # save to file if self.debug: - self.log.dbg('saving: {}'.format(content)) - with open(self.path, 'w') as f: - f.write(data) + self.log.dbg('saving to {}'.format(self.path)) + try: + self._yaml_dump(content, self.path) + except Exception as e: + self.log.err(e) + raise YamlException('error saving config: {}'.format(self.path)) self.dirty = False return True @@ -780,3 +762,18 @@ class CfgYaml: def dump(self): """dump the config dictionary""" return self.yaml_dict + + def _yaml_load(self, path): + """load from yaml""" + with open(path, 'r') as f: + content = yaml(typ='safe').load(f) + return content + + def _yaml_dump(self, content, path): + """dump to yaml""" + with open(self.path, 'w') as f: + y = yaml() + y.default_flow_style = False + y.indent = 2 + y.typ = 'safe' + y.dump(content, f) diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 31eb1cf..d60f656 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -472,7 +472,10 @@ def cmd_remove(o): LOG.raw(o.conf.dump()) else: o.conf.save() - LOG.log('\ndotfile(s) removed: {}'.format(','.join(removed))) + if removed: + LOG.log('\ndotfile(s) removed: {}'.format(','.join(removed))) + else: + LOG.log('\nno dotfile removed') return True diff --git a/packages/arch-dotdrop/.SRCINFO b/packages/arch-dotdrop/.SRCINFO index 26b449c..67b5eaf 100644 --- a/packages/arch-dotdrop/.SRCINFO +++ b/packages/arch-dotdrop/.SRCINFO @@ -10,7 +10,7 @@ pkgbase = dotdrop depends = python-setuptools depends = python-jinja depends = python-docopt - depends = python-pyaml + depends = python-ruamel-yaml source = git+https://github.com/deadc0de6/dotdrop.git#tag=v0.28.0 md5sums = SKIP diff --git a/packages/arch-dotdrop/PKGBUILD b/packages/arch-dotdrop/PKGBUILD index 5891d6a..9e4e779 100644 --- a/packages/arch-dotdrop/PKGBUILD +++ b/packages/arch-dotdrop/PKGBUILD @@ -8,7 +8,7 @@ arch=('any') url="https://github.com/deadc0de6/dotdrop" license=('GPL') groups=() -depends=('python' 'python-setuptools' 'python-jinja' 'python-docopt' 'python-pyaml') +depends=('python' 'python-setuptools' 'python-jinja' 'python-docopt' 'python-ruamel-yaml') makedepends=('git') source=("git+https://github.com/deadc0de6/dotdrop.git#tag=v${pkgver}") md5sums=('SKIP') diff --git a/requirements.txt b/requirements.txt index 4fbbcb1..2c6a2c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ Jinja2; python_version >= '3.0' docopt; python_version >= '3.0' -PyYAML; python_version >= '3.0' +ruamel.yaml; python_version >= '3.0' diff --git a/scripts/change-link.py b/scripts/change-link.py index f2ee974..7a0854b 100755 --- a/scripts/change-link.py +++ b/scripts/change-link.py @@ -13,7 +13,7 @@ usage example: from docopt import docopt import sys import os -import yaml +from ruamel.yaml import YAML as yaml USAGE = """ change-link.py @@ -42,7 +42,7 @@ def main(): ignores = args['--ignore'] with open(path, 'r') as f: - content = yaml.safe_load(f) + content = yaml(typ='safe').load(f) for k, v in content[key].items(): if k in ignores: continue diff --git a/setup.py b/setup.py index 73a3a78..455eb35 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ setup( keywords='dotfiles jinja2', packages=find_packages(exclude=['tests*']), - install_requires=['docopt', 'Jinja2', 'PyYAML'], + install_requires=['docopt', 'Jinja2', 'ruamel.yaml'], extras_require={ 'dev': ['check-manifest'], diff --git a/tests/helpers.py b/tests/helpers.py index 7bd57f6..836101b 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -11,7 +11,7 @@ import string import tempfile from unittest import TestCase -import yaml +from ruamel.yaml import YAML as yaml from dotdrop.options import Options from dotdrop.linktypes import LinkTypes @@ -221,9 +221,8 @@ def create_yaml_keyval(pairs, parent_dir=None, top_key=None): if not parent_dir: parent_dir = get_tempdir() - fd, file_name = tempfile.mkstemp(dir=parent_dir, suffix='.yaml', text=True) - with os.fdopen(fd, 'w') as f: - yaml.safe_dump(pairs, f) + _, file_name = tempfile.mkstemp(dir=parent_dir, suffix='.yaml', text=True) + yaml_dump(pairs, file_name) return file_name @@ -234,8 +233,7 @@ def populate_fake_config(config, dotfiles={}, profiles={}, actions={}, is_path = isinstance(config, str) if is_path: config_path = config - with open(config_path) as config_file: - config = yaml.safe_load(config_file) + config = yaml_load(config_path) config['dotfiles'] = dotfiles config['profiles'] = profiles @@ -246,9 +244,7 @@ def populate_fake_config(config, dotfiles={}, profiles={}, actions={}, config['dynvariables'] = dynvariables if is_path: - with open(config_path, 'w') as config_file: - yaml.safe_dump(config, config_file, default_flow_style=False, - indent=2) + yaml_dump(config, config_path) def file_in_yaml(yaml_file, path, link=False): @@ -256,8 +252,7 @@ def file_in_yaml(yaml_file, path, link=False): strip = get_path_strip_version(path) if isinstance(yaml_file, str): - with open(yaml_file) as f: - yaml_conf = yaml.safe_load(f) + yaml_conf = yaml_load(yaml_file) else: yaml_conf = yaml_file @@ -275,3 +270,18 @@ def file_in_yaml(yaml_file, path, link=False): return False return in_src and in_dst and has_link return in_src and in_dst + + +def yaml_load(path): + with open(path, 'r') as f: + content = yaml(typ='safe').load(f) + return content + + +def yaml_dump(content, path): + with open(path, 'w') as f: + y = yaml() + y.default_flow_style = False + y.indent = 2 + y.typ = 'safe' + y.dump(content, f) diff --git a/tests/test_import.py b/tests/test_import.py index 98efdea..801ff55 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -7,7 +7,6 @@ basic unittest for the import function import unittest import os -import yaml from dotdrop.dotdrop import cmd_importer from dotdrop.dotdrop import cmd_list_profiles @@ -18,7 +17,8 @@ from dotdrop.linktypes import LinkTypes from tests.helpers import (clean, create_dir, create_fake_config, create_random_file, edit_content, file_in_yaml, get_path_strip_version, get_string, get_tempdir, - load_options, populate_fake_config) + load_options, populate_fake_config, + yaml_load) class TestImport(unittest.TestCase): @@ -31,10 +31,7 @@ class TestImport(unittest.TestCase): def load_yaml(self, path): """Load yaml to dict""" self.assertTrue(os.path.exists(path)) - content = '' - with open(path, 'r') as f: - content = yaml.safe_load(f) - return content + return yaml_load(path) def assert_file(self, path, o, profile): """Make sure path has been inserted in conf for profile""" diff --git a/tests/test_remove.py b/tests/test_remove.py index df01dce..2c8d262 100644 --- a/tests/test_remove.py +++ b/tests/test_remove.py @@ -4,7 +4,6 @@ Copyright (c) 2019, deadc0de6 basic unittest for the remove function """ -import yaml import unittest import os @@ -12,7 +11,7 @@ import os from dotdrop.dotdrop import cmd_remove from tests.helpers import (clean, create_dir, create_random_file, load_options, - get_tempdir) + get_tempdir, yaml_load, yaml_dump) class TestRemove(unittest.TestCase): @@ -20,10 +19,7 @@ class TestRemove(unittest.TestCase): def load_yaml(self, path): """Load yaml to dict""" self.assertTrue(os.path.exists(path)) - content = '' - with open(path, 'r') as f: - content = yaml.safe_load(f) - return content + return yaml_load(path) def test_remove(self): """test the remove command""" @@ -71,8 +67,7 @@ class TestRemove(unittest.TestCase): }, } - with open(confpath, 'w') as f: - yaml.safe_dump(configdic, f) + yaml_dump(configdic, confpath) o = load_options(confpath, 'host1') o.remove_path = ['f_test1'] o.remove_iskey = True @@ -87,7 +82,7 @@ class TestRemove(unittest.TestCase): self.assertTrue(os.path.exists(df3)) # load dict - y = self.load_yaml(confpath) + y = yaml_load(confpath) # ensure not present self.assertTrue('f_test1' not in y['dotfiles']) @@ -114,7 +109,7 @@ class TestRemove(unittest.TestCase): self.assertFalse(os.path.exists(df3)) # load dict - y = self.load_yaml(confpath) + y = yaml_load(confpath) # ensure not present self.assertTrue('f_test3' not in y['dotfiles']) diff --git a/tests/test_yamlcfg.py b/tests/test_yamlcfg.py index 0a22420..33570b1 100644 --- a/tests/test_yamlcfg.py +++ b/tests/test_yamlcfg.py @@ -8,14 +8,13 @@ basic unittest for the config parser import unittest from unittest.mock import patch import os -import yaml from dotdrop.cfg_yaml import CfgYaml as Cfg from dotdrop.options import Options from dotdrop.linktypes import LinkTypes from tests.helpers import (SubsetTestCase, _fake_args, clean, create_fake_config, create_yaml_keyval, get_tempdir, - populate_fake_config) + populate_fake_config, yaml_load, yaml_dump) class TestConfig(SubsetTestCase): @@ -142,8 +141,7 @@ profiles: create=self.CONFIG_CREATE) # edit the config - with open(confpath, 'r') as f: - content = yaml.safe_load(f) + content = yaml_load(confpath) # adding dotfiles df1key = 'f_vimrc' @@ -162,9 +160,7 @@ profiles: } # save the new config - with open(confpath, 'w') as f: - yaml.safe_dump(content, f, default_flow_style=False, - indent=2) + yaml_dump(content, confpath) # do the tests conf = Cfg(confpath, debug=True) @@ -185,17 +181,14 @@ profiles: # test not existing included profile # edit the config - with open(confpath, 'r') as f: - content = yaml.safe_load(f) + content = yaml_load(confpath) content['profiles'] = { pf1key: {'dotfiles': [df2key], 'include': ['host2']}, pf2key: {'dotfiles': [df1key], 'include': ['host3']} } # save the new config - with open(confpath, 'w') as f: - yaml.safe_dump(content, f, default_flow_style=False, - indent=2) + yaml_dump(content, confpath) # do the tests conf = Cfg(confpath, debug=True)