From 27af9bb55e663432fc8ce283a3acb2a2328028dc Mon Sep 17 00:00:00 2001 From: Cedric Leporcq Date: Sat, 13 Nov 2021 18:20:08 +0100 Subject: [PATCH] Refactoring collection options (2) --- ordigi.conf | 17 ++-- ordigi/cli.py | 8 -- ordigi/collection.py | 86 ++++++++++++------- ordigi/config.py | 191 ++++++++++++++++++++++++++++++++----------- tests/test_config.py | 19 +++-- 5 files changed, 218 insertions(+), 103 deletions(-) diff --git a/ordigi.conf b/ordigi.conf index 3e217da..86cccc8 100644 --- a/ordigi.conf +++ b/ordigi.conf @@ -1,3 +1,11 @@ +[Filters] +exclude= ["**/.directory", "**/.DS_Store"] + +[Geolocation] +geocoder=Nominatim +prefer_english_names=hjkh +timeout=1 + [Path] # day_begins: what hour of the day you want the day to begin (only for # classification purposes). Defaults at 0 as midnight. Can be @@ -5,14 +13,7 @@ # be a number between 0-23') day_begins=4 +# Path format dirs_path={%Y}/{%m-%b}-{city}-{folder} name={%Y%m%d-%H%M%S}-%u{original_name}|%u{basename}.%l{ext} -[Exclusions] -path1=**/.directory -path2=**/.DS_Store - -[Geolocation] -geocoder=Nominatim -prefer_english_names=False -# timeout=1 diff --git a/ordigi/cli.py b/ordigi/cli.py index e56edce..84c31eb 100755 --- a/ordigi/cli.py +++ b/ordigi/cli.py @@ -8,7 +8,6 @@ import sys import click from ordigi import constants, log, LOG -from ordigi.config import Config from ordigi.collection import Collection from ordigi.geolocation import GeoLocation @@ -116,13 +115,6 @@ def add_options(options): return _add_options -def _get_exclude(opt, exclude): - # if no exclude list was passed in we check if there's a config - if len(exclude) == 0: - exclude = opt['exclude'] - return set(exclude) - - def _get_paths(paths, root): root = Path(root).expanduser().absolute() if not paths: diff --git a/ordigi/collection.py b/ordigi/collection.py index 72d3f05..d501649 100644 --- a/ordigi/collection.py +++ b/ordigi/collection.py @@ -14,6 +14,7 @@ from pathlib import Path, PurePath import inquirer from ordigi import LOG +from ordigi.config import Config from ordigi.database import Sqlite from ordigi.media import Medias from ordigi.images import Image, Images @@ -220,6 +221,9 @@ class FPath: Returns file path. """ path_format = self.path_format + + # Each element in the list represents a folder. + # Fallback folders are supported and are nested lists. path = [] path_parts = path_format.split('/') for path_part in path_parts: @@ -706,50 +710,58 @@ class Collection(SortMedias): use_file_dates=False, ): - day_begins=0 - max_deep=None + # TODO move to cli + cli_options = { + 'album_from_folder': album_from_folder, + 'cache': cache, + 'dry_run': dry_run, + 'exclude': exclude, + 'extensions': extensions, + 'glob': '**/*', + 'interactive': interactive, + 'ignore_tags': ignore_tags, + 'use_date_filename': use_date_filename, + 'use_file_dates': use_file_dates, + } - # Options - self.day_begins = day_begins self.log = LOG.getChild(self.__class__.__name__) - self.glob = glob # Check if collection path is valid if not root.exists(): self.log.error(f'Collection path {root} does not exist') sys.exit(1) - # def get_collection_config(root): - # return Config(root.joinpath('.ordigi', 'ordigi.conf')) + self.root = root - # TODO Collection options - # config = get_collection_config(root) - # opt = config.get_options() - # exclude = _get_exclude(opt, kwargs['exclude']) - # path_format = opt['path_format'] - # if kwargs['path_format']: - # path_format = kwargs['path_format'] + # Get config options + config = self.get_config() + self.opt = config.get_options() + + # Set client options + for option, value in cli_options.items(): + self.opt[option] = value + self._set_cli_option('exclude', exclude) self.db = CollectionDb(root) - self.fileio = FileIO(dry_run) + self.fileio = FileIO(self.opt['dry_run']) self.paths = Paths( - exclude, - extensions, - glob, - interactive, - max_deep, + self.opt['exclude'], + self.opt['extensions'], + self.opt['glob'], + self.opt['interactive'], + self.opt['max_deep'], ) self.medias = Medias( self.paths, root, - album_from_folder, - cache, + self.opt['album_from_folder'], + self.opt['cache'], self.db, - interactive, - ignore_tags, - use_date_filename, - use_file_dates, + self.opt['interactive'], + self.opt['ignore_tags'], + self.opt['use_date_filename'], + self.opt['use_file_dates'], ) # Features @@ -758,21 +770,30 @@ class Collection(SortMedias): self.medias, root, self.db, - dry_run, - interactive, + self.opt['dry_run'], + self.opt['interactive'], ) # Attributes self.summary = Summary(self.root) self.theme = request.load_theme() + def get_config(self): + """Get collection config""" + return Config(self.root.joinpath('.ordigi', 'ordigi.conf')) + + def _set_cli_option(self, option, cli_option): + """if client option is set overwrite collection option value""" + if cli_option: + self.opt['option'] = cli_option + def get_collection_files(self, exclude=True): if exclude: exclude = self.paths.exclude paths = Paths( exclude, - interactive=self.interactive, + interactive=self.opt['interactive'], ) for file_path in paths.get_files(self.root): yield file_path @@ -932,7 +953,7 @@ class Collection(SortMedias): files = os.listdir(directory) if len(files) == 0 and remove_root: self.log.info(f"Removing empty folder: {directory}") - if not self.dry_run: + if not self.opt['dry_run']: os.rmdir(directory) self.summary.append('remove', True, directory) @@ -948,11 +969,14 @@ class Collection(SortMedias): # Check db self._init_check_db(loc) + # if path format client option is set overwrite it + self._set_cli_option('path_format', path_format) + # Get medias data subdirs = set() for src_path, metadata in self.medias.get_metadatas(src_dirs, imp=imp, loc=loc): # Get the destination path according to metadata - fpath = FPath(path_format, self.day_begins) + fpath = FPath(path_format, self.opt['day_begins']) metadata['file_path'] = fpath.get_path(metadata) subdirs.add(src_path.parent) diff --git a/ordigi/config.py b/ordigi/config.py index c5cc984..ca723bc 100644 --- a/ordigi/config.py +++ b/ordigi/config.py @@ -1,15 +1,80 @@ +import json +import re + from configparser import RawConfigParser from os import path from ordigi import constants from geopy.geocoders import options as gopt +# TODO make one method??? +def check_option(getoption): + """Check option type int or boolean""" + try: + getoption + except ValueError as e: + # TODO + return None + else: + return getoption + +def check_json(getoption): + """Check if json string is valid""" + try: + getoption + except json.JSONDecodeError as e: + # TODO + return None + else: + return getoption + +def check_re(getoption): + """Check if regex string is valid""" + try: + getoption + except re.error as e: + # TODO + return None + else: + return getoption + + class Config: """Manage config file""" - def __init__(self, conf_path=constants.CONFIG_FILE, conf={}): + # Initialize with default options + options: dict = { + 'Console': { + 'dry_run': False, + 'interactive': False, + }, + 'Database': { + 'cache': False, + 'album_from_folder': False, + 'ignore_tags': None, + 'use_date_filename': False, + 'use_file_dates': False, + }, + 'Filters': { + 'exclude': None, + 'extensions': None, + 'glob': '**/*', + 'max_deep': None, + }, + 'Geolocation': { + 'geocoder': constants.DEFAULT_GEOCODER, + 'prefer_english_names': False, + 'timeout': gopt.default_timeout, + }, + 'Path': { + 'day_begins': 0, + 'path_format': constants.DEFAULT_PATH_FORMAT, + }, + } + + def __init__(self, conf_path=constants.CONFIG_FILE, conf=None): self.conf_path = conf_path - if conf == {}: + if conf is None: self.conf = self.load_config() if self.conf == {}: # Fallback to default config @@ -33,66 +98,94 @@ class Config: conf.read(self.conf_path) return conf - def get_option(self, option, section): - + def is_option(self, section, option): + """Get ConfigParser options""" if section in self.conf and option in self.conf[section]: - return self.conf[section][option] + return True return False - def get_path_definition(self): - """Returns a list of folder definitions. + @check_option + def _getboolean(self, section, option): + return self.conf.getboolean(section, option) + getboolean = check_option(_getboolean) - Each element in the list represents a folder. - Fallback folders are supported and are nested lists. + @check_option + def _getint(self, section, option): + return self.conf.getint(section, option) + getint = check_option(_getint) - :returns: string - """ + @check_json + def _getjson(self, section, option): + return json.loads(self.conf.get(section, option)) + getjson = check_json(_getjson) - if 'Path' in self.conf: - if 'format' in self.conf['Path']: - return self.conf['Path']['format'] - elif 'dirs_path' and 'name' in self.conf['Path']: - return self.conf['Path']['dirs_path'] + '/' + self.conf['Path']['name'] + @check_re + def _getre(self, section, option): + return re.compile(self.conf.get(section, option)) + getre = check_re(_getre) - return constants.DEFAULT_PATH_FORMAT + def get_option(self, section, option): + bool_options = { + 'cache', + 'dry_run', + 'prefer_english_names', + 'album_from_folder', + 'interactive', + 'use_date_filename', + 'use_file_dates', + } - def get_options(self): - """Get config options - :returns: dict - """ + int_options = { + 'day_begins', + 'max_deep', + 'timeout', + } - options = {} - geocoder = self.get_option('geocoder', 'Geolocation') - if geocoder and geocoder in ('Nominatim',): - options['geocoder'] = geocoder - else: - options['geocoder'] = constants.DEFAULT_GEOCODER + string_options = { + 'glob', + 'geocoder', + } - prefer_english_names = self.get_option('prefer_english_names', 'Geolocation') - if prefer_english_names: - options['prefer_english_names'] = bool(prefer_english_names) - else: - options['prefer_english_names'] = False + multi_options = { + 'exclude', + 'extensions', + 'ignore_tags', + } - timeout = self.get_option('timeout', 'Geolocation') - if timeout: - options['timeout'] = timeout - else: - options['timeout'] = gopt.default_timeout + value = self.options[section][option] + if self.is_option(section, option): + if option in bool_options: + return self.getboolean(section, option) + if option in int_options: + return self.getint(section, option) + if option == 'geocoder' and value in ('Nominatim',): + return self.conf[section][option] + if option == 'glob': + return self.conf[section][option] + if option == 'path_format': + return self.getre(section, option) + if option in multi_options: + return set(self.getjson(section, option)) - options['path_format'] = self.get_path_definition() + return value - options['day_begins'] = 0 - options['max_deep'] = None - if 'Path' in self.conf: - if 'day_begins' in self.conf['Path']: - options['day_begins'] = int(self.conf['Path']['day_begins']) - if 'max_deep' in self.conf['Path']: - options['max_deep'] = int(self.conf['Path']['max_deep']) + if self.is_option('Path', 'name') and self.is_option('dirs_path', option): + # Path format is split in two parts + value = self.conf['Path']['dirs_path'] + '/' + self.conf['Path']['name'] - options['exclude'] = [] - if 'Exclusions' in self.conf: - options['exclude'] = [value for key, value in self.conf.items('Exclusions')] + return value - return options + def get_options(self) -> dict: + """Get config options""" + + old_options = {} + for section in self.options: + for option in self.options[section]: + # Option is in section + # TODO make a function + value = self.get_option(section, option) + old_options[option] = value + self.options[section][option] = value + + return old_options diff --git a/tests/test_config.py b/tests/test_config.py index aa4bceb..5d49f98 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -50,11 +50,16 @@ class TestConfig: config = Config(conf_path) assert e.typename == 'MissingSectionHeaderError' - def test_get_path_definition(self, conf): - """ - Get path definition from config - """ - config = Config(conf=conf) - path = config.get_path_definition() - assert path == '%u{%Y-%m}/{city}|{city}-{%Y}/{folders[:1]}/{folder}/{%Y-%m-%b-%H-%M-%S}-{basename}.%l{ext}' + # def test_get_path_definition(self, conf): + # """ + # Get path definition from config + # """ + # config = Config(conf=conf) + # path = config.get_path_definition() + # assert path == '%u{%Y-%m}/{city}|{city}-{%Y}/{folders[:1]}/{folder}/{%Y-%m-%b-%H-%M-%S}-{basename}.%l{ext}' + def test_get_options(self, conf): + config = Config(conf=conf) + options = config.get_options() + assert isinstance(options, dict) + # assert isinstance(options['Path'], dict)