From 3baa184a170338053873562dbf353bd3e870c387 Mon Sep 17 00:00:00 2001 From: Cedric Leporcq Date: Sat, 23 Jul 2022 20:12:03 +0200 Subject: [PATCH] Revamp and fix options --- ordigi.conf | 20 ++++++++++++++-- ordigi/cli.py | 41 +++++++++++++++------------------ ordigi/collection.py | 49 ++++++++++++++++++++-------------------- ordigi/config.py | 6 +++-- tests/test_cli.py | 1 - tests/test_collection.py | 20 ++++++++-------- 6 files changed, 74 insertions(+), 63 deletions(-) diff --git a/ordigi.conf b/ordigi.conf index e920de5..de4629a 100644 --- a/ordigi.conf +++ b/ordigi.conf @@ -1,5 +1,17 @@ +[Exif] +#album_from_folder=False +fill_date_original=True +#cache=True +#ignore_tags=None +use_date_filename=True +#use_file_dates=False + [Filters] -exclude= ["**/.directory", "**/.DS_Store"] +exclude=["**/.directory", "**/.DS_Store"] +#extensions=None +#glob=**/* +#max_deep=None +remove_duplicates=True [Geolocation] geocoder=Nominatim @@ -15,5 +27,9 @@ day_begins=4 # Path format dirs_path=<%Y>/<%m-%b>__ -name=<%Y%m%d-%H%M%S>_<|>.%l +name=<%Y%m%d-%H%M%S>_<.%l|> # name=<%Y%m%d-%H%M%S>-%u.%l + +[Terminal] +dry_run=False +interactive=False diff --git a/ordigi/cli.py b/ordigi/cli.py index 30ae81c..7d1f53a 100755 --- a/ordigi/cli.py +++ b/ordigi/cli.py @@ -7,6 +7,7 @@ import click from ordigi import log, LOG from ordigi.collection import Collection +from ordigi import constants from ordigi.geolocation import GeoLocation from ordigi import utils @@ -85,7 +86,7 @@ _sort_options = [ click.option( '--path-format', '-p', - default=None, + default=constants.DEFAULT_PATH_FORMAT, help='Custom featured path format', ), click.option( @@ -147,15 +148,10 @@ def _cli_get_location(collection): ) -def _cli_sort(collection, src_paths, import_mode, remove_duplicates): +def _cli_sort(collection, src_paths, import_mode): loc = _cli_get_location(collection) - path_format = collection.opt['Path']['path_format'] - LOG.debug(f'path_format: {path_format}') - - return collection.sort_files( - src_paths, path_format, loc, import_mode, remove_duplicates - ) + return collection.sort_files(src_paths, loc, import_mode) @click.group() @@ -231,24 +227,21 @@ def _clean(**kwargs): collection = Collection( root, { - "dry_run": kwargs['dry_run'], - "extensions": kwargs['ext'], - "glob": kwargs['glob'], + 'dry_run': kwargs['dry_run'], + 'extensions': kwargs['ext'], + 'glob': kwargs['glob'], + 'remove_duplicates': kwargs['remove_duplicates'], }, ) # os.path.join( # TODO make function to remove duplicates # path_format = collection.opt['Path']['path_format'] - # summary = collection.sort_files( - # paths, path_format, None, remove_duplicates=kwargs['remove_duplicates'] - # ) + # summary = collection.sort_files(paths, None) if kwargs['path_string']: dedup_regex = set(kwargs['dedup_regex']) - collection.dedup_path( - paths, dedup_regex, kwargs['remove_duplicates'] - ) + collection.dedup_path(paths, dedup_regex) for path in paths: if folders: @@ -334,9 +327,10 @@ def _compare(**kwargs): collection = Collection( root, { - "extensions": kwargs['ext'], - "glob": kwargs['glob'], - "dry_run": kwargs['dry_run'], + 'extensions': kwargs['ext'], + 'glob': kwargs['glob'], + 'dry_run': kwargs['dry_run'], + 'remove_duplicates': kwargs['remove_duplicates'], }, ) @@ -509,7 +503,7 @@ def _import(**kwargs): 'dry_run': kwargs['dry_run'], 'interactive': kwargs['interactive'], 'path_format': kwargs['path_format'], - + 'remove_duplicates': kwargs['remove_duplicates'], } ) @@ -517,7 +511,7 @@ def _import(**kwargs): import_mode = 'copy' else: import_mode = 'move' - summary = _cli_sort(collection, src_paths, import_mode, kwargs['remove_duplicates']) + summary = _cli_sort(collection, src_paths, import_mode) if log_level < 30: summary.print() @@ -566,10 +560,11 @@ def _sort(**kwargs): 'glob': kwargs['glob'], 'dry_run': kwargs['dry_run'], 'interactive': kwargs['interactive'], + 'remove_duplicates': kwargs['remove_duplicates'], } ) - summary = _cli_sort(collection, paths, False, kwargs['remove_duplicates']) + summary = _cli_sort(collection, paths, False) if kwargs['clean']: collection.remove_empty_folders(root) diff --git a/ordigi/collection.py b/ordigi/collection.py index 3375fef..99b89c2 100644 --- a/ordigi/collection.py +++ b/ordigi/collection.py @@ -478,6 +478,7 @@ class SortMedias: db=None, dry_run=False, interactive=False, + remove_duplicates=False, ): # Arguments @@ -490,6 +491,7 @@ class SortMedias: self.dry_run = dry_run self.interactive = interactive self.log = LOG.getChild(self.__class__.__name__) + self.remove_duplicates = remove_duplicates self.summary = Summary(self.root) # Attributes @@ -601,7 +603,7 @@ class SortMedias: directory_path.mkdir(parents=True, exist_ok=True) self.log.info(f'Create {directory_path}') - def check_conflicts(self, src_path, dest_path, remove_duplicates=False): + def check_conflicts(self, src_path, dest_path): """ Check if file can be copied or moved file to dest_path. """ @@ -617,7 +619,7 @@ class SortMedias: if dest_path.is_file(): self.log.info(f"File {dest_path} already exist") - if remove_duplicates: + if self.remove_duplicates: if filecmp.cmp(src_path, dest_path): self.log.info( "File in source and destination are identical. Duplicate will be ignored." @@ -634,13 +636,13 @@ class SortMedias: return 0 - def _solve_conflicts(self, conflicts, remove_duplicates): + def _solve_conflicts(self, conflicts): unresolved_conflicts = [] while conflicts != []: src_path, dest_path, metadata = conflicts.pop() # Check for conflict status again in case is has changed - conflict = self.check_conflicts(src_path, dest_path, remove_duplicates) + conflict = self.check_conflicts(src_path, dest_path) for i in range(1, 1000): if conflict != 1: @@ -653,7 +655,7 @@ class SortMedias: else: stem = dest_path.stem dest_path = dest_path.parent / (stem + '_' + str(i) + suffix) - conflict = self.check_conflicts(src_path, dest_path, remove_duplicates) + conflict = self.check_conflicts(src_path, dest_path) if conflict == 1: # i = 100: @@ -664,7 +666,7 @@ class SortMedias: yield (src_path, dest_path, metadata), conflict - def sort_medias(self, imp=False, remove_duplicates=False): + def sort_medias(self, imp=False): """ sort files and solve conflicts """ @@ -675,7 +677,7 @@ class SortMedias: for src_path, metadata in self.medias.datas.items(): dest_path = self.root / metadata['file_path'] - conflict = self.check_conflicts(src_path, dest_path, remove_duplicates) + conflict = self.check_conflicts(src_path, dest_path) if not conflict: self.sort_file( @@ -693,9 +695,7 @@ class SortMedias: pass if conflicts != []: - for files_data, conflict in self._solve_conflicts( - conflicts, remove_duplicates - ): + for files_data, conflict in self._solve_conflicts(conflicts): src_path, dest_path, metadata = files_data if not conflict: @@ -728,13 +728,13 @@ class Collection(SortMedias): self.log = LOG.getChild(self.__class__.__name__) # Get config options - self.opt = self.get_config_options() + self.opt, default_options = self.get_config_options() # Set client options for option, value in cli_options.items(): - if value not in (None, set()): - for section in self.opt: - if option in self.opt[section]: + for section in self.opt: + if option in self.opt[section]: + if value != default_options[section][option]: if option == 'exclude': self.opt[section][option].union(set(value)) elif option in ('ignore_tags', 'extensions'): @@ -773,6 +773,7 @@ class Collection(SortMedias): self.db, self.opt['Terminal']['dry_run'], self.opt['Terminal']['interactive'], + self.opt['Filters']['remove_duplicates'], ) # Attributes @@ -793,7 +794,7 @@ class Collection(SortMedias): """Get collection config""" config = Config(self.root.joinpath('.ordigi', 'ordigi.conf')) - return config.get_config_options() + return config.get_config_options(), config.get_default_options() def _set_option(self, section, option, cli_option): """if client option is set overwrite collection option value""" @@ -1004,17 +1005,15 @@ class Collection(SortMedias): return self.summary - def sort_files( - self, src_dirs, path_format, loc, imp=False, remove_duplicates=False - ): + def sort_files(self, src_dirs, loc, imp=False): """ Sort files into appropriate folder """ # Check db self._init_check_db(loc) - # if path format client option is set overwrite it - self._set_option('Path', 'path_format', path_format) + path_format = self.opt['Path']['path_format'] + self.log.debug(f'path_format: {path_format}') # Get medias data subdirs = set() @@ -1028,7 +1027,7 @@ class Collection(SortMedias): self.medias.datas[src_path] = copy(metadata) # Sort files and solve conflicts - self.summary = self.sort_medias(imp, remove_duplicates) + self.summary = self.sort_medias(imp) if imp != 'copy': self.remove_empty_subdirs(subdirs, src_dirs) @@ -1038,7 +1037,7 @@ class Collection(SortMedias): return self.summary - def dedup_path(self, paths, dedup_regex=None, remove_duplicates=False): + def dedup_path(self, paths, dedup_regex=None): """Deduplicate file path parts""" # Check db @@ -1079,7 +1078,7 @@ class Collection(SortMedias): self.medias.datas[src_path] = copy(metadata) # Sort files and solve conflicts - self.sort_medias(remove_duplicates=remove_duplicates) + self.sort_medias() if not self.check_db(): self.summary.append('check', False) @@ -1111,7 +1110,7 @@ class Collection(SortMedias): return True - def sort_similar_images(self, path, similarity=80, remove_duplicates=False): + def sort_similar_images(self, path, similarity=80): """Sort similar images using imagehash library""" # Check db self._init_check_db() @@ -1130,7 +1129,7 @@ class Collection(SortMedias): ) if similar_images: # Move the simlars file into the destination directory - self.sort_medias(remove_duplicates=remove_duplicates) + self.sort_medias() nb_row_end = self.db.sqlite.len('metadata') if nb_row_ini and nb_row_ini != nb_row_end: diff --git a/ordigi/config.py b/ordigi/config.py index 1c1b468..de1bad5 100644 --- a/ordigi/config.py +++ b/ordigi/config.py @@ -53,9 +53,9 @@ class Config: else: self.conf = conf - self.options = self.set_default_options() + self.options = self.get_default_options() - def set_default_options(self) -> dict: + def get_default_options(self) -> dict: # Initialize with default options return { 'Exif': { @@ -71,6 +71,7 @@ class Config: 'extensions': None, 'glob': '**/*', 'max_deep': None, + 'remove_duplicates': False, }, 'Geolocation': { 'geocoder': constants.DEFAULT_GEOCODER, @@ -137,6 +138,7 @@ class Config: 'dry_run', 'interactive', 'prefer_english_names', + 'remove_duplicates', 'use_date_filename', 'use_file_dates', } diff --git a/tests/test_cli.py b/tests/test_cli.py index eb519ae..952af28 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -206,7 +206,6 @@ class TestOrdigi: ('--exclude', '.DS_Store'), *self.filter_options, ('--path-format', '{%Y}/{folder}/{stem}.{ext}'), - ) paths = (str(self.src_path), str(tmp_path)) diff --git a/tests/test_collection.py b/tests/test_collection.py index 0ad6734..22b3a93 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -137,11 +137,12 @@ class TestCollection: assert summary.success_table.sum('sort') == nb def test_sort_files(self, tmp_path): - cli_options = {'album_from_folder': True, 'cache': False} + cli_options = { + 'album_from_folder': True, 'cache': False, 'path_format': self.path_format + } collection = Collection(tmp_path, cli_options=cli_options) loc = GeoLocation() - summary = collection.sort_files([self.src_path], - self.path_format, loc, imp='copy') + summary = collection.sort_files([self.src_path], loc, imp='copy') self.assert_import(summary, 29) @@ -166,16 +167,16 @@ class TestCollection: collection = Collection(tmp_path, cli_options=cli_options) # Try to change path format and sort files again path_format = 'test_exif//<%Y>-.%l' - summary = collection.sort_files([tmp_path], path_format, loc) + summary = collection.sort_files([tmp_path], loc) - self.assert_sort(summary, 27) + self.assert_sort(summary, 23) shutil.copytree(tmp_path / 'test_exif', tmp_path / 'test_exif_copy') collection.summary = Summary(tmp_path) assert collection.summary.success_table.sum() == 0 summary = collection.update(loc) - assert summary.success_table.sum('update') == 29 - assert summary.success_table.sum() == 29 + assert summary.success_table.sum('update') == 2 + assert summary.success_table.sum() == 2 assert not summary.errors collection.summary = Summary(tmp_path) summary = collection.update(loc) @@ -195,12 +196,11 @@ class TestCollection: assert not summary.errors def test_sort_files_invalid_db(self, tmp_path): - collection = Collection(tmp_path) + collection = Collection(tmp_path, {'path_format': self.path_format}) loc = GeoLocation() randomize_db(tmp_path) with pytest.raises(sqlite3.DatabaseError) as e: - summary = collection.sort_files([self.src_path], - self.path_format, loc, imp='copy') + summary = collection.sort_files([self.src_path], loc, imp='copy') def test_sort_file(self, tmp_path): for imp in ('copy', 'move', False):