Revamp and fix options

ordigi.conf

@@ -1,5 +1,17 @@
+[Exif]
+#album_from_folder=False
+fill_date_original=True
+#cache=True
+#ignore_tags=None
+use_date_filename=True
+#use_file_dates=False
+
 [Filters]
-exclude= ["**/.directory", "**/.DS_Store"]
+exclude=["**/.directory", "**/.DS_Store"]
+#extensions=None
+#glob=**/*
+#max_deep=None
+remove_duplicates=True
 
 [Geolocation]
 geocoder=Nominatim
@@ -15,5 +27,9 @@ day_begins=4
 
 # Path format
 dirs_path=<%Y>/<%m-%b>_<location>_<folder>
-name=<%Y%m%d-%H%M%S>_<<original_name>|<name>>.%l<ext>
+name=<%Y%m%d-%H%M%S>_<<name>.%l<ext>|<original_name>>
 # name=<%Y%m%d-%H%M%S>-%u<original_name>.%l<ext>
+
+[Terminal]
+dry_run=False
+interactive=False

ordigi/cli.py

@@ -7,6 +7,7 @@ import click
 
 from ordigi import log, LOG
 from ordigi.collection import Collection
+from ordigi import constants
 from ordigi.geolocation import GeoLocation
 from ordigi import utils
 
@@ -85,7 +86,7 @@ _sort_options = [
     click.option(
         '--path-format',
         '-p',
-        default=None,
+        default=constants.DEFAULT_PATH_FORMAT,
         help='Custom featured path format',
     ),
     click.option(
@@ -147,15 +148,10 @@ def _cli_get_location(collection):
     )
 
 
-def _cli_sort(collection, src_paths, import_mode, remove_duplicates):
+def _cli_sort(collection, src_paths, import_mode):
     loc = _cli_get_location(collection)
 
-    path_format = collection.opt['Path']['path_format']
+    return collection.sort_files(src_paths, loc, import_mode)
-    LOG.debug(f'path_format: {path_format}')
-
-    return collection.sort_files(
-        src_paths, path_format, loc, import_mode, remove_duplicates
-    )
 
 
 @click.group()
@@ -231,24 +227,21 @@ def _clean(**kwargs):
     collection = Collection(
         root,
         {
-            "dry_run": kwargs['dry_run'],
+            'dry_run': kwargs['dry_run'],
-            "extensions": kwargs['ext'],
+            'extensions': kwargs['ext'],
-            "glob": kwargs['glob'],
+            'glob': kwargs['glob'],
+            'remove_duplicates': kwargs['remove_duplicates'],
         },
     )
 
     # os.path.join(
     # TODO make function to remove duplicates
     # path_format = collection.opt['Path']['path_format']
-    # summary = collection.sort_files(
+    # summary = collection.sort_files(paths, None)
-    #     paths, path_format, None, remove_duplicates=kwargs['remove_duplicates']
-    # )
 
     if kwargs['path_string']:
         dedup_regex = set(kwargs['dedup_regex'])
-        collection.dedup_path(
+        collection.dedup_path(paths, dedup_regex)
-            paths, dedup_regex, kwargs['remove_duplicates']
-        )
 
     for path in paths:
         if folders:
@@ -334,9 +327,10 @@ def _compare(**kwargs):
     collection = Collection(
         root,
         {
-            "extensions": kwargs['ext'],
+            'extensions': kwargs['ext'],
-            "glob": kwargs['glob'],
+            'glob': kwargs['glob'],
-            "dry_run": kwargs['dry_run'],
+            'dry_run': kwargs['dry_run'],
+            'remove_duplicates': kwargs['remove_duplicates'],
         },
     )
 
@@ -515,7 +509,7 @@ def _import(**kwargs):
             'dry_run': kwargs['dry_run'],
             'interactive': kwargs['interactive'],
             'path_format': kwargs['path_format'],
-
+            'remove_duplicates': kwargs['remove_duplicates'],
         }
     )
 
@@ -523,7 +517,7 @@ def _import(**kwargs):
         import_mode = 'copy'
     else:
         import_mode = 'move'
-    summary = _cli_sort(collection, src_paths, import_mode, kwargs['remove_duplicates'])
+    summary = _cli_sort(collection, src_paths, import_mode)
 
     if log_level < 30:
         summary.print()
@@ -572,10 +566,11 @@ def _sort(**kwargs):
             'glob': kwargs['glob'],
             'dry_run': kwargs['dry_run'],
             'interactive': kwargs['interactive'],
+            'remove_duplicates': kwargs['remove_duplicates'],
         }
     )
 
-    summary = _cli_sort(collection, paths, False, kwargs['remove_duplicates'])
+    summary = _cli_sort(collection, paths, False)
 
     if kwargs['clean']:
         collection.remove_empty_folders(root)

ordigi/collection.py

@@ -477,6 +477,7 @@ class SortMedias:
         db=None,
         dry_run=False,
         interactive=False,
+        remove_duplicates=False,
     ):
 
         # Arguments
@@ -489,6 +490,7 @@ class SortMedias:
         self.dry_run = dry_run
         self.interactive = interactive
         self.log = LOG.getChild(self.__class__.__name__)
+        self.remove_duplicates = remove_duplicates
         self.summary = Summary(self.root)
 
         # Attributes
@@ -600,7 +602,7 @@ class SortMedias:
                 directory_path.mkdir(parents=True, exist_ok=True)
             self.log.info(f'Create {directory_path}')
 
-    def check_conflicts(self, src_path, dest_path, remove_duplicates=False):
+    def check_conflicts(self, src_path, dest_path):
         """
         Check if file can be copied or moved file to dest_path.
         """
@@ -616,7 +618,7 @@ class SortMedias:
 
         if dest_path.is_file():
             self.log.info(f"File {dest_path} already exist")
-            if remove_duplicates:
+            if self.remove_duplicates:
                 if filecmp.cmp(src_path, dest_path):
                     self.log.info(
                         "File in source and destination are identical. Duplicate will be ignored."
@@ -633,13 +635,13 @@ class SortMedias:
 
         return 0
 
-    def _solve_conflicts(self, conflicts, remove_duplicates):
+    def _solve_conflicts(self, conflicts):
         unresolved_conflicts = []
         while conflicts != []:
             src_path, dest_path, metadata = conflicts.pop()
             # Check for conflict status again in case is has changed
 
-            conflict = self.check_conflicts(src_path, dest_path, remove_duplicates)
+            conflict = self.check_conflicts(src_path, dest_path)
 
             for i in range(1, 1000):
                 if conflict != 1:
@@ -652,7 +654,7 @@ class SortMedias:
                 else:
                     stem = dest_path.stem
                 dest_path = dest_path.parent / (stem + '_' + str(i) + suffix)
-                conflict = self.check_conflicts(src_path, dest_path, remove_duplicates)
+                conflict = self.check_conflicts(src_path, dest_path)
 
             if conflict == 1:
                 # i = 100:
@@ -663,7 +665,7 @@ class SortMedias:
 
             yield (src_path, dest_path, metadata), conflict
 
-    def sort_medias(self, imp=False, remove_duplicates=False):
+    def sort_medias(self, imp=False):
         """
         sort files and solve conflicts
         """
@@ -674,7 +676,7 @@ class SortMedias:
         for src_path, metadata in self.medias.datas.items():
             dest_path = self.root / metadata['file_path']
 
-            conflict = self.check_conflicts(src_path, dest_path, remove_duplicates)
+            conflict = self.check_conflicts(src_path, dest_path)
 
             if not conflict:
                 self.sort_file(
@@ -692,9 +694,7 @@ class SortMedias:
                 pass
 
         if conflicts != []:
-            for files_data, conflict in self._solve_conflicts(
+            for files_data, conflict in self._solve_conflicts(conflicts):
-                conflicts, remove_duplicates
-            ):
 
                 src_path, dest_path, metadata = files_data
                 if not conflict:
@@ -727,13 +727,13 @@ class Collection(SortMedias):
         self.log = LOG.getChild(self.__class__.__name__)
 
         # Get config options
-        self.opt = self.get_config_options()
+        self.opt, default_options = self.get_config_options()
 
         # Set client options
         for option, value in cli_options.items():
-            if value not in (None, set()):
+            for section in self.opt:
-                for section in self.opt:
+                if option in self.opt[section]:
-                    if option in self.opt[section]:
+                    if value != default_options[section][option]:
                         if option == 'exclude':
                             self.opt[section][option].union(set(value))
                         elif option in ('ignore_tags', 'extensions'):
@@ -772,6 +772,7 @@ class Collection(SortMedias):
             self.db,
             self.opt['Terminal']['dry_run'],
             self.opt['Terminal']['interactive'],
+            self.opt['Filters']['remove_duplicates'],
         )
 
         # Attributes
@@ -792,7 +793,7 @@ class Collection(SortMedias):
         """Get collection config"""
         config = Config(self.root.joinpath('.ordigi', 'ordigi.conf'))
 
-        return config.get_config_options()
+        return config.get_config_options(), config.get_default_options()
 
     def _set_option(self, section, option, cli_option):
         """if client option is set overwrite collection option value"""
@@ -1003,17 +1004,15 @@ class Collection(SortMedias):
 
         return self.summary
 
-    def sort_files(
+    def sort_files(self, src_dirs, loc, imp=False):
-        self, src_dirs, path_format, loc, imp=False, remove_duplicates=False
-    ):
         """
         Sort files into appropriate folder
         """
         # Check db
         self._init_check_db(loc)
 
-        # if path format client option is set overwrite it
+        path_format = self.opt['Path']['path_format']
-        self._set_option('Path', 'path_format', path_format)
+        self.log.debug(f'path_format: {path_format}')
 
         # Get medias data
         subdirs = set()
@@ -1027,7 +1026,7 @@ class Collection(SortMedias):
             self.medias.datas[src_path] = copy(metadata)
 
         # Sort files and solve conflicts
-        self.summary = self.sort_medias(imp, remove_duplicates)
+        self.summary = self.sort_medias(imp)
 
         if imp != 'copy':
             self.remove_empty_subdirs(subdirs, src_dirs)
@@ -1037,7 +1036,7 @@ class Collection(SortMedias):
 
         return self.summary
 
-    def dedup_path(self, paths, dedup_regex=None, remove_duplicates=False):
+    def dedup_path(self, paths, dedup_regex=None):
         """Deduplicate file path parts"""
 
         # Check db
@@ -1078,7 +1077,7 @@ class Collection(SortMedias):
             self.medias.datas[src_path] = copy(metadata)
 
         # Sort files and solve conflicts
-        self.sort_medias(remove_duplicates=remove_duplicates)
+        self.sort_medias()
 
         if not self.check_db():
             self.summary.append('check', False)
@@ -1110,7 +1109,7 @@ class Collection(SortMedias):
 
         return True
 
-    def sort_similar_images(self, path, similarity=80, remove_duplicates=False):
+    def sort_similar_images(self, path, similarity=80):
         """Sort similar images using imagehash library"""
         # Check db
         self._init_check_db()
@@ -1129,7 +1128,7 @@ class Collection(SortMedias):
             )
             if similar_images:
                 # Move the simlars file into the destination directory
-                self.sort_medias(remove_duplicates=remove_duplicates)
+                self.sort_medias()
 
         nb_row_end = self.db.sqlite.len('metadata')
         if nb_row_ini and nb_row_ini != nb_row_end:

ordigi/config.py

@@ -53,9 +53,9 @@ class Config:
         else:
             self.conf = conf
 
-        self.options = self.set_default_options()
+        self.options = self.get_default_options()
 
-    def set_default_options(self) -> dict:
+    def get_default_options(self) -> dict:
         # Initialize with default options
         return {
             'Exif': {
@@ -71,6 +71,7 @@ class Config:
                 'extensions': None,
                 'glob': '**/*',
                 'max_deep': None,
+                'remove_duplicates': False,
             },
             'Geolocation': {
                 'geocoder': constants.DEFAULT_GEOCODER,
@@ -137,6 +138,7 @@ class Config:
             'dry_run',
             'interactive',
             'prefer_english_names',
+            'remove_duplicates',
             'use_date_filename',
             'use_file_dates',
         }

tests/test_cli.py

@@ -207,7 +207,6 @@ class TestOrdigi:
             ('--exclude', '.DS_Store'),
             *self.filter_options,
             ('--path-format', '{%Y}/{folder}/{stem}.{ext}'),
-
         )
 
         paths = (str(self.src_path), str(tmp_path))

tests/test_collection.py

@@ -137,11 +137,12 @@ class TestCollection:
         assert summary.success_table.sum('sort') == nb
 
     def test_sort_files(self, tmp_path):
-        cli_options = {'album_from_folder': True, 'cache': False}
+        cli_options = {
+            'album_from_folder': True, 'cache': False, 'path_format': self.path_format
+        }
         collection = Collection(tmp_path, cli_options=cli_options)
         loc = GeoLocation()
-        summary = collection.sort_files([self.src_path],
+        summary = collection.sort_files([self.src_path], loc, imp='copy')
-                self.path_format, loc, imp='copy')
 
         self.assert_import(summary, 29)
 
@@ -166,16 +167,16 @@ class TestCollection:
         collection = Collection(tmp_path, cli_options=cli_options)
         # Try to change path format and sort files again
         path_format = 'test_exif/<city>/<%Y>-<name>.%l<ext>'
-        summary = collection.sort_files([tmp_path], path_format, loc)
+        summary = collection.sort_files([tmp_path], loc)
 
-        self.assert_sort(summary, 27)
+        self.assert_sort(summary, 23)
 
         shutil.copytree(tmp_path / 'test_exif', tmp_path / 'test_exif_copy')
         collection.summary = Summary(tmp_path)
         assert collection.summary.success_table.sum() == 0
         summary = collection.update(loc)
-        assert summary.success_table.sum('update') == 29
+        assert summary.success_table.sum('update') == 2
-        assert summary.success_table.sum() == 29
+        assert summary.success_table.sum() == 2
         assert not summary.errors
         collection.summary = Summary(tmp_path)
         summary = collection.update(loc)
@@ -195,12 +196,11 @@ class TestCollection:
         assert not summary.errors
 
     def test_sort_files_invalid_db(self, tmp_path):
-        collection = Collection(tmp_path)
+        collection = Collection(tmp_path, {'path_format': self.path_format})
         loc = GeoLocation()
         randomize_db(tmp_path)
         with pytest.raises(sqlite3.DatabaseError) as e:
-            summary = collection.sort_files([self.src_path],
+            summary = collection.sort_files([self.src_path], loc, imp='copy')
-                    self.path_format, loc, imp='copy')
 
     def test_sort_file(self, tmp_path):
         for imp in ('copy', 'move', False):