Refactoring sort_file

ordigi.conf

@@ -6,7 +6,7 @@
 day_begins=4
 
 dirs_path={%Y}/{%m-%b}-{city}-{folder}
-name={%Y%m%d-%H%M%S}-%u{original_name}.%l{ext}
+name={%Y%m%d-%H%M%S}-%u{original_name}|%u{basename}.%l{ext}
 
 [Exclusions]
 path1=**/.directory

ordigi/cli.py

@@ -29,6 +29,12 @@ _logger_options = [
     ),
 ]
 
+_input_options = [
+    click.option(
+        '--interactive', '-i', default=False, is_flag=True, help="Interactive mode"
+    ),
+]
+
 _dry_run_options = [
     click.option(
         '--dry-run',
@@ -38,7 +44,7 @@ _dry_run_options = [
     )
 ]
 
-_filter_option = [
+_filter_options = [
     click.option(
         '--exclude',
         '-e',
@@ -59,6 +65,50 @@ _filter_option = [
     click.option('--glob', '-g', default='**/*', help='Glob file selection'),
 ]
 
+_sort_options = [
+    click.option(
+        '--album-from-folder',
+        default=False,
+        is_flag=True,
+        help="Use images' folders as their album names.",
+    ),
+    click.option(
+        '--ignore-tags',
+        '-I',
+        default=set(),
+        multiple=True,
+        help='Specific tags or group that will be ignored when\
+                  searching for file data. Example \'File:FileModifyDate\' or \'Filename\'',
+    ),
+    click.option(
+        '--path-format',
+        '-p',
+        default=None,
+        help='Custom featured path format',
+    ),
+    click.option(
+        '--remove-duplicates',
+        '-R',
+        default=False,
+        is_flag=True,
+        help='True to remove files that are exactly the same in name\
+                          and a file hash',
+    ),
+    click.option(
+        '--use-date-filename',
+        '-f',
+        default=False,
+        is_flag=True,
+        help="Use filename date for media original date.",
+    ),
+    click.option(
+       '--use-file-dates',
+        '-F',
+        default=False,
+        is_flag=True,
+        help="Use file date created or modified for media original date.",
+    ),
+]
 
 def print_help(command):
     click.echo(command.get_help(click.Context(sort)))
@@ -79,27 +129,25 @@ def _get_exclude(opt, exclude):
         exclude = opt['exclude']
     return set(exclude)
 
+
 def get_collection_config(root):
     return Config(os.path.join(root, '.ordigi', 'ordigi.conf'))
 
-@click.command('sort')
+
+def _get_paths(paths, root):
+    if not paths:
+        paths = root
+    paths = set(paths)
+
+    return paths, root
+
+
+@click.command('import')
 @add_options(_logger_options)
+@add_options(_input_options)
 @add_options(_dry_run_options)
-@add_options(_filter_option)
+@add_options(_filter_options)
-@click.option(
+@add_options(_sort_options)
-    '--album-from-folder',
-    default=False,
-    is_flag=True,
-    help="Use images' folders as their album names.",
-)
-@click.option(
-    '--destination',
-    '-d',
-    type=click.Path(file_okay=False),
-    default=None,
-    help='Sort files into this directory.',
-)
-@click.option('--clean', '-C', default=False, is_flag=True, help='Clean empty folders')
 @click.option(
     '--copy',
     '-c',
@@ -108,31 +156,70 @@ def get_collection_config(root):
     help='True if you want files to be copied over from src_dir to\
               dest_dir rather than moved',
 )
-@click.option(
+@click.argument('src', required=False, nargs=-1, type=click.Path())
-    '--ignore-tags',
+@click.argument('dest', required=True, nargs=1, type=click.Path())
-    '-I',
+def _import(**kwargs):
-    default=set(),
+    """Sort files or directories by reading their EXIF and organizing them
-    multiple=True,
+    according to ordigi.conf preferences.
-    help='Specific tags or group that will be ignored when\
+    """
-              searching for file data. Example \'File:FileModifyDate\' or \'Filename\'',
+
-)
+    log_level = log.level(kwargs['verbose'], kwargs['debug'])
-@click.option(
+    logger = log.get_logger(level=log_level)
-    '--interactive', '-i', default=False, is_flag=True, help="Interactive mode"
+
-)
+    src_paths = kwargs['src']
-@click.option(
+    root = kwargs['dest']
-    '--path-format',
+    src_paths, root = _get_paths(src_paths, root)
-    '-p',
+
-    default=None,
+    if kwargs['copy']:
-    help='set custom featured path format',
+        import_mode = 'copy'
-)
+    else:
-@click.option(
+        import_mode = 'move'
-    '--remove-duplicates',
+
-    '-R',
+    config = get_collection_config(root)
-    default=False,
+    opt = config.get_options()
-    is_flag=True,
+
-    help='True to remove files that are exactly the same in name\
+    path_format = opt['path_format']
-                      and a file hash',
+    if kwargs['path_format']:
-)
+        path_format = kwargs['path_format']
+
+    exclude = _get_exclude(opt, kwargs['exclude'])
+    filter_by_ext = set(kwargs['filter_by_ext'])
+
+    collection = Collection(
+        root,
+        kwargs['album_from_folder'],
+        False,
+        opt['day_begins'],
+        kwargs['dry_run'],
+        exclude,
+        filter_by_ext,
+        kwargs['glob'],
+        kwargs['interactive'],
+        logger,
+        opt['max_deep'],
+        kwargs['use_date_filename'],
+        kwargs['use_file_dates'],
+    )
+
+    loc = GeoLocation(opt['geocoder'], logger, opt['prefer_english_names'], opt['timeout'])
+
+    summary = collection.sort_files(
+        src_paths, path_format, loc, import_mode, kwargs['remove_duplicates'], kwargs['ignore_tags']
+    )
+
+    if log_level < 30:
+        summary.print()
+
+    if summary.errors:
+        sys.exit(1)
+
+@click.command('sort')
+@add_options(_logger_options)
+@add_options(_input_options)
+@add_options(_dry_run_options)
+@add_options(_filter_options)
+@add_options(_sort_options)
+@click.option('--clean', '-C', default=False, is_flag=True, help='Clean empty folders')
 @click.option(
     '--reset-cache',
     '-r',
@@ -140,56 +227,25 @@ def get_collection_config(root):
     is_flag=True,
     help='Regenerate the hash.json and location.json database ',
 )
-@click.option(
+@click.argument('subdirs', required=False, nargs=-1, type=click.Path())
-    '--use-date-filename',
+@click.argument('dest', required=True, nargs=1, type=click.Path())
-    '-f',
+def _sort(**kwargs):
-    default=False,
-    is_flag=True,
-    help="Use filename date for media original date.",
-)
-@click.option(
-    '--use-file-dates',
-    '-F',
-    default=False,
-    is_flag=True,
-    help="Use file date created or modified for media original date.",
-)
-@click.argument('paths', required=True, nargs=-1, type=click.Path())
-def sort(**kwargs):
     """Sort files or directories by reading their EXIF and organizing them
     according to ordigi.conf preferences.
     """
 
-    root = kwargs['destination']
     log_level = log.level(kwargs['verbose'], kwargs['debug'])
-
-    paths = kwargs['paths']
-
-    if kwargs['copy']:
-        mode = 'copy'
-    else:
-        mode = 'move'
-
     logger = log.get_logger(level=log_level)
 
+    subdirs = kwargs['subdirs']
+    root = kwargs['dest']
+    paths, root = _get_paths(subdirs, root)
+    paths = os.path.join(root, subdirs)
+
     cache = True
     if kwargs['reset_cache']:
         cache = False
 
-    if len(paths) > 1:
-        if not root:
-            # Use last path argument as destination
-            root = paths[-1]
-            paths = paths[0:-1]
-    elif paths:
-        # Source and destination are the same
-        root = paths[0]
-    else:
-        logger.error(f'`ordigi sort` need at least one path argument')
-        sys.exit(1)
-
-    paths = set(paths)
-
     config = get_collection_config(root)
     opt = config.get_options()
 
@@ -212,15 +268,14 @@ def sort(**kwargs):
         kwargs['interactive'],
         logger,
         opt['max_deep'],
-        mode,
         kwargs['use_date_filename'],
         kwargs['use_file_dates'],
     )
 
     loc = GeoLocation(opt['geocoder'], logger, opt['prefer_english_names'], opt['timeout'])
 
-    summary, result = collection.sort_files(
+    summary = collection.sort_files(
-        paths, loc, kwargs['remove_duplicates'], kwargs['ignore_tags']
+        paths, path_format, loc, kwargs['remove_duplicates'], kwargs['ignore_tags']
     )
 
     if kwargs['clean']:
@@ -229,14 +284,14 @@ def sort(**kwargs):
     if log_level < 30:
         summary.print()
 
-    if not result:
+    if summary.errors:
         sys.exit(1)
 
 
 @click.command('clean')
 @add_options(_logger_options)
 @add_options(_dry_run_options)
-@add_options(_filter_option)
+@add_options(_filter_options)
 @click.option(
     '--dedup-regex',
     '-d',
@@ -260,32 +315,25 @@ def sort(**kwargs):
     is_flag=True,
     help='True to remove files that are exactly the same in name and a file hash',
 )
-@click.option(
+@click.argument('subdirs', required=False, nargs=-1, type=click.Path())
-    '--root',
+@click.argument('dest', required=True, nargs=1, type=click.Path())
-    '-r',
+def _clean(**kwargs):
-    type=click.Path(file_okay=False),
-    default=None,
-    help='Root dir of media collection. If not set, use path',
-)
-@click.argument('path', required=True, nargs=1, type=click.Path())
-def clean(**kwargs):
     """Remove empty folders
     Usage: clean [--verbose|--debug] directory [removeRoot]"""
 
-    result = True
     dry_run = kwargs['dry_run']
     folders = kwargs['folders']
     log_level = log.level(kwargs['verbose'], kwargs['debug'])
-    root = kwargs['root']
-
-    path = kwargs['path']
-
     logger = log.get_logger(level=log_level)
+
+    subdirs = kwargs['subdirs']
+    root = kwargs['dest']
+    paths, root = _get_paths(subdirs, root)
+    paths = os.path.join(root, subdirs)
+
     clean_all = False
     if not folders:
         clean_all = True
-    if not root:
-        root = path
 
     config = get_collection_config(root)
     opt = config.get_options()
@@ -303,30 +351,35 @@ def clean(**kwargs):
         max_deep=opt['max_deep'],
     )
 
+    for path in paths:
         if kwargs['path_string']:
             dedup_regex = list(kwargs['dedup_regex'])
-        summary, result = collection.dedup_regex(
+            collection.dedup_regex(
                 path, dedup_regex, kwargs['remove_duplicates']
             )
 
         if clean_all or folders:
-        summary = collection.remove_empty_folders(path)
+            collection.remove_empty_folders(path)
 
         if kwargs['delete_excluded']:
-        summary = collection.remove_excluded_files()
+            collection.remove_excluded_files()
+
+    summary = collection.summary
 
     if log_level < 30:
         summary.print()
 
-    if not result:
+    if summary.errors:
         sys.exit(1)
 
 
 @click.command('init')
 @add_options(_logger_options)
 @click.argument('path', required=True, nargs=1, type=click.Path())
-def init(**kwargs):
+def _init(**kwargs):
-    """Regenerate the hash.json database which contains all of the sha256 signatures of media files."""
+    """
+    Init media collection database.
+    """
     root = kwargs['path']
     config = get_collection_config(root)
     opt = config.get_options()
@@ -344,8 +397,10 @@ def init(**kwargs):
 @click.command('update')
 @add_options(_logger_options)
 @click.argument('path', required=True, nargs=1, type=click.Path())
-def update(**kwargs):
+def _update(**kwargs):
-    """Regenerate the hash.json database which contains all of the sha256 signatures of media files."""
+    """
+    Update media collection database.
+    """
     root = kwargs['path']
     config = get_collection_config(root)
     opt = config.get_options()
@@ -363,8 +418,10 @@ def update(**kwargs):
 @click.command('check')
 @add_options(_logger_options)
 @click.argument('path', required=True, nargs=1, type=click.Path())
-def check(**kwargs):
+def _check(**kwargs):
-    """check db and verify hashes"""
+    """
+    Check media collection.
+    """
     log_level = log.level(kwargs['verbose'], kwargs['debug'])
     logger = log.get_logger(level=log_level)
     root = kwargs['path']
@@ -373,10 +430,10 @@ def check(**kwargs):
     collection = Collection(root, exclude=opt['exclude'], logger=logger)
     result = collection.check_db()
     if result:
-        summary, result = collection.check_files()
+        summary = collection.check_files()
         if log_level < 30:
             summary.print()
-        if not result:
+        if summary.errors:
             sys.exit(1)
     else:
         logger.error('Db data is not accurate run `ordigi update`')
@@ -386,7 +443,7 @@ def check(**kwargs):
 @click.command('compare')
 @add_options(_logger_options)
 @add_options(_dry_run_options)
-@add_options(_filter_option)
+@add_options(_filter_options)
 @click.option('--find-duplicates', '-f', default=False, is_flag=True)
 @click.option(
     '--output-dir',
@@ -403,13 +460,6 @@ def check(**kwargs):
     is_flag=True,
     help='Revert compare',
 )
-@click.option(
-    '--root',
-    '-r',
-    type=click.Path(file_okay=False),
-    default=None,
-    help='Root dir of media collection. If not set, use path',
-)
 @click.option(
     '--similar-to',
     '-s',
@@ -422,15 +472,23 @@ def check(**kwargs):
     default=80,
     help='Similarity level for images',
 )
-@click.argument('path', nargs=1, required=True)
+@click.argument('subdirs', required=False, nargs=-1, type=click.Path())
-def compare(**kwargs):
+@click.argument('dest', required=True, nargs=1, type=click.Path())
-    '''Compare files in directories'''
+def _compare(**kwargs):
+    """
+    Sort similar images in directories
+    """
 
     dry_run = kwargs['dry_run']
     log_level = log.level(kwargs['verbose'], kwargs['debug'])
-    root = kwargs['root']
+
+    subdirs = kwargs['subdirs']
+    root = kwargs['dest']
+    paths, root = _get_paths(subdirs, root)
+    paths = os.path.join(root, subdirs)
 
     path = kwargs['path']
+    root = kwargs['root']
 
     logger = log.get_logger(level=log_level)
     if not root:
@@ -451,15 +509,18 @@ def compare(**kwargs):
         logger=logger,
     )
 
+    for path in paths:
         if kwargs['revert_compare']:
-        summary, result = collection.revert_compare(path)
+            collection.revert_compare(path)
         else:
-        summary, result = collection.sort_similar_images(path, kwargs['similarity'])
+            collection.sort_similar_images(path, kwargs['similarity'])
+
+    summary = collection.summary
 
     if log_level < 30:
         summary.print()
 
-    if not result:
+    if summary.errors:
         sys.exit(1)
 
 
@@ -468,12 +529,13 @@ def main(**kwargs):
     pass
 
 
-main.add_command(clean)
+main.add_command(_clean)
-main.add_command(check)
+main.add_command(_check)
-main.add_command(compare)
+main.add_command(_compare)
-main.add_command(init)
+main.add_command(_init)
-main.add_command(sort)
+main.add_command(_import)
-main.add_command(update)
+main.add_command(_sort)
+main.add_command(_update)
 
 
 if __name__ == '__main__':

ordigi/collection.py

@@ -1,21 +1,19 @@
 """
-General file system methods.
+Collection methods.
 """
-from builtins import object
-
 from copy import copy
 from datetime import datetime, timedelta
 import filecmp
 from fnmatch import fnmatch
-import inquirer
-import logging
 import os
-from pathlib import Path, PurePath
 import re
-import sys
 import shutil
+import sys
+import logging
+from pathlib import Path, PurePath
+
+import inquirer
 
-from ordigi import media
 from ordigi.database import Sqlite
 from ordigi.media import Media
 from ordigi.images import Image, Images
@@ -170,6 +168,17 @@ class FPath:
 
         return part
 
+    def _set_case(self, regex, part, this_part):
+        # Capitalization
+        u_regex = '%u' + regex
+        l_regex = '%l' + regex
+        if re.search(u_regex, this_part):
+            this_part = re.sub(u_regex, part.upper(), this_part)
+        elif re.search(l_regex, this_part):
+            this_part = re.sub(l_regex, part.lower(), this_part)
+        else:
+            this_part = re.sub(regex, part, this_part)
+
     def get_path_part(self, this_part, metadata):
         """Build path part
         :returns: part (string)"""
@@ -185,15 +194,7 @@ class FPath:
                     regex = '[-_ .]?(%[ul])?' + regex
                     this_part = re.sub(regex, part, this_part)
                 else:
-                    # Capitalization
+                    self._set_case(regex, part, this_part)
-                    u_regex = '%u' + regex
-                    l_regex = '%l' + regex
-                    if re.search(u_regex, this_part):
-                        this_part = re.sub(u_regex, part.upper(), this_part)
-                    elif re.search(l_regex, this_part):
-                        this_part = re.sub(l_regex, part.lower(), this_part)
-                    else:
-                        this_part = re.sub(regex, part, this_part)
 
         # Delete separator char at the begining of the string if any:
         if this_part:
@@ -263,7 +264,6 @@ class Collection:
         interactive=False,
         logger=logging.getLogger(),
         max_deep=None,
-        mode='move',
         use_date_filename=False,
         use_file_dates=False,
     ):
@@ -285,7 +285,7 @@ class Collection:
 
         if '%media' in filter_by_ext:
             filter_by_ext.remove('%media')
-            self.filter_by_ext = filter_by_ext.union(media.extensions)
+            self.filter_by_ext = filter_by_ext.union(Media.extensions)
         else:
             self.filter_by_ext = filter_by_ext
 
@@ -293,10 +293,9 @@ class Collection:
         self.interactive = interactive
         self.logger = logger.getChild(self.__class__.__name__)
         self.max_deep = max_deep
-        self.mode = mode
         # List to store media metadata
         self.medias = []
-        self.summary = Summary()
+        self.summary = Summary(self.root)
         self.use_date_filename = use_date_filename
         self.use_file_dates = use_file_dates
 
@@ -355,37 +354,6 @@ class Collection:
 
         return False
 
-    def _record_file(self, src_path, dest_path, media):
-        """Check file and record the file to db"""
-
-        # Check if file remain the same
-        record = False
-        checksum = media.metadata['checksum']
-        if self._checkcomp(dest_path, checksum):
-            # change media file_path to dest_path
-            media.file_path = dest_path
-            if not self.dry_run:
-                updated = self._update_exif_data(dest_path, media)
-                if updated:
-                    checksum = utils.checksum(dest_path)
-                    media.metadata['checksum'] = checksum
-
-                media.metadata['file_path'] = os.path.relpath(dest_path, self.root)
-                self._add_db_data(media.metadata)
-                if self.mode == 'move':
-                    # Delete file path entry in db when file is moved inside collection
-                    if self.root in src_path.parents:
-                        self.db.delete_filepath(str(src_path.relative_to(self.root)))
-
-            self.summary.append((src_path, self.mode))
-            record = True
-
-        else:
-            self.logger.error(f'Files {src_path} and {dest_path} are not identical')
-            self.summary.append((src_path, False))
-
-        return record
-
     def remove(self, file_path):
         if not self.dry_run:
             os.remove(file_path)
@@ -404,71 +372,71 @@ class Collection:
                     if fnmatch(file_path, exclude):
                         if not self.dry_run:
                             self.remove(file_path)
-                        self.summary.append((file_path, 'delete'))
+                        self.summary.append((file_path, 'remove_excluded'))
                         break
 
         return self.summary
 
-    def sort_file(self, src_path, dest_path, remove_duplicates=False):
+    def _check_file(self, src_path, dest_path, media):
-        '''
+        checksum = media.metadata['checksum']
-        Copy or move file to dest_path.
+        if not self._checkcomp(dest_path, checksum):
-        Return True if success, None is no filesystem action, False if
+            self.logger.error(f'Files {src_path} and {dest_path} are not identical')
-        conflicts.
+            self.summary.append((src_path, False))
-        :params: str, str, bool
+            return False
-        :returns: bool or None
-        '''
 
-        mode = self.mode
+        # change media file_path to dest_path
-        dry_run = self.dry_run
+        media.file_path = dest_path
+        if not self.dry_run:
+            updated = self._update_exif_data(dest_path, media)
+            if updated:
+                checksum = utils.checksum(dest_path)
+                media.metadata['checksum'] = checksum
+
+        media.metadata['file_path'] = os.path.relpath(dest_path, self.root)
 
-        # check for collisions
-        if src_path == dest_path:
-            self.logger.info(f'File {dest_path} already sorted')
-            return None
-        elif dest_path.is_dir():
-            self.logger.info(f'File {dest_path} is a existing directory')
-            return False
-        elif dest_path.is_file():
-            self.logger.info(f'File {dest_path} already exist')
-            if remove_duplicates:
-                if filecmp.cmp(src_path, dest_path):
-                    self.logger.info(
-                        f'File in source and destination are identical. Duplicate will be ignored.'
-                    )
-                    if mode == 'move':
-                        if not dry_run:
-                            self.remove(src_path)
-                        self.summary.append((src_path, 'delete'))
-                    return None
-                else:  # name is same, but file is different
-                    self.logger.warning(
-                        f'File in source and destination are different.'
-                    )
-                    return False
-            else:
-                return False
-        else:
-            if mode == 'move':
-                if not dry_run:
-                    # Move the processed file into the destination directory
-                    shutil.move(src_path, dest_path)
-                self.logger.info(f'move: {src_path} -> {dest_path}')
-            elif mode == 'copy':
-                if not dry_run:
-                    shutil.copy2(src_path, dest_path)
-                self.logger.info(f'copy: {src_path} -> {dest_path}')
         return True
 
-    def _solve_conflicts(self, conflict_file_list, remove_duplicates):
+    def _copy(self, src_path, dest_path):
+        if not self.dry_run:
+            shutil.copy2(src_path, dest_path)
+        self.logger.info(f'copy: {src_path} -> {dest_path}')
+
+    def _move(self, src_path, dest_path):
+        if not self.dry_run:
+            # Move the file into the destination directory
+            shutil.move(src_path, dest_path)
+
+        self.logger.info(f'move: {src_path} -> {dest_path}')
+
+    def _remove(self, path):
+        if not self.dry_run:
+            self.remove(path)
+
+        self.logger.info(f'remove: {path}')
+
+    def _record_file(self, src_path, dest_path, media, import_mode=False):
+        """Check file and record the file to db"""
+        # Check if file remain the same
+        if not self._check_file(src_path, dest_path, media):
+            self.summary.append((src_path, False))
+            return False
+
+        if not self.dry_run:
+            self._add_db_data(media.metadata)
+            if import_mode != 'copy' and self.root in src_path.parents:
+                self.db.delete_filepath(str(src_path.relative_to(self.root)))
+
+        return True
+
+    def _solve_conflicts(self, conflicts, remove_duplicates):
         result = False
         unresolved_conflicts = []
-        while conflict_file_list != []:
+        while conflicts != []:
-            src_path, dest_path, media = conflict_file_list.pop()
+            src_path, dest_path, media = conflicts.pop()
-            # Try to sort the file
+            # Check for conflict status again in case is has changed
-            result = self.sort_file(src_path, dest_path, remove_duplicates)
+            conflict = self.check_conflicts(src_path, dest_path, remove_duplicates)
-            # remove to conflict file list if file as be successfully copied or ignored
             n = 1
-            while result is False and n < 100:
+            while conflict == 1 and n < 100:
                 # Add appendix to the name
                 suffix = dest_path.suffix
                 if n > 1:
@@ -476,25 +444,15 @@ class Collection:
                 else:
                     stem = dest_path.stem
                 dest_path = dest_path.parent / (stem + '_' + str(n) + suffix)
-                result = self.sort_file(src_path, dest_path, remove_duplicates)
+                conflict = self.check_conflicts(src_path, dest_path, remove_duplicates)
                 n = n + 1
 
-            record = False
+            if conflict == 1:
-            if result is True:
-                record = self._record_file(src_path, dest_path, media)
-            elif result is None:
-                record = True
-            else:
                 # n > 100:
                 unresolved_conflicts.append((src_path, dest_path, media))
-                self.logger.error(f'{self.mode}: too many append for {dest_path}...')
+                self.logger.error(f"Too many appends for {dest_path}")
-                self.summary.append((src_path, False))
 
-            if record:
+            yield (src_path, dest_path, media), conflict
-                # result is true or None
-                self.dest_list.append(dest_path)
-
-        return record
 
     def _split_part(self, dedup_regex, path_part, items):
         """Split part from regex
@@ -641,6 +599,40 @@ class Collection:
             file_path, (int(datetime.now().timestamp()), int(date_media.timestamp()))
         )
 
+    def check_conflicts(self, src_path, dest_path, remove_duplicates=False):
+        '''
+        Check if file can be copied or moved file to dest_path.
+        Return True if success, None is no filesystem action, False if
+        conflicts.
+        :params: str, str, bool
+        :returns: bool or None
+        '''
+
+        # check for collisions
+        if src_path == dest_path:
+            self.logger.info(f"File {dest_path} already sorted")
+            return 2
+        if dest_path.is_dir():
+            self.logger.info(f"File {dest_path} is a existing directory")
+            return 1
+        elif dest_path.is_file():
+            self.logger.info(f"File {dest_path} already exist")
+            if remove_duplicates:
+                if filecmp.cmp(src_path, dest_path):
+                    self.logger.info(
+                        f"File in source and destination are identical. Duplicate will be ignored."
+                    )
+                    return 3
+                else:  # name is same, but file is different
+                    self.logger.info(
+                        f"File {src_path} and {dest_path} are different."
+                    )
+                    return 1
+            else:
+                return 1
+        else:
+            return 0
+
     def dedup_regex(self, path, dedup_regex, remove_duplicates=False):
         # cycle throught files
         result = False
@@ -661,7 +653,7 @@ class Collection:
             default = re.compile(r'([^-_ .]+[-_ .])')
             dedup_regex = [date_num3, date_num2, default]
 
-        conflict_file_list = []
+        conflicts = []
         self.src_list = [
             x
             for x in self._get_files_in_path(
@@ -689,28 +681,42 @@ class Collection:
             dest_path = self.root.joinpath(*dedup_path)
             self._create_directory(dest_path.parent.name, media)
 
-            result = self.sort_file(src_path, dest_path, remove_duplicates)
+            conflicts = self.check_conflicts(src_path, dest_path, remove_duplicates)
 
-            record = False
+            result = False
-            if result is True:
+            if not conflict:
                 record = self._record_file(src_path, dest_path, media)
-            elif result is None:
+            elif conflict == 1:
-                record = True
+                # There is conflict and file are different
-            else:
+                conflicts.append((src_path, dest_path, media))
-                # There is conflict files
+            elif conflict in (2, 3):
-                conflict_file_list.append(src_path, dest_path, copy(media))
+                result = True
 
-            if record:
+            if result:
                 # result is true or None
                 self.dest_list.append(dest_path)
 
-        if conflict_file_list != []:
+        if conflicts != []:
-            record = self._solve_conflicts(conflict_file_list, remove_duplicates)
+            files_data, conflict = self._solve_conflicts(conflicts, remove_duplicates)
+            src_path, dest_path, media = file_data
+
+            result = False
+            if not conflict:
+                self._record_file(src_path, dest_path, media)
+            elif conflict == 1:
+                # There is unresolved conflict
+                self.summary.append((src_path, False))
+            elif conflict in (2, 3):
+                result = True
+
+            if result:
+                # result is true or None
+                self.dest_list.append(dest_path)
 
         if not self._check_processed():
-            return False
+            self.summary.append((None, False))
 
-        return self.summary, record
+        return self.summary
 
     def _modify_selection(self):
         """
@@ -780,7 +786,7 @@ class Collection:
     def init(self, loc, ignore_tags=set()):
         for media, file_path in self.get_medias(loc):
             self._add_db_data(media.metadata)
-            self.summary.append((file_path, 'record'))
+            self.summary.append((file_path, 'update'))
 
         return self.summary
 
@@ -792,32 +798,33 @@ class Collection:
             sys.exit(1)
 
     def check_files(self):
-        result = True
         for file_path in self._get_all_files():
             checksum = utils.checksum(file_path)
             relpath = file_path.relative_to(self.root)
             if checksum == self.db.get_checksum(relpath):
-                self.summary.append((file_path, 'record'))
+                self.summary.append((file_path, 'check'))
             else:
                 self.logger.error('{file_path} is corrupted')
                 self.summary.append((file_path, False))
-                result = False
 
-        return self.summary, result
+        return self.summary
 
     def update(self, loc, ignore_tags=set()):
         file_paths = [x for x in self._get_all_files()]
         db_rows = [row for row in self.db.get_rows('metadata')]
         invalid_db_rows = set()
+        db_paths = set()
         for db_row in db_rows:
             abspath = self.root / db_row['FilePath']
             if abspath not in file_paths:
                 invalid_db_rows.add(db_row)
 
+            db_paths.add(db_row['FilePath'])
+
         for file_path in file_paths:
             relpath = os.path.relpath(file_path, self.root)
             # If file not in database
-            if relpath not in db_rows:
+            if relpath not in db_paths:
                 media = Media(
                     file_path,
                     self.root,
@@ -842,7 +849,7 @@ class Collection:
                         break
                 # set row attribute to the file
                 self._add_db_data(media.metadata)
-                self.summary.append((file_path, 'record'))
+                self.summary.append((file_path, 'update'))
 
         # Finally delete invalid rows
         for row in invalid_db_rows:
@@ -879,27 +886,54 @@ class Collection:
 
         return src_list
 
-    def sort_files(self, src_dirs, path_format, loc, remove_duplicates=False, ignore_tags=set()):
+    def sort_file(self, src_path, dest_path, media, import_mode=False):
+        if import_mode == 'copy':
+            self._copy(src_path, dest_path)
+        else:
+            self._move(src_path, dest_path)
+        if import_mode:
+            update = False
+
+        result = self._record_file(
+            src_path, dest_path, media, import_mode=import_mode
+            )
+
+        if result:
+            self.dest_list.append(dest_path)
+            if import_mode:
+                self.summary.append((src_path, 'import'))
+            else:
+                self.summary.append((src_path, 'sort'))
+        else:
+            self.summary.append((src_path, False))
+
+        return self.summary
+
+    def sort_files(self, src_dirs, path_format, loc, import_mode=False, remove_duplicates=False, ignore_tags=set()):
         """
         Sort files into appropriate folder
         """
         # Check db
         self._init_check_db(loc, ignore_tags)
 
-        result = False
         files_data = []
         src_dirs_in_collection = set()
         for src_dir in src_dirs:
             self.dest_list = []
             src_dir = self._check_path(src_dir)
-            conflict_file_list = []
+            conflicts = []
             self.src_list = self._get_path_list(src_dir)
 
             # Get medias and src_dirs
             for src_path in self.src_list:
-                # List all src dirs in collection
                 if self.root in src_path.parents:
                     src_dirs_in_collection.add(src_path.parent)
+                else:
+                    if not import_mode:
+                        self.logger.error(f"""{src_path} not in {self.root}
+                                collection, use `ordigi import`""")
+                        sys.exit(1)
+
                 # Get file metadata
                 media = Media(
                     src_path,
@@ -928,29 +962,50 @@ class Collection:
                 src_path = media.file_path
                 dest_path = self.root / relpath
 
-                result = self.sort_file(src_path, dest_path, remove_duplicates)
+                conflict = self.check_conflicts(src_path, dest_path, remove_duplicates)
 
-                record = False
+                if not conflict:
-                if result is True:
+                    self.sort_file(
-                    record = self._record_file(src_path, dest_path, media)
+                        src_path, dest_path, media, import_mode=import_mode
-                elif result is None:
+                        )
-                    record = True
+                elif conflict == 1:
-                else:
+                    # There is conflict and file are different
-                    # There is conflict files
+                    conflicts.append((src_path, dest_path, media))
-                    conflict_file_list.append((src_path, dest_path, media))
+                elif conflict == 3:
-                if record:
+                    # Same file checksum
-                    # result is true or None
+                    if import_mode == 'move':
+                        self._remove(src_path)
+                    self.dest_list.append(dest_path)
+                elif conflict == 2:
+                    # File already sorted
                     self.dest_list.append(dest_path)
 
-            if conflict_file_list != []:
+            if conflicts != []:
-                record = self._solve_conflicts(conflict_file_list, remove_duplicates)
+                files_data, conflict = self._solve_conflicts(conflicts, remove_duplicates)
+                src_path, dest_path, media = file_data
+
+                if not conflict:
+                    self.sort_file(
+                        src_path, dest_path, media, import_mode=import_mode
+                        )
+                elif conflict == 1:
+                    # There is unresolved conflict
+                    self.summary.append((src_path, False))
+                elif conflict == 3:
+                    # Same file checksum
+                    if import_mode == 'move':
+                        self._remove(src_path)
+                    self.dest_list.append(dest_path)
+                elif conflict == 2:
+                    # File already sorted
+                    self.dest_list.append(dest_path)
 
             self.remove_empty_subdirs(src_dirs_in_collection)
 
             if not self._check_processed():
-                record = False
+                self.summary.append((None, False))
 
-        return self.summary, record
+        return self.summary
 
     def remove_empty_folders(self, directory, remove_root=True):
         'Function to remove empty folders'
@@ -972,16 +1027,10 @@ class Collection:
             self.logger.info(f"Removing empty folder: {directory}")
             if not self.dry_run:
                 os.rmdir(directory)
-            self.summary.append((directory, 'delete'))
+            self.summary.append((directory, 'remove_empty_folders'))
 
         return self.summary
 
-    def move_file(self, img_path, dest_path):
-        if not self.dry_run:
-            shutil.move(img_path, dest_path)
-
-        self.logger.info(f'move: {img_path} -> {dest_path}')
-
     def _get_images(self, path):
         """
         :returns: iter
@@ -1007,9 +1056,8 @@ class Collection:
             self.logger.error('Db data is not accurate run `ordigi init`')
             sys.exit(1)
 
-        result = True
         path = self._check_path(path)
-        images = set([x for x in self._get_images(path)])
+        images = set(x for x in self._get_images(path))
         i = Images(images, logger=self.logger)
         nb_row_ini = self.db.len('metadata')
         for image in images:
@@ -1032,34 +1080,32 @@ class Collection:
                 dest_directory.mkdir(exist_ok=True)
 
                 # Move the simlars file into the destination directory
-                self.move_file(img_path, dest_path)
+                self._move(img_path, dest_path)
                 moved_imgs.add(img_path)
                 if self._record_file(img_path, dest_path, media):
-                    self.summary.append((img_path, self.mode))
+                    self.summary.append((img_path, 'sort'))
                 else:
                     self.summary.append((img_path, False))
-                    result = False
 
             if similar:
                 img_path = image.img_path
                 dest_path = dest_directory / img_path.name
-                self.move_file(img_path, dest_path)
+                self._move(img_path, dest_path)
                 moved_imgs.add(img_path)
                 if self._record_file(img_path, dest_path, media_ref):
-                    self.summary.append((img_path, self.mode))
+                    self.summary.append((img_path, 'sort'))
                 else:
                     self.summary.append((img_path, False))
-                    result = False
 
         nb_row_end = self.db.len('metadata')
         if nb_row_ini and nb_row_ini != nb_row_end:
             self.logger.error('Nb of row have changed unexpectedly')
-            result = False
 
         if result:
             result = self.check_db()
+            self.summary.append((None, False))
 
-        return self.summary, result
+        return self.summary
 
     def revert_compare(self, path):
 
@@ -1067,7 +1113,6 @@ class Collection:
             self.logger.error('Db data is not accurate run `ordigi init`')
             sys.exit(1)
 
-        result = True
         path = self._check_path(path)
         dirnames = set()
         moved_files = set()
@@ -1084,13 +1129,12 @@ class Collection:
                 media = Media(src_path, path, self.logger)
                 metadata = media.get_metadata(self.root, db=self.db, cache=self.cache)
                 dest_path = Path(src_path.parent.parent, src_path.name)
-                self.move_file(src_path, dest_path)
+                self._move(src_path, dest_path)
                 moved_files.add(src_path)
                 if self._record_file(src_path, dest_path, media):
-                    self.summary.append((src_path, self.mode))
+                    self.summary.append((src_path, 'sort'))
                 else:
                     self.summary.append((src_path, False))
-                    result = False
 
         for dirname in dirnames:
             # remove 'similar_to*' directories
@@ -1102,13 +1146,12 @@ class Collection:
         nb_row_end = self.db.len('metadata')
         if nb_row_ini and nb_row_ini != nb_row_end:
             self.logger.error('Nb of row have changed unexpectedly')
-            result = False
 
         if result:
             result = self.check_db()
+            self.summary.append((None, False))
 
-        return self.summary, result
+        return self.summary
-
 
     def fill_data(self, path, key, loc=None, edit=False):
         """Fill metadata and exif data for given key"""
@@ -1172,7 +1215,7 @@ class Collection:
                 # Update exif data
                 media.set_key_values(key, value)
 
-                self.summary.append((file_path, 'record'))
+                self.summary.append((file_path, 'update'))
 
         return self.summary
 

ordigi/summary.py

@@ -2,43 +2,56 @@ from tabulate import tabulate
 
 
 class Summary:
-
+    def __init__(self, path):
-    def __init__(self):
+        self.actions = (
-        self.modes = ('record', 'copy', 'move', 'delete')
+            'check',
+            'import',
+            'remove_empty_folders',
+            'remove_excluded',
+            'sort',
+            'update',
+        )
+        self.path = path
         self.result = {}
-        for mode in self.modes:
+        for action in self.actions:
-            self.result[mode] = 0
+            self.result[action] = 0
 
         self.errors = 0
         self.errors_items = []
 
     def append(self, row):
-        file_path, mode = row
+        file_path, action = row
 
-        if mode:
+        if action:
-            for m in self.modes:
+            for m in self.actions:
-                if mode == m:
+                if action == m:
-                    self.result[mode] += 1
+                    self.result[action] += 1
         else:
             self.errors += 1
+            if file_path:
                 self.errors_items.append(file_path)
 
     def print(self):
 
         print()
-        for mode in self.result:
+        for action in self.result:
-            nb = self.result[mode]
+            nb = self.result[action]
-            if self.result[mode] != 0:
+            if self.result[action] != 0:
-                if mode == 'record':
+                if action == 'check':
-                    print(f"SUMMARY: {nb} files recorded.")
+                    print(f"SUMMARY: {nb} files checked in {self.path}.")
-                elif mode == 'copy':
+                elif action == 'import':
-                    print(f"SUMMARY: {nb} files copied.")
+                    print(f"SUMMARY: {nb} files imported into {self.path}.")
-                elif mode == 'move':
+                elif action == 'sort':
-                    print(f"SUMMARY: {nb} files moved.")
+                    print(f"SUMMARY: {nb} files sorted inside {self.path}.")
-                else:
+                elif action == 'remove_excluded':
-                    print(f"SUMMARY: {nb} files deleted.")
+                    print(f"SUMMARY: {nb} files deleted in {self.path}.")
+                elif action == 'remove_empty_folders':
+                    print(f"SUMMARY: {nb} empty folders removed in {self.path}.")
+                elif action == 'update':
+                    print(f"SUMMARY: {nb} files updated in {self.path} database.")
+
         if sum(self.result.values()) == 0 and not self.errors:
-            print(f"OK !!")
+            print(f"SUMMARY: no file imported, sorted or deleted from {self.path}.")
 
         if self.errors > 0:
             print()
@@ -49,4 +62,3 @@ class Summary:
 
             print(tabulate(errors_result, headers=errors_headers))
             print()
-

tests/test_collection.py

@@ -1,24 +1,21 @@
 # TODO to be removed later
 from datetime import datetime
-import inquirer
-import os
-import pytest
 import shutil
 import sqlite3
 from pathlib import Path
 import re
-from sys import platform
+import pytest
-from time import sleep
+import inquirer
 
-from .conftest import randomize_files, randomize_db
 from ordigi import constants
 from ordigi.collection import Collection, FPath
-from ordigi.database import Sqlite
 from ordigi.exiftool import ExifToolCaching, exiftool_is_running, terminate_exiftool
 from ordigi.geolocation import GeoLocation
 from ordigi import log
 from ordigi.media import Media
 from ordigi import utils
+from .conftest import randomize_files, randomize_db
+from ordigi.summary import Summary
 
 
 class TestFPath:
@@ -68,7 +65,7 @@ class TestFPath:
 
             exif_data = ExifToolCaching(str(file_path)).asdict()
             loc = GeoLocation()
-            metadata = media.get_metadata(loc)
+            metadata = media.get_metadata(self.src_path, loc)
             for item, regex in items.items():
                 for mask in masks:
                     matched = re.search(regex, mask)
@@ -131,21 +128,30 @@ class TestCollection:
         terminate_exiftool()
         assert not exiftool_is_running()
 
+    def assert_import(self, summary, nb):
+        # Summary is created and there is no errors
+        assert summary.errors == 0
+        assert summary.result['import'] == nb
+
+    def assert_sort(self, summary, nb):
+        # Summary is created and there is no errors
+        assert summary.errors == 0
+        assert summary.result['sort'] == nb
+
     def test_sort_files(self, tmp_path):
         collection = Collection(tmp_path, album_from_folder=True,
-                logger=self.logger, mode='copy')
+                logger=self.logger)
         loc = GeoLocation()
-        summary, result = collection.sort_files([self.src_path],
+        summary = collection.sort_files([self.src_path],
-                self.path_format, loc)
+                self.path_format, loc, import_mode='copy')
 
-        # Summary is created and there is no errors
+        self.assert_import(summary, 30)
-        assert summary, summary
-        assert result, result
 
-        summary, result = collection.check_files()
+        summary = collection.check_files()
-        assert summary, summary
+        assert summary.result['check'] == 30
-        assert result, result
+        assert not summary.errors
 
+        # check if album value are set
         for file_path in tmp_path.glob('**/*'):
             if '.db' not in str(file_path):
                 media = Media(file_path, tmp_path, album_from_folder=True)
@@ -153,55 +159,66 @@ class TestCollection:
                 for value in media._get_key_values('album'):
                     assert value != '' or None
 
+        collection = Collection(tmp_path, album_from_folder=True)
+        # Try to change path format and sort files again
+        path = '{city}/{%Y}-{name}.%l{ext}'
+        summary = collection.sort_files([tmp_path],
+                self.path_format, loc)
+
+        self.assert_sort(summary, 24)
+
+        shutil.copytree(tmp_path / 'test_exif', tmp_path / 'test_exif_copy')
+        collection.summary = Summary(tmp_path)
+        assert sum(collection.summary.result.values()) == 0
+        summary = collection.update(loc)
+        assert summary.result['update'] == 2
+        assert not summary.errors
+        collection.summary = Summary(tmp_path)
+        summary = collection.update(loc)
+        assert not summary.result['update']
+        assert not summary.errors
+
         # test with populated dest dir
         randomize_files(tmp_path)
-        summary, result = collection.check_files()
+        summary = collection.check_files()
-        assert summary, summary
+        assert summary.errors
-        assert not result, result
 
-        collection = Collection(tmp_path, logger=self.logger)
+        # test summary update
+        collection.summary = Summary(tmp_path)
         summary = collection.update(loc)
-        assert summary, summary
+        assert summary.result['update']
-
+        assert not summary.errors
-        collection = Collection(tmp_path, mode='copy', album_from_folder=True)
-        loc = GeoLocation()
-        summary, result = collection.sort_files([self.src_path], self.path_format, loc)
-
-        assert summary, summary
-        assert result, result
-
-        # TODO check if path follow path_format
 
     def test_sort_files_invalid_db(self, tmp_path):
-        collection = Collection(tmp_path, mode='copy')
+        collection = Collection(tmp_path)
         loc = GeoLocation()
         randomize_db(tmp_path)
         with pytest.raises(sqlite3.DatabaseError) as e:
-            summary, result = collection.sort_files([self.src_path],
+            summary = collection.sort_files([self.src_path],
-                    self.path_format, loc)
+                    self.path_format, loc, import_mode='copy')
 
     def test_sort_file(self, tmp_path):
-
+        for import_mode in 'copy', 'move', False:
-        for mode in 'copy', 'move':
+            collection = Collection(tmp_path)
-            collection = Collection(tmp_path, mode=mode)
             # copy mode
             src_path = Path(self.src_path, 'test_exif', 'photo.png')
-            name = 'photo_' + mode + '.png'
+            media = Media(src_path, self.src_path)
+            metadata = media.get_metadata(tmp_path)
+            name = 'photo_' + str(import_mode) + '.png'
             dest_path = Path(tmp_path, name)
             src_checksum = utils.checksum(src_path)
-            result_copy = collection.sort_file(src_path, dest_path)
+            summary = collection.sort_file(src_path, dest_path, media,
-            assert result_copy
+                    import_mode=import_mode)
+            assert not summary.errors
             # Ensure files remain the same
             assert collection._checkcomp(dest_path, src_checksum)
 
-            if mode == 'copy':
+            if import_mode == 'copy':
                 assert src_path.exists()
             else:
                 assert not src_path.exists()
                 shutil.copyfile(dest_path, src_path)
 
-        # TODO check for conflicts
-
     def test__get_files_in_path(self, tmp_path):
         collection = Collection(tmp_path, exclude={'**/*.dng',}, max_deep=1,
                 use_date_filename=True, use_file_dates=True)
@@ -217,16 +234,35 @@ class TestCollection:
         collection = Collection(path, logger=self.logger)
         loc = GeoLocation()
         summary = collection.init(loc)
-        summary, result = collection.sort_similar_images(path, similarity=60)
+        summary = collection.sort_similar_images(path, similarity=60)
 
         # Summary is created and there is no errors
-        assert summary, summary
+        assert not summary.errors
-        assert result, result
 
-        summary, result = collection.revert_compare(path)
+        summary = collection.revert_compare(path)
 
         # Summary is created and there is no errors
-        assert summary, summary
+        assert not summary.errors
-        assert result, result
+
+    @pytest.mark.skip()
+    def test_fill_data(self, tmp_path, monkeypatch):
+        path = tmp_path / 'collection'
+        shutil.copytree(self.src_path, path)
+        collection = Collection(path, logger=self.logger)
+        # loc = GeoLocation()
+        import ipdb; ipdb.set_trace()
+
+#         def mockreturn(prompt, theme):
+#             return {'value': '03-12-2021 08:12:35'}
+
+#         monkeypatch.setattr(inquirer, 'prompt', mockreturn)
+#         collection.fill_data(path, 'date_original')
+#         # check if db value is set
+#         import ipdb; ipdb.set_trace()
+#         date = collection.db.get_metadata_data('test_exif/invalid.invalid',
+#                 'DateOriginal')
+#         assert date == '2021-03-12 08:12:35'
+        # Check if exif value is set
 
 
+        collection.fill_data(path, 'date_original', edit=True)