Revamp and fix options

This commit is contained in:
Cédric Leporcq 2022-07-23 20:12:03 +02:00
parent 01b47c8c40
commit 573a63998e
6 changed files with 74 additions and 63 deletions

View File

@ -1,5 +1,17 @@
[Exif]
#album_from_folder=False
fill_date_original=True
#cache=True
#ignore_tags=None
use_date_filename=True
#use_file_dates=False
[Filters] [Filters]
exclude=["**/.directory", "**/.DS_Store"] exclude=["**/.directory", "**/.DS_Store"]
#extensions=None
#glob=**/*
#max_deep=None
remove_duplicates=True
[Geolocation] [Geolocation]
geocoder=Nominatim geocoder=Nominatim
@ -15,5 +27,9 @@ day_begins=4
# Path format # Path format
dirs_path=<%Y>/<%m-%b>_<location>_<folder> dirs_path=<%Y>/<%m-%b>_<location>_<folder>
name=<%Y%m%d-%H%M%S>_<<original_name>|<name>>.%l<ext> name=<%Y%m%d-%H%M%S>_<<name>.%l<ext>|<original_name>>
# name=<%Y%m%d-%H%M%S>-%u<original_name>.%l<ext> # name=<%Y%m%d-%H%M%S>-%u<original_name>.%l<ext>
[Terminal]
dry_run=False
interactive=False

View File

@ -7,6 +7,7 @@ import click
from ordigi import log, LOG from ordigi import log, LOG
from ordigi.collection import Collection from ordigi.collection import Collection
from ordigi import constants
from ordigi.geolocation import GeoLocation from ordigi.geolocation import GeoLocation
from ordigi import utils from ordigi import utils
@ -85,7 +86,7 @@ _sort_options = [
click.option( click.option(
'--path-format', '--path-format',
'-p', '-p',
default=None, default=constants.DEFAULT_PATH_FORMAT,
help='Custom featured path format', help='Custom featured path format',
), ),
click.option( click.option(
@ -147,15 +148,10 @@ def _cli_get_location(collection):
) )
def _cli_sort(collection, src_paths, import_mode, remove_duplicates): def _cli_sort(collection, src_paths, import_mode):
loc = _cli_get_location(collection) loc = _cli_get_location(collection)
path_format = collection.opt['Path']['path_format'] return collection.sort_files(src_paths, loc, import_mode)
LOG.debug(f'path_format: {path_format}')
return collection.sort_files(
src_paths, path_format, loc, import_mode, remove_duplicates
)
@click.group() @click.group()
@ -231,24 +227,21 @@ def _clean(**kwargs):
collection = Collection( collection = Collection(
root, root,
{ {
"dry_run": kwargs['dry_run'], 'dry_run': kwargs['dry_run'],
"extensions": kwargs['ext'], 'extensions': kwargs['ext'],
"glob": kwargs['glob'], 'glob': kwargs['glob'],
'remove_duplicates': kwargs['remove_duplicates'],
}, },
) )
# os.path.join( # os.path.join(
# TODO make function to remove duplicates # TODO make function to remove duplicates
# path_format = collection.opt['Path']['path_format'] # path_format = collection.opt['Path']['path_format']
# summary = collection.sort_files( # summary = collection.sort_files(paths, None)
# paths, path_format, None, remove_duplicates=kwargs['remove_duplicates']
# )
if kwargs['path_string']: if kwargs['path_string']:
dedup_regex = set(kwargs['dedup_regex']) dedup_regex = set(kwargs['dedup_regex'])
collection.dedup_path( collection.dedup_path(paths, dedup_regex)
paths, dedup_regex, kwargs['remove_duplicates']
)
for path in paths: for path in paths:
if folders: if folders:
@ -334,9 +327,10 @@ def _compare(**kwargs):
collection = Collection( collection = Collection(
root, root,
{ {
"extensions": kwargs['ext'], 'extensions': kwargs['ext'],
"glob": kwargs['glob'], 'glob': kwargs['glob'],
"dry_run": kwargs['dry_run'], 'dry_run': kwargs['dry_run'],
'remove_duplicates': kwargs['remove_duplicates'],
}, },
) )
@ -515,7 +509,7 @@ def _import(**kwargs):
'dry_run': kwargs['dry_run'], 'dry_run': kwargs['dry_run'],
'interactive': kwargs['interactive'], 'interactive': kwargs['interactive'],
'path_format': kwargs['path_format'], 'path_format': kwargs['path_format'],
'remove_duplicates': kwargs['remove_duplicates'],
} }
) )
@ -523,7 +517,7 @@ def _import(**kwargs):
import_mode = 'copy' import_mode = 'copy'
else: else:
import_mode = 'move' import_mode = 'move'
summary = _cli_sort(collection, src_paths, import_mode, kwargs['remove_duplicates']) summary = _cli_sort(collection, src_paths, import_mode)
if log_level < 30: if log_level < 30:
summary.print() summary.print()
@ -572,10 +566,11 @@ def _sort(**kwargs):
'glob': kwargs['glob'], 'glob': kwargs['glob'],
'dry_run': kwargs['dry_run'], 'dry_run': kwargs['dry_run'],
'interactive': kwargs['interactive'], 'interactive': kwargs['interactive'],
'remove_duplicates': kwargs['remove_duplicates'],
} }
) )
summary = _cli_sort(collection, paths, False, kwargs['remove_duplicates']) summary = _cli_sort(collection, paths, False)
if kwargs['clean']: if kwargs['clean']:
collection.remove_empty_folders(root) collection.remove_empty_folders(root)

View File

@ -477,6 +477,7 @@ class SortMedias:
db=None, db=None,
dry_run=False, dry_run=False,
interactive=False, interactive=False,
remove_duplicates=False,
): ):
# Arguments # Arguments
@ -489,6 +490,7 @@ class SortMedias:
self.dry_run = dry_run self.dry_run = dry_run
self.interactive = interactive self.interactive = interactive
self.log = LOG.getChild(self.__class__.__name__) self.log = LOG.getChild(self.__class__.__name__)
self.remove_duplicates = remove_duplicates
self.summary = Summary(self.root) self.summary = Summary(self.root)
# Attributes # Attributes
@ -600,7 +602,7 @@ class SortMedias:
directory_path.mkdir(parents=True, exist_ok=True) directory_path.mkdir(parents=True, exist_ok=True)
self.log.info(f'Create {directory_path}') self.log.info(f'Create {directory_path}')
def check_conflicts(self, src_path, dest_path, remove_duplicates=False): def check_conflicts(self, src_path, dest_path):
""" """
Check if file can be copied or moved file to dest_path. Check if file can be copied or moved file to dest_path.
""" """
@ -616,7 +618,7 @@ class SortMedias:
if dest_path.is_file(): if dest_path.is_file():
self.log.info(f"File {dest_path} already exist") self.log.info(f"File {dest_path} already exist")
if remove_duplicates: if self.remove_duplicates:
if filecmp.cmp(src_path, dest_path): if filecmp.cmp(src_path, dest_path):
self.log.info( self.log.info(
"File in source and destination are identical. Duplicate will be ignored." "File in source and destination are identical. Duplicate will be ignored."
@ -633,13 +635,13 @@ class SortMedias:
return 0 return 0
def _solve_conflicts(self, conflicts, remove_duplicates): def _solve_conflicts(self, conflicts):
unresolved_conflicts = [] unresolved_conflicts = []
while conflicts != []: while conflicts != []:
src_path, dest_path, metadata = conflicts.pop() src_path, dest_path, metadata = conflicts.pop()
# Check for conflict status again in case is has changed # Check for conflict status again in case is has changed
conflict = self.check_conflicts(src_path, dest_path, remove_duplicates) conflict = self.check_conflicts(src_path, dest_path)
for i in range(1, 1000): for i in range(1, 1000):
if conflict != 1: if conflict != 1:
@ -652,7 +654,7 @@ class SortMedias:
else: else:
stem = dest_path.stem stem = dest_path.stem
dest_path = dest_path.parent / (stem + '_' + str(i) + suffix) dest_path = dest_path.parent / (stem + '_' + str(i) + suffix)
conflict = self.check_conflicts(src_path, dest_path, remove_duplicates) conflict = self.check_conflicts(src_path, dest_path)
if conflict == 1: if conflict == 1:
# i = 100: # i = 100:
@ -663,7 +665,7 @@ class SortMedias:
yield (src_path, dest_path, metadata), conflict yield (src_path, dest_path, metadata), conflict
def sort_medias(self, imp=False, remove_duplicates=False): def sort_medias(self, imp=False):
""" """
sort files and solve conflicts sort files and solve conflicts
""" """
@ -674,7 +676,7 @@ class SortMedias:
for src_path, metadata in self.medias.datas.items(): for src_path, metadata in self.medias.datas.items():
dest_path = self.root / metadata['file_path'] dest_path = self.root / metadata['file_path']
conflict = self.check_conflicts(src_path, dest_path, remove_duplicates) conflict = self.check_conflicts(src_path, dest_path)
if not conflict: if not conflict:
self.sort_file( self.sort_file(
@ -692,9 +694,7 @@ class SortMedias:
pass pass
if conflicts != []: if conflicts != []:
for files_data, conflict in self._solve_conflicts( for files_data, conflict in self._solve_conflicts(conflicts):
conflicts, remove_duplicates
):
src_path, dest_path, metadata = files_data src_path, dest_path, metadata = files_data
if not conflict: if not conflict:
@ -727,13 +727,13 @@ class Collection(SortMedias):
self.log = LOG.getChild(self.__class__.__name__) self.log = LOG.getChild(self.__class__.__name__)
# Get config options # Get config options
self.opt = self.get_config_options() self.opt, default_options = self.get_config_options()
# Set client options # Set client options
for option, value in cli_options.items(): for option, value in cli_options.items():
if value not in (None, set()):
for section in self.opt: for section in self.opt:
if option in self.opt[section]: if option in self.opt[section]:
if value != default_options[section][option]:
if option == 'exclude': if option == 'exclude':
self.opt[section][option].union(set(value)) self.opt[section][option].union(set(value))
elif option in ('ignore_tags', 'extensions'): elif option in ('ignore_tags', 'extensions'):
@ -772,6 +772,7 @@ class Collection(SortMedias):
self.db, self.db,
self.opt['Terminal']['dry_run'], self.opt['Terminal']['dry_run'],
self.opt['Terminal']['interactive'], self.opt['Terminal']['interactive'],
self.opt['Filters']['remove_duplicates'],
) )
# Attributes # Attributes
@ -792,7 +793,7 @@ class Collection(SortMedias):
"""Get collection config""" """Get collection config"""
config = Config(self.root.joinpath('.ordigi', 'ordigi.conf')) config = Config(self.root.joinpath('.ordigi', 'ordigi.conf'))
return config.get_config_options() return config.get_config_options(), config.get_default_options()
def _set_option(self, section, option, cli_option): def _set_option(self, section, option, cli_option):
"""if client option is set overwrite collection option value""" """if client option is set overwrite collection option value"""
@ -1003,17 +1004,15 @@ class Collection(SortMedias):
return self.summary return self.summary
def sort_files( def sort_files(self, src_dirs, loc, imp=False):
self, src_dirs, path_format, loc, imp=False, remove_duplicates=False
):
""" """
Sort files into appropriate folder Sort files into appropriate folder
""" """
# Check db # Check db
self._init_check_db(loc) self._init_check_db(loc)
# if path format client option is set overwrite it path_format = self.opt['Path']['path_format']
self._set_option('Path', 'path_format', path_format) self.log.debug(f'path_format: {path_format}')
# Get medias data # Get medias data
subdirs = set() subdirs = set()
@ -1027,7 +1026,7 @@ class Collection(SortMedias):
self.medias.datas[src_path] = copy(metadata) self.medias.datas[src_path] = copy(metadata)
# Sort files and solve conflicts # Sort files and solve conflicts
self.summary = self.sort_medias(imp, remove_duplicates) self.summary = self.sort_medias(imp)
if imp != 'copy': if imp != 'copy':
self.remove_empty_subdirs(subdirs, src_dirs) self.remove_empty_subdirs(subdirs, src_dirs)
@ -1037,7 +1036,7 @@ class Collection(SortMedias):
return self.summary return self.summary
def dedup_path(self, paths, dedup_regex=None, remove_duplicates=False): def dedup_path(self, paths, dedup_regex=None):
"""Deduplicate file path parts""" """Deduplicate file path parts"""
# Check db # Check db
@ -1078,7 +1077,7 @@ class Collection(SortMedias):
self.medias.datas[src_path] = copy(metadata) self.medias.datas[src_path] = copy(metadata)
# Sort files and solve conflicts # Sort files and solve conflicts
self.sort_medias(remove_duplicates=remove_duplicates) self.sort_medias()
if not self.check_db(): if not self.check_db():
self.summary.append('check', False) self.summary.append('check', False)
@ -1110,7 +1109,7 @@ class Collection(SortMedias):
return True return True
def sort_similar_images(self, path, similarity=80, remove_duplicates=False): def sort_similar_images(self, path, similarity=80):
"""Sort similar images using imagehash library""" """Sort similar images using imagehash library"""
# Check db # Check db
self._init_check_db() self._init_check_db()
@ -1129,7 +1128,7 @@ class Collection(SortMedias):
) )
if similar_images: if similar_images:
# Move the simlars file into the destination directory # Move the simlars file into the destination directory
self.sort_medias(remove_duplicates=remove_duplicates) self.sort_medias()
nb_row_end = self.db.sqlite.len('metadata') nb_row_end = self.db.sqlite.len('metadata')
if nb_row_ini and nb_row_ini != nb_row_end: if nb_row_ini and nb_row_ini != nb_row_end:

View File

@ -53,9 +53,9 @@ class Config:
else: else:
self.conf = conf self.conf = conf
self.options = self.set_default_options() self.options = self.get_default_options()
def set_default_options(self) -> dict: def get_default_options(self) -> dict:
# Initialize with default options # Initialize with default options
return { return {
'Exif': { 'Exif': {
@ -71,6 +71,7 @@ class Config:
'extensions': None, 'extensions': None,
'glob': '**/*', 'glob': '**/*',
'max_deep': None, 'max_deep': None,
'remove_duplicates': False,
}, },
'Geolocation': { 'Geolocation': {
'geocoder': constants.DEFAULT_GEOCODER, 'geocoder': constants.DEFAULT_GEOCODER,
@ -137,6 +138,7 @@ class Config:
'dry_run', 'dry_run',
'interactive', 'interactive',
'prefer_english_names', 'prefer_english_names',
'remove_duplicates',
'use_date_filename', 'use_date_filename',
'use_file_dates', 'use_file_dates',
} }

View File

@ -207,7 +207,6 @@ class TestOrdigi:
('--exclude', '.DS_Store'), ('--exclude', '.DS_Store'),
*self.filter_options, *self.filter_options,
('--path-format', '{%Y}/{folder}/{stem}.{ext}'), ('--path-format', '{%Y}/{folder}/{stem}.{ext}'),
) )
paths = (str(self.src_path), str(tmp_path)) paths = (str(self.src_path), str(tmp_path))

View File

@ -137,11 +137,12 @@ class TestCollection:
assert summary.success_table.sum('sort') == nb assert summary.success_table.sum('sort') == nb
def test_sort_files(self, tmp_path): def test_sort_files(self, tmp_path):
cli_options = {'album_from_folder': True, 'cache': False} cli_options = {
'album_from_folder': True, 'cache': False, 'path_format': self.path_format
}
collection = Collection(tmp_path, cli_options=cli_options) collection = Collection(tmp_path, cli_options=cli_options)
loc = GeoLocation() loc = GeoLocation()
summary = collection.sort_files([self.src_path], summary = collection.sort_files([self.src_path], loc, imp='copy')
self.path_format, loc, imp='copy')
self.assert_import(summary, 29) self.assert_import(summary, 29)
@ -166,16 +167,16 @@ class TestCollection:
collection = Collection(tmp_path, cli_options=cli_options) collection = Collection(tmp_path, cli_options=cli_options)
# Try to change path format and sort files again # Try to change path format and sort files again
path_format = 'test_exif/<city>/<%Y>-<name>.%l<ext>' path_format = 'test_exif/<city>/<%Y>-<name>.%l<ext>'
summary = collection.sort_files([tmp_path], path_format, loc) summary = collection.sort_files([tmp_path], loc)
self.assert_sort(summary, 27) self.assert_sort(summary, 23)
shutil.copytree(tmp_path / 'test_exif', tmp_path / 'test_exif_copy') shutil.copytree(tmp_path / 'test_exif', tmp_path / 'test_exif_copy')
collection.summary = Summary(tmp_path) collection.summary = Summary(tmp_path)
assert collection.summary.success_table.sum() == 0 assert collection.summary.success_table.sum() == 0
summary = collection.update(loc) summary = collection.update(loc)
assert summary.success_table.sum('update') == 29 assert summary.success_table.sum('update') == 2
assert summary.success_table.sum() == 29 assert summary.success_table.sum() == 2
assert not summary.errors assert not summary.errors
collection.summary = Summary(tmp_path) collection.summary = Summary(tmp_path)
summary = collection.update(loc) summary = collection.update(loc)
@ -195,12 +196,11 @@ class TestCollection:
assert not summary.errors assert not summary.errors
def test_sort_files_invalid_db(self, tmp_path): def test_sort_files_invalid_db(self, tmp_path):
collection = Collection(tmp_path) collection = Collection(tmp_path, {'path_format': self.path_format})
loc = GeoLocation() loc = GeoLocation()
randomize_db(tmp_path) randomize_db(tmp_path)
with pytest.raises(sqlite3.DatabaseError) as e: with pytest.raises(sqlite3.DatabaseError) as e:
summary = collection.sort_files([self.src_path], summary = collection.sort_files([self.src_path], loc, imp='copy')
self.path_format, loc, imp='copy')
def test_sort_file(self, tmp_path): def test_sort_file(self, tmp_path):
for imp in ('copy', 'move', False): for imp in ('copy', 'move', False):