takino/ordigi
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix sort_file and add solve_conflict function

Browse Source
This commit is contained in:
Cédric Leporcq 2021-08-14 21:36:22 +02:00
parent 3bd06f0321
commit b9c30a4d85
2 changed files with 62 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ordigi.conf
View File

@ -5,8 +5,8 @@
# be a number between 0-23') # be a number between 0-23')
day_begins=4 day_begins=4
dirs_path=%u{%Y-%m}/{city}|{city}-{%Y}/{folders[:1]}/{folder} dirs_path={%Y}/{%m-%b}-{city}-{folder}
name={%Y-%m-%b-%H-%M-%S}-{basename}.%l{ext} name={%Y%m%d-%H%M%S}-%u{original_name}.%l{ext}
[Exclusions] [Exclusions]
name1=.directory name1=.directory

123
ordigi/filesystem.py
View File

@ -76,7 +76,6 @@ class FileSystem(object):
return False return False
def get_items(self): def get_items(self):
return { return {
'album': '{album}', 'album': '{album}',
@ -98,7 +97,6 @@ class FileSystem(object):
'date': '{(%[a-zA-Z][^a-zA-Z]*){1,8}}' # search for date format string 'date': '{(%[a-zA-Z][^a-zA-Z]*){1,8}}' # search for date format string
} }
def walklevel(self, src_path, maxlevel=None): def walklevel(self, src_path, maxlevel=None):
""" """
Walk into input directory recursively until desired maxlevel Walk into input directory recursively until desired maxlevel
@ -115,7 +113,6 @@ class FileSystem(object):
if maxlevel is not None and level >= maxlevel: if maxlevel is not None and level >= maxlevel:
del dirs[:] del dirs[:]
def get_all_files(self, path, extensions=False, exclude_regex_list=set()): def get_all_files(self, path, extensions=False, exclude_regex_list=set()):
"""Recursively get all files which match a path and extension. """Recursively get all files which match a path and extension.
@ -150,7 +147,6 @@ class FileSystem(object):
): ):
yield filename_path yield filename_path
def check_for_early_morning_photos(self, date): def check_for_early_morning_photos(self, date):
"""check for early hour photos to be grouped with previous day""" """check for early hour photos to be grouped with previous day"""
@ -161,7 +157,6 @@ class FileSystem(object):
return date return date
def get_location_part(self, mask, part, place_name): def get_location_part(self, mask, part, place_name):
"""Takes a mask for a location and interpolates the actual place names. """Takes a mask for a location and interpolates the actual place names.
@ -194,7 +189,6 @@ class FileSystem(object):
return folder_name return folder_name
def get_part(self, item, mask, metadata, db, subdirs): def get_part(self, item, mask, metadata, db, subdirs):
"""Parse a specific folder's name given a mask and metadata. """Parse a specific folder's name given a mask and metadata.
@ -282,15 +276,20 @@ class FileSystem(object):
part = part.strip() part = part.strip()
# Capitalization if part == '':
u_regex = '%u' + regex # delete separator if any
l_regex = '%l' + regex regex = '[-_ .]?(%[ul])?' + regex
if re.search(u_regex, this_part):
this_part = re.sub(u_regex, part.upper(), this_part)
elif re.search(l_regex, this_part):
this_part = re.sub(l_regex, part.lower(), this_part)
else:
this_part = re.sub(regex, part, this_part) this_part = re.sub(regex, part, this_part)
else:
# Capitalization
u_regex = '%u' + regex
l_regex = '%l' + regex
if re.search(u_regex, this_part):
this_part = re.sub(u_regex, part.upper(), this_part)
elif re.search(l_regex, this_part):
this_part = re.sub(l_regex, part.lower(), this_part)
else:
this_part = re.sub(regex, part, this_part)
if this_part: if this_part:
@ -411,7 +410,6 @@ class FileSystem(object):
elif metadata['date_modified'] is not None: elif metadata['date_modified'] is not None:
return metadata['date_modified'] return metadata['date_modified']
def checksum(self, file_path, blocksize=65536): def checksum(self, file_path, blocksize=65536):
"""Create a hash value for the given file. """Create a hash value for the given file.
@ -432,7 +430,6 @@ class FileSystem(object):
return hasher.hexdigest() return hasher.hexdigest()
return None return None
def checkcomp(self, dest_path, src_checksum): def checkcomp(self, dest_path, src_checksum):
"""Check file. """Check file.
""" """
@ -449,7 +446,6 @@ class FileSystem(object):
return src_checksum return src_checksum
def sort_file(self, src_path, dest_path, remove_duplicates=True): def sort_file(self, src_path, dest_path, remove_duplicates=True):
'''Copy or move file to dest_path.''' '''Copy or move file to dest_path.'''
@ -459,8 +455,8 @@ class FileSystem(object):
# check for collisions # check for collisions
if(src_path == dest_path): if(src_path == dest_path):
self.logger.info(f'File {dest_path} already sorted') self.logger.info(f'File {dest_path} already sorted')
return True return None
if os.path.isfile(dest_path): elif os.path.isfile(dest_path):
self.logger.info(f'File {dest_path} already exist') self.logger.info(f'File {dest_path} already exist')
if remove_duplicates: if remove_duplicates:
if filecmp.cmp(src_path, dest_path): if filecmp.cmp(src_path, dest_path):
@ -469,7 +465,7 @@ class FileSystem(object):
if not dry_run: if not dry_run:
os.remove(src_path) os.remove(src_path)
self.logger.info(f'remove: {src_path}') self.logger.info(f'remove: {src_path}')
return True return None
else: # name is same, but file is different else: # name is same, but file is different
self.logger.info(f'File in source and destination are different.') self.logger.info(f'File in source and destination are different.')
return False return False
@ -487,9 +483,6 @@ class FileSystem(object):
self.logger.info(f'copy: {src_path} -> {dest_path}') self.logger.info(f'copy: {src_path} -> {dest_path}')
return True return True
return False
def check_file(self, src_path, dest_path, src_checksum, db): def check_file(self, src_path, dest_path, src_checksum, db):
# Check if file remain the same # Check if file remain the same
@ -500,9 +493,6 @@ class FileSystem(object):
db.add_hash(checksum, dest_path) db.add_hash(checksum, dest_path)
db.update_hash_db() db.update_hash_db()
if dest_path:
self.logger.info(f'{src_path} -> {dest_path}')
self.summary.append((src_path, dest_path)) self.summary.append((src_path, dest_path))
else: else:
@ -550,6 +540,35 @@ class FileSystem(object):
return file_list return file_list
def _conflict_solved(self, conflict_file_list, item, dest_path):
self.logger.warning(f'Same name already exists...renaming to: {dest_path}')
del(conflict_file_list[item])
def solve_conflicts(self, conflict_file_list, remove_duplicates):
file_list = conflict_file_list.copy()
for item, file_paths in enumerate(file_list):
src_path = file_paths['src_path']
dest_path = file_paths['dest_path']
# Try to sort the file
result = self.sort_file(src_path, dest_path, remove_duplicates)
# remove to conflict file list if file as be successfully copied or ignored
if result is True or None:
self._conflict_solved(conflict_file_list, item, dest_path)
else:
n = 1
while result is False:
if n > 100:
self.logger.warning(f'{self.mode}: to many append for {dest_path}...')
break
# Add appendix to the name
pre, ext = os.path.splitext(dest_path)
dest_path = pre + '_' + str(n) + ext
conflict_file_list[item]['dest_path'] = dest_path
result = self.sort_file(src_path, dest_path, remove_duplicates)
else:
self._conflict_solved(conflict_file_list, item, dest_path)
return result
def sort_files(self, paths, destination, db, remove_duplicates=False, def sort_files(self, paths, destination, db, remove_duplicates=False,
ignore_tags=set()): ignore_tags=set()):
@ -564,11 +583,12 @@ class FileSystem(object):
path = os.path.expanduser(path) path = os.path.expanduser(path)
conflict_file_list = set() conflict_file_list = []
for src_path, subdirs in self.get_files_in_path(path): for src_path, subdirs in self.get_files_in_path(path,
extensions=self.filter_by_ext):
# Process files # Process files
src_checksum = self.checksum(src_path) src_checksum = self.checksum(src_path)
media = get_media_class(src_path, ignore_tags, self.logger) media = Media(src_path, ignore_tags, self.logger)
if media: if media:
metadata = media.get_metadata() metadata = media.get_metadata()
# Get the destination path according to metadata # Get the destination path according to metadata
@ -583,40 +603,23 @@ class FileSystem(object):
self.create_directory(dest_directory) self.create_directory(dest_directory)
result = self.sort_file(src_path, dest_path, remove_duplicates) result = self.sort_file(src_path, dest_path, remove_duplicates)
if result:
self.summary, has_errors = self.check_file(src_path, if result is False:
dest_path, src_checksum, db)
else:
# There is conflict files # There is conflict files
conflict_file_list.add((src_path, dest_path)) conflict_file_list.append({'src_path': src_path, 'dest_path': dest_path})
result = self.solve_conflicts(conflict_file_list, remove_duplicates)
for src_path, dest_path in conflict_file_list: if result is True:
# Try to sort the file
result = self.sort_file(src_path, dest_path, remove_duplicates)
if result:
conflict_file_list.remove((src_path, dest_path))
else:
n = 1
while not result:
# Add appendix to the name
pre, ext = os.path.splitext(dest_path)
dest_path = pre + '_' + str(n) + ext
result = self.sort_file(src_path, dest_path, remove_duplicates)
if n > 100:
self.logger.error(f'{self.mode}: to many append for {dest_path}...')
break
self.logger.info(f'Same name already exists...renaming to: {dest_path}')
if result:
self.summary, has_errors = self.check_file(src_path, self.summary, has_errors = self.check_file(src_path,
dest_path, src_checksum, db) dest_path, src_checksum, db)
elif result is None:
has_errors = False
else: else:
self.summary.append((src_path, False)) self.summary.append((src_path, False))
has_errors = True has_errors = True
return self.summary, has_errors return self.summary, has_errors
def check_path(self, path): def check_path(self, path):
path = os.path.abspath(os.path.expanduser(path)) path = os.path.abspath(os.path.expanduser(path))
@ -627,7 +630,6 @@ class FileSystem(object):
return path return path
def set_hash(self, result, src_path, dest_path, src_checksum, db): def set_hash(self, result, src_path, dest_path, src_checksum, db):
if result: if result:
# Check if file remain the same # Check if file remain the same
@ -654,7 +656,6 @@ class FileSystem(object):
return has_errors return has_errors
def move_file(self, img_path, dest_path, checksum, db): def move_file(self, img_path, dest_path, checksum, db):
if not self.dry_run: if not self.dry_run:
try: try:
@ -665,7 +666,6 @@ class FileSystem(object):
self.logger.info(f'move: {img_path} -> {dest_path}') self.logger.info(f'move: {img_path} -> {dest_path}')
return self.set_hash(True, img_path, dest_path, checksum, db) return self.set_hash(True, img_path, dest_path, checksum, db)
def sort_similar_images(self, path, db, similarity=80): def sort_similar_images(self, path, db, similarity=80):
has_errors = False has_errors = False
@ -680,21 +680,21 @@ class FileSystem(object):
for filename in filenames: for filename in filenames:
file_paths.add(os.path.join(dirname, filename)) file_paths.add(os.path.join(dirname, filename))
photo = Photo(logger=self.logger) i = Images(file_paths, logger=self.logger)
images = set([ i for i in photo.get_images(file_paths) ]) images = set([ i for i in i.get_images() ])
for image in images: for image in images:
if not os.path.isfile(image): if not os.path.isfile(image):
continue continue
checksum1 = self.checksum(image) checksum1 = self.checksum(image)
# Process files # Process files
# media = get_media_class(src_path, False, self.logger) # media = Media(src_path, False, self.logger)
# TODO compare metadata # TODO compare metadata
# if media: # if media:
# metadata = media.get_metadata() # metadata = media.get_metadata()
similar = False similar = False
moved_imgs = set() moved_imgs = set()
for img_path in photo.find_similar(image, file_paths, similarity): for img_path in i.find_similar(image, similarity):
similar = True similar = True
checksum2 = self.checksum(img_path) checksum2 = self.checksum(img_path)
# move image into directory # move image into directory
@ -728,7 +728,6 @@ class FileSystem(object):
return self.summary, has_errors return self.summary, has_errors
def revert_compare(self, path, db): def revert_compare(self, path, db):
has_errors = False has_errors = False
@ -760,7 +759,6 @@ class FileSystem(object):
return self.summary, has_errors return self.summary, has_errors
def set_utime_from_metadata(self, date_taken, file_path): def set_utime_from_metadata(self, date_taken, file_path):
""" Set the modification time on the file based on the file name. """ Set the modification time on the file based on the file name.
""" """
@ -768,7 +766,6 @@ class FileSystem(object):
# Initialize date taken to what's returned from the metadata function. # Initialize date taken to what's returned from the metadata function.
os.utime(file_path, (int(datetime.now().timestamp()), int(date_taken.timestamp()))) os.utime(file_path, (int(datetime.now().timestamp()), int(date_taken.timestamp())))
def should_exclude(self, path, regex_list=set(), needs_compiled=False): def should_exclude(self, path, regex_list=set(), needs_compiled=False):
if(len(regex_list) == 0): if(len(regex_list) == 0):
return False return False