Fix sort_file and add solve_conflict function
This commit is contained in:
parent
3bd06f0321
commit
b9c30a4d85
|
@ -5,8 +5,8 @@
|
||||||
# be a number between 0-23')
|
# be a number between 0-23')
|
||||||
day_begins=4
|
day_begins=4
|
||||||
|
|
||||||
dirs_path=%u{%Y-%m}/{city}|{city}-{%Y}/{folders[:1]}/{folder}
|
dirs_path={%Y}/{%m-%b}-{city}-{folder}
|
||||||
name={%Y-%m-%b-%H-%M-%S}-{basename}.%l{ext}
|
name={%Y%m%d-%H%M%S}-%u{original_name}.%l{ext}
|
||||||
|
|
||||||
[Exclusions]
|
[Exclusions]
|
||||||
name1=.directory
|
name1=.directory
|
||||||
|
|
|
@ -76,7 +76,6 @@ class FileSystem(object):
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def get_items(self):
|
def get_items(self):
|
||||||
return {
|
return {
|
||||||
'album': '{album}',
|
'album': '{album}',
|
||||||
|
@ -98,7 +97,6 @@ class FileSystem(object):
|
||||||
'date': '{(%[a-zA-Z][^a-zA-Z]*){1,8}}' # search for date format string
|
'date': '{(%[a-zA-Z][^a-zA-Z]*){1,8}}' # search for date format string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def walklevel(self, src_path, maxlevel=None):
|
def walklevel(self, src_path, maxlevel=None):
|
||||||
"""
|
"""
|
||||||
Walk into input directory recursively until desired maxlevel
|
Walk into input directory recursively until desired maxlevel
|
||||||
|
@ -115,7 +113,6 @@ class FileSystem(object):
|
||||||
if maxlevel is not None and level >= maxlevel:
|
if maxlevel is not None and level >= maxlevel:
|
||||||
del dirs[:]
|
del dirs[:]
|
||||||
|
|
||||||
|
|
||||||
def get_all_files(self, path, extensions=False, exclude_regex_list=set()):
|
def get_all_files(self, path, extensions=False, exclude_regex_list=set()):
|
||||||
"""Recursively get all files which match a path and extension.
|
"""Recursively get all files which match a path and extension.
|
||||||
|
|
||||||
|
@ -150,7 +147,6 @@ class FileSystem(object):
|
||||||
):
|
):
|
||||||
yield filename_path
|
yield filename_path
|
||||||
|
|
||||||
|
|
||||||
def check_for_early_morning_photos(self, date):
|
def check_for_early_morning_photos(self, date):
|
||||||
"""check for early hour photos to be grouped with previous day"""
|
"""check for early hour photos to be grouped with previous day"""
|
||||||
|
|
||||||
|
@ -161,7 +157,6 @@ class FileSystem(object):
|
||||||
|
|
||||||
return date
|
return date
|
||||||
|
|
||||||
|
|
||||||
def get_location_part(self, mask, part, place_name):
|
def get_location_part(self, mask, part, place_name):
|
||||||
"""Takes a mask for a location and interpolates the actual place names.
|
"""Takes a mask for a location and interpolates the actual place names.
|
||||||
|
|
||||||
|
@ -194,7 +189,6 @@ class FileSystem(object):
|
||||||
|
|
||||||
return folder_name
|
return folder_name
|
||||||
|
|
||||||
|
|
||||||
def get_part(self, item, mask, metadata, db, subdirs):
|
def get_part(self, item, mask, metadata, db, subdirs):
|
||||||
"""Parse a specific folder's name given a mask and metadata.
|
"""Parse a specific folder's name given a mask and metadata.
|
||||||
|
|
||||||
|
@ -282,15 +276,20 @@ class FileSystem(object):
|
||||||
|
|
||||||
part = part.strip()
|
part = part.strip()
|
||||||
|
|
||||||
# Capitalization
|
if part == '':
|
||||||
u_regex = '%u' + regex
|
# delete separator if any
|
||||||
l_regex = '%l' + regex
|
regex = '[-_ .]?(%[ul])?' + regex
|
||||||
if re.search(u_regex, this_part):
|
|
||||||
this_part = re.sub(u_regex, part.upper(), this_part)
|
|
||||||
elif re.search(l_regex, this_part):
|
|
||||||
this_part = re.sub(l_regex, part.lower(), this_part)
|
|
||||||
else:
|
|
||||||
this_part = re.sub(regex, part, this_part)
|
this_part = re.sub(regex, part, this_part)
|
||||||
|
else:
|
||||||
|
# Capitalization
|
||||||
|
u_regex = '%u' + regex
|
||||||
|
l_regex = '%l' + regex
|
||||||
|
if re.search(u_regex, this_part):
|
||||||
|
this_part = re.sub(u_regex, part.upper(), this_part)
|
||||||
|
elif re.search(l_regex, this_part):
|
||||||
|
this_part = re.sub(l_regex, part.lower(), this_part)
|
||||||
|
else:
|
||||||
|
this_part = re.sub(regex, part, this_part)
|
||||||
|
|
||||||
|
|
||||||
if this_part:
|
if this_part:
|
||||||
|
@ -411,7 +410,6 @@ class FileSystem(object):
|
||||||
elif metadata['date_modified'] is not None:
|
elif metadata['date_modified'] is not None:
|
||||||
return metadata['date_modified']
|
return metadata['date_modified']
|
||||||
|
|
||||||
|
|
||||||
def checksum(self, file_path, blocksize=65536):
|
def checksum(self, file_path, blocksize=65536):
|
||||||
"""Create a hash value for the given file.
|
"""Create a hash value for the given file.
|
||||||
|
|
||||||
|
@ -432,7 +430,6 @@ class FileSystem(object):
|
||||||
return hasher.hexdigest()
|
return hasher.hexdigest()
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def checkcomp(self, dest_path, src_checksum):
|
def checkcomp(self, dest_path, src_checksum):
|
||||||
"""Check file.
|
"""Check file.
|
||||||
"""
|
"""
|
||||||
|
@ -449,7 +446,6 @@ class FileSystem(object):
|
||||||
|
|
||||||
return src_checksum
|
return src_checksum
|
||||||
|
|
||||||
|
|
||||||
def sort_file(self, src_path, dest_path, remove_duplicates=True):
|
def sort_file(self, src_path, dest_path, remove_duplicates=True):
|
||||||
'''Copy or move file to dest_path.'''
|
'''Copy or move file to dest_path.'''
|
||||||
|
|
||||||
|
@ -459,8 +455,8 @@ class FileSystem(object):
|
||||||
# check for collisions
|
# check for collisions
|
||||||
if(src_path == dest_path):
|
if(src_path == dest_path):
|
||||||
self.logger.info(f'File {dest_path} already sorted')
|
self.logger.info(f'File {dest_path} already sorted')
|
||||||
return True
|
return None
|
||||||
if os.path.isfile(dest_path):
|
elif os.path.isfile(dest_path):
|
||||||
self.logger.info(f'File {dest_path} already exist')
|
self.logger.info(f'File {dest_path} already exist')
|
||||||
if remove_duplicates:
|
if remove_duplicates:
|
||||||
if filecmp.cmp(src_path, dest_path):
|
if filecmp.cmp(src_path, dest_path):
|
||||||
|
@ -469,7 +465,7 @@ class FileSystem(object):
|
||||||
if not dry_run:
|
if not dry_run:
|
||||||
os.remove(src_path)
|
os.remove(src_path)
|
||||||
self.logger.info(f'remove: {src_path}')
|
self.logger.info(f'remove: {src_path}')
|
||||||
return True
|
return None
|
||||||
else: # name is same, but file is different
|
else: # name is same, but file is different
|
||||||
self.logger.info(f'File in source and destination are different.')
|
self.logger.info(f'File in source and destination are different.')
|
||||||
return False
|
return False
|
||||||
|
@ -487,9 +483,6 @@ class FileSystem(object):
|
||||||
self.logger.info(f'copy: {src_path} -> {dest_path}')
|
self.logger.info(f'copy: {src_path} -> {dest_path}')
|
||||||
return True
|
return True
|
||||||
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def check_file(self, src_path, dest_path, src_checksum, db):
|
def check_file(self, src_path, dest_path, src_checksum, db):
|
||||||
|
|
||||||
# Check if file remain the same
|
# Check if file remain the same
|
||||||
|
@ -500,9 +493,6 @@ class FileSystem(object):
|
||||||
db.add_hash(checksum, dest_path)
|
db.add_hash(checksum, dest_path)
|
||||||
db.update_hash_db()
|
db.update_hash_db()
|
||||||
|
|
||||||
if dest_path:
|
|
||||||
self.logger.info(f'{src_path} -> {dest_path}')
|
|
||||||
|
|
||||||
self.summary.append((src_path, dest_path))
|
self.summary.append((src_path, dest_path))
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
@ -550,6 +540,35 @@ class FileSystem(object):
|
||||||
|
|
||||||
return file_list
|
return file_list
|
||||||
|
|
||||||
|
def _conflict_solved(self, conflict_file_list, item, dest_path):
|
||||||
|
self.logger.warning(f'Same name already exists...renaming to: {dest_path}')
|
||||||
|
del(conflict_file_list[item])
|
||||||
|
|
||||||
|
def solve_conflicts(self, conflict_file_list, remove_duplicates):
|
||||||
|
file_list = conflict_file_list.copy()
|
||||||
|
for item, file_paths in enumerate(file_list):
|
||||||
|
src_path = file_paths['src_path']
|
||||||
|
dest_path = file_paths['dest_path']
|
||||||
|
# Try to sort the file
|
||||||
|
result = self.sort_file(src_path, dest_path, remove_duplicates)
|
||||||
|
# remove to conflict file list if file as be successfully copied or ignored
|
||||||
|
if result is True or None:
|
||||||
|
self._conflict_solved(conflict_file_list, item, dest_path)
|
||||||
|
else:
|
||||||
|
n = 1
|
||||||
|
while result is False:
|
||||||
|
if n > 100:
|
||||||
|
self.logger.warning(f'{self.mode}: to many append for {dest_path}...')
|
||||||
|
break
|
||||||
|
# Add appendix to the name
|
||||||
|
pre, ext = os.path.splitext(dest_path)
|
||||||
|
dest_path = pre + '_' + str(n) + ext
|
||||||
|
conflict_file_list[item]['dest_path'] = dest_path
|
||||||
|
result = self.sort_file(src_path, dest_path, remove_duplicates)
|
||||||
|
else:
|
||||||
|
self._conflict_solved(conflict_file_list, item, dest_path)
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
def sort_files(self, paths, destination, db, remove_duplicates=False,
|
def sort_files(self, paths, destination, db, remove_duplicates=False,
|
||||||
ignore_tags=set()):
|
ignore_tags=set()):
|
||||||
|
@ -564,11 +583,12 @@ class FileSystem(object):
|
||||||
|
|
||||||
path = os.path.expanduser(path)
|
path = os.path.expanduser(path)
|
||||||
|
|
||||||
conflict_file_list = set()
|
conflict_file_list = []
|
||||||
for src_path, subdirs in self.get_files_in_path(path):
|
for src_path, subdirs in self.get_files_in_path(path,
|
||||||
|
extensions=self.filter_by_ext):
|
||||||
# Process files
|
# Process files
|
||||||
src_checksum = self.checksum(src_path)
|
src_checksum = self.checksum(src_path)
|
||||||
media = get_media_class(src_path, ignore_tags, self.logger)
|
media = Media(src_path, ignore_tags, self.logger)
|
||||||
if media:
|
if media:
|
||||||
metadata = media.get_metadata()
|
metadata = media.get_metadata()
|
||||||
# Get the destination path according to metadata
|
# Get the destination path according to metadata
|
||||||
|
@ -583,40 +603,23 @@ class FileSystem(object):
|
||||||
|
|
||||||
self.create_directory(dest_directory)
|
self.create_directory(dest_directory)
|
||||||
result = self.sort_file(src_path, dest_path, remove_duplicates)
|
result = self.sort_file(src_path, dest_path, remove_duplicates)
|
||||||
if result:
|
|
||||||
self.summary, has_errors = self.check_file(src_path,
|
if result is False:
|
||||||
dest_path, src_checksum, db)
|
|
||||||
else:
|
|
||||||
# There is conflict files
|
# There is conflict files
|
||||||
conflict_file_list.add((src_path, dest_path))
|
conflict_file_list.append({'src_path': src_path, 'dest_path': dest_path})
|
||||||
|
result = self.solve_conflicts(conflict_file_list, remove_duplicates)
|
||||||
|
|
||||||
for src_path, dest_path in conflict_file_list:
|
if result is True:
|
||||||
# Try to sort the file
|
|
||||||
result = self.sort_file(src_path, dest_path, remove_duplicates)
|
|
||||||
if result:
|
|
||||||
conflict_file_list.remove((src_path, dest_path))
|
|
||||||
else:
|
|
||||||
n = 1
|
|
||||||
while not result:
|
|
||||||
# Add appendix to the name
|
|
||||||
pre, ext = os.path.splitext(dest_path)
|
|
||||||
dest_path = pre + '_' + str(n) + ext
|
|
||||||
result = self.sort_file(src_path, dest_path, remove_duplicates)
|
|
||||||
if n > 100:
|
|
||||||
self.logger.error(f'{self.mode}: to many append for {dest_path}...')
|
|
||||||
break
|
|
||||||
self.logger.info(f'Same name already exists...renaming to: {dest_path}')
|
|
||||||
|
|
||||||
if result:
|
|
||||||
self.summary, has_errors = self.check_file(src_path,
|
self.summary, has_errors = self.check_file(src_path,
|
||||||
dest_path, src_checksum, db)
|
dest_path, src_checksum, db)
|
||||||
|
elif result is None:
|
||||||
|
has_errors = False
|
||||||
else:
|
else:
|
||||||
self.summary.append((src_path, False))
|
self.summary.append((src_path, False))
|
||||||
has_errors = True
|
has_errors = True
|
||||||
|
|
||||||
return self.summary, has_errors
|
return self.summary, has_errors
|
||||||
|
|
||||||
|
|
||||||
def check_path(self, path):
|
def check_path(self, path):
|
||||||
path = os.path.abspath(os.path.expanduser(path))
|
path = os.path.abspath(os.path.expanduser(path))
|
||||||
|
|
||||||
|
@ -627,7 +630,6 @@ class FileSystem(object):
|
||||||
|
|
||||||
return path
|
return path
|
||||||
|
|
||||||
|
|
||||||
def set_hash(self, result, src_path, dest_path, src_checksum, db):
|
def set_hash(self, result, src_path, dest_path, src_checksum, db):
|
||||||
if result:
|
if result:
|
||||||
# Check if file remain the same
|
# Check if file remain the same
|
||||||
|
@ -654,7 +656,6 @@ class FileSystem(object):
|
||||||
|
|
||||||
return has_errors
|
return has_errors
|
||||||
|
|
||||||
|
|
||||||
def move_file(self, img_path, dest_path, checksum, db):
|
def move_file(self, img_path, dest_path, checksum, db):
|
||||||
if not self.dry_run:
|
if not self.dry_run:
|
||||||
try:
|
try:
|
||||||
|
@ -665,7 +666,6 @@ class FileSystem(object):
|
||||||
self.logger.info(f'move: {img_path} -> {dest_path}')
|
self.logger.info(f'move: {img_path} -> {dest_path}')
|
||||||
return self.set_hash(True, img_path, dest_path, checksum, db)
|
return self.set_hash(True, img_path, dest_path, checksum, db)
|
||||||
|
|
||||||
|
|
||||||
def sort_similar_images(self, path, db, similarity=80):
|
def sort_similar_images(self, path, db, similarity=80):
|
||||||
|
|
||||||
has_errors = False
|
has_errors = False
|
||||||
|
@ -680,21 +680,21 @@ class FileSystem(object):
|
||||||
for filename in filenames:
|
for filename in filenames:
|
||||||
file_paths.add(os.path.join(dirname, filename))
|
file_paths.add(os.path.join(dirname, filename))
|
||||||
|
|
||||||
photo = Photo(logger=self.logger)
|
i = Images(file_paths, logger=self.logger)
|
||||||
|
|
||||||
images = set([ i for i in photo.get_images(file_paths) ])
|
images = set([ i for i in i.get_images() ])
|
||||||
for image in images:
|
for image in images:
|
||||||
if not os.path.isfile(image):
|
if not os.path.isfile(image):
|
||||||
continue
|
continue
|
||||||
checksum1 = self.checksum(image)
|
checksum1 = self.checksum(image)
|
||||||
# Process files
|
# Process files
|
||||||
# media = get_media_class(src_path, False, self.logger)
|
# media = Media(src_path, False, self.logger)
|
||||||
# TODO compare metadata
|
# TODO compare metadata
|
||||||
# if media:
|
# if media:
|
||||||
# metadata = media.get_metadata()
|
# metadata = media.get_metadata()
|
||||||
similar = False
|
similar = False
|
||||||
moved_imgs = set()
|
moved_imgs = set()
|
||||||
for img_path in photo.find_similar(image, file_paths, similarity):
|
for img_path in i.find_similar(image, similarity):
|
||||||
similar = True
|
similar = True
|
||||||
checksum2 = self.checksum(img_path)
|
checksum2 = self.checksum(img_path)
|
||||||
# move image into directory
|
# move image into directory
|
||||||
|
@ -728,7 +728,6 @@ class FileSystem(object):
|
||||||
|
|
||||||
return self.summary, has_errors
|
return self.summary, has_errors
|
||||||
|
|
||||||
|
|
||||||
def revert_compare(self, path, db):
|
def revert_compare(self, path, db):
|
||||||
|
|
||||||
has_errors = False
|
has_errors = False
|
||||||
|
@ -760,7 +759,6 @@ class FileSystem(object):
|
||||||
|
|
||||||
return self.summary, has_errors
|
return self.summary, has_errors
|
||||||
|
|
||||||
|
|
||||||
def set_utime_from_metadata(self, date_taken, file_path):
|
def set_utime_from_metadata(self, date_taken, file_path):
|
||||||
""" Set the modification time on the file based on the file name.
|
""" Set the modification time on the file based on the file name.
|
||||||
"""
|
"""
|
||||||
|
@ -768,7 +766,6 @@ class FileSystem(object):
|
||||||
# Initialize date taken to what's returned from the metadata function.
|
# Initialize date taken to what's returned from the metadata function.
|
||||||
os.utime(file_path, (int(datetime.now().timestamp()), int(date_taken.timestamp())))
|
os.utime(file_path, (int(datetime.now().timestamp()), int(date_taken.timestamp())))
|
||||||
|
|
||||||
|
|
||||||
def should_exclude(self, path, regex_list=set(), needs_compiled=False):
|
def should_exclude(self, path, regex_list=set(), needs_compiled=False):
|
||||||
if(len(regex_list) == 0):
|
if(len(regex_list) == 0):
|
||||||
return False
|
return False
|
||||||
|
|
Loading…
Reference in New Issue