ordigi/elodie.py

548 lines
20 KiB
Python
Raw Normal View History

#!/usr/bin/env python
import os
import re
import sys
2021-07-16 21:26:42 +02:00
import logging
from datetime import datetime
import click
2016-01-08 19:04:45 +01:00
from send2trash import send2trash
# Verify that external dependencies are present first, so the user gets a
# more user-friendly error instead of an ImportError traceback.
from elodie.dependencies import verify_dependencies
if not verify_dependencies():
sys.exit(1)
from elodie import constants
from elodie import geolocation
2016-11-10 05:33:52 +01:00
from elodie import log
from elodie.compatability import _decode
from elodie.config import load_config
from elodie.filesystem import FileSystem
2021-07-26 20:50:51 +02:00
from elodie.gui import CompareImageApp
from elodie.localstorage import Db
from elodie.media.media import Media, get_all_subclasses
from elodie.media.audio import Audio
from elodie.media.photo import Photo
from elodie.media.video import Video
from elodie.plugins.plugins import Plugins
from elodie.result import Result
2021-07-16 21:26:42 +02:00
from elodie.summary import Summary
from elodie.external.pyexiftool import ExifTool
from elodie.dependencies import get_exiftool
from elodie import constants
2015-12-31 00:29:13 +01:00
FILESYSTEM = FileSystem()
2021-07-16 21:26:42 +02:00
def print_help(command):
click.echo(command.get_help(click.Context(sort)))
def import_file(_file, destination, db, album_from_folder, mode, trash, allow_duplicates):
2015-12-31 00:29:13 +01:00
"""Set file metadata and move it to destination.
"""
if not os.path.exists(_file):
2016-11-10 05:33:52 +01:00
log.warn('Could not find %s' % _file)
log.all('{"source":"%s", "error_msg":"Could not find %s"}' %
(_file, _file))
return
# Check if the source, _file, is a child folder within destination
elif destination.startswith(os.path.abspath(os.path.dirname(_file))+os.sep):
log.all('{"source": "%s", "destination": "%s", "error_msg": "Source cannot be in destination"}' % (
_file, destination))
return
media = Media.get_class_by_file(_file, get_all_subclasses())
if not media:
2016-11-10 05:33:52 +01:00
log.warn('Not a supported file (%s)' % _file)
log.all('{"source":"%s", "error_msg":"Not a supported file"}' % _file)
return
dest_path = FILESYSTEM.process_file(_file, destination, db,
2021-07-16 21:26:42 +02:00
media, album_from_folder, mode, allowDuplicate=allow_duplicates)
if dest_path:
log.all('%s -> %s' % (_file, dest_path))
2016-01-08 19:04:45 +01:00
if trash:
send2trash(_file)
return dest_path or None
2021-07-16 21:26:42 +02:00
@click.command('batch')
@click.option('--debug', default=False, is_flag=True,
help='Override the value in constants.py with True.')
def _batch(debug):
"""Run batch() for all plugins.
"""
constants.debug = debug
plugins = Plugins()
plugins.run_batch()
@click.command('import')
@click.option('--destination', type=click.Path(file_okay=False),
required=True, help='Copy imported files into this directory.')
@click.option('--source', type=click.Path(file_okay=False),
help='Import files from this directory, if specified.')
@click.option('--file', type=click.Path(dir_okay=False),
help='Import this file, if specified.')
@click.option('--album-from-folder', default=False, is_flag=True,
help="Use images' folders as their album names.")
@click.option('--trash', default=False, is_flag=True,
help='After copying files, move the old files to the trash.')
@click.option('--allow-duplicates', default=False, is_flag=True,
help='Import the file even if it\'s already been imported.')
@click.option('--debug', default=False, is_flag=True,
help='Override the value in constants.py with True.')
2021-06-12 21:34:39 +02:00
@click.option('--dry-run', default=False, is_flag=True,
help='Dry run only, no change made to the filesystem.')
@click.option('--exclude-regex', default=set(), multiple=True,
help='Regular expression for directories or files to exclude.')
@click.argument('paths', nargs=-1, type=click.Path())
2021-06-12 21:34:39 +02:00
def _import(destination, source, file, album_from_folder, trash,
allow_duplicates, debug, dry_run, exclude_regex, paths):
"""Import files or directories by reading their EXIF and organizing them accordingly.
2015-12-31 00:29:13 +01:00
"""
2021-06-12 21:34:39 +02:00
if dry_run:
2021-07-16 21:26:42 +02:00
mode = 'dry_run'
2021-06-12 21:34:39 +02:00
else:
2021-07-16 21:26:42 +02:00
mode = 'copy'
2021-06-12 21:34:39 +02:00
constants.debug = debug
has_errors = False
result = Result()
destination = _decode(destination)
destination = os.path.abspath(os.path.expanduser(destination))
files = set()
paths = set(paths)
if source:
source = _decode(source)
paths.add(source)
if file:
paths.add(file)
# if no exclude list was passed in we check if there's a config
if len(exclude_regex) == 0:
2021-04-16 20:02:14 +02:00
config = load_config(constants.CONFIG_FILE)
if 'Exclusions' in config:
exclude_regex = [value for key, value in config.items('Exclusions')]
exclude_regex_list = set(exclude_regex)
for path in paths:
path = os.path.expanduser(path)
if os.path.isdir(path):
2021-06-27 07:18:35 +02:00
files.update(FILESYSTEM.get_all_files(path, False, exclude_regex_list))
else:
if not FILESYSTEM.should_exclude(path, exclude_regex_list, True):
files.add(path)
# Initialize Db
if os.path.exists(destination):
db = Db(destination)
for current_file in files:
dest_path = import_file(current_file, destination, db,
2021-07-16 21:26:42 +02:00
album_from_folder, mode, trash, allow_duplicates)
result.append((current_file, dest_path))
has_errors = has_errors is True or not dest_path
else:
result.append((destination, False))
has_errors = True
result.write()
2015-12-31 00:29:13 +01:00
if has_errors:
sys.exit(1)
2021-07-16 21:26:42 +02:00
@click.command('sort')
@click.option('--debug', default=False, is_flag=True,
help='Override the value in constants.py with True.')
@click.option('--dry-run', default=False, is_flag=True,
help='Dry run only, no change made to the filesystem.')
@click.option('--destination', '-d', type=click.Path(file_okay=False),
default=None, help='Sort files into this directory.')
@click.option('--copy', '-c', default=False, is_flag=True,
help='True if you want files to be copied over from src_dir to\
dest_dir rather than moved')
@click.option('--exclude-regex', '-e', default=set(), multiple=True,
help='Regular expression for directories or files to exclude.')
2021-07-17 14:09:47 +02:00
@click.option('--filter-by-ext', '-f', default=set(), multiple=True, help='''Use filename
extension to filter files for sorting. If value is '*', use
2021-07-16 21:26:42 +02:00
common media file extension for filtering. Ignored files remain in
the same directory structure''' )
@click.option('--ignore-tags', '-i', default=set(), multiple=True,
help='Specific tags or group that will be ignored when\
searching for file data. Example \'File:FileModifyDate\' or \'Filename\'' )
2021-07-18 07:31:11 +02:00
@click.option('--keep-folders', '-k', default=None,
help='Folder from given level are keep back')
2021-07-18 08:14:01 +02:00
@click.option('--max-deep', '-m', default=None,
help='Maximum level to proceed. Number from 0 to desired level.')
2021-07-16 21:26:42 +02:00
@click.option('--remove-duplicates', '-r', default=False, is_flag=True,
help='True to remove files that are exactly the same in name\
and a file hash')
@click.option('--verbose', '-v', default=False, is_flag=True,
help='True if you want to see details of file processing')
@click.argument('paths', required=True, nargs=-1, type=click.Path())
2021-07-17 09:27:14 +02:00
def _sort(debug, dry_run, destination, copy, exclude_regex, filter_by_ext, ignore_tags,
2021-07-18 08:14:01 +02:00
keep_folders, max_deep, remove_duplicates, verbose, paths):
2021-07-16 21:26:42 +02:00
"""Sort files or directories by reading their EXIF and organizing them
according to config.ini preferences.
"""
if copy:
mode = 'copy'
else:
mode = 'move'
if debug:
constants.debug = logging.DEBUG
elif verbose:
constants.debug = logging.INFO
else:
constants.debug = logging.ERROR
2021-07-18 07:31:11 +02:00
if keep_folders is not None:
keep_folders = int(keep_folders)
2021-07-18 08:14:01 +02:00
if max_deep is not None:
max_deep = int(max_deep)
2021-07-16 21:26:42 +02:00
logger = logging.getLogger('elodie')
logger.setLevel(constants.debug)
if not destination and paths:
destination = paths[-1]
paths = paths[0:-1]
else:
sys.exit(1)
paths = set(paths)
destination = _decode(destination)
destination = os.path.abspath(os.path.expanduser(destination))
if not os.path.exists(destination):
logger.error(f'Directory {destination} does not exist')
# if no exclude list was passed in we check if there's a config
if len(exclude_regex) == 0:
config = load_config(constants.CONFIG_FILE)
if 'Exclusions' in config:
exclude_regex = [value for key, value in config.items('Exclusions')]
exclude_regex_list = set(exclude_regex)
# Initialize Db
db = Db(destination)
2021-07-17 09:27:14 +02:00
if 'Directory' in config and 'day_begins' in config['Directory']:
config_directory = config['Directory']
day_begins = config_directory['day_begins']
else:
day_begins = 0
filesystem = FileSystem(mode, dry_run, exclude_regex_list, logger,
2021-07-18 08:14:01 +02:00
day_begins, filter_by_ext, keep_folders, max_deep)
2021-07-17 09:27:14 +02:00
summary, has_errors = filesystem.sort_files(paths, destination, db,
2021-07-17 16:47:31 +02:00
remove_duplicates, ignore_tags)
2021-07-16 21:26:42 +02:00
if verbose or debug:
summary.write()
if has_errors:
sys.exit(1)
@click.command('generate-db')
2021-06-20 19:51:21 +02:00
@click.option('--path', type=click.Path(file_okay=False),
required=True, help='Path of your photo library.')
@click.option('--debug', default=False, is_flag=True,
help='Override the value in constants.py with True.')
def _generate_db(path, debug):
2018-03-02 09:24:18 +01:00
"""Regenerate the hash.json database which contains all of the sha256 signatures of media files. The hash.json file is located at ~/.elodie/.
"""
constants.debug = debug
result = Result()
2021-06-20 19:51:21 +02:00
path = os.path.abspath(os.path.expanduser(path))
2021-06-20 19:51:21 +02:00
if not os.path.isdir(path):
log.error('path is not a valid directory %s' % path)
sys.exit(1)
db = Db(path)
db.backup_hash_db()
db.reset_hash_db()
2021-06-20 19:51:21 +02:00
for current_file in FILESYSTEM.get_all_files(path):
result.append((current_file, True))
db.add_hash(db.checksum(current_file), current_file)
log.progress()
2021-06-20 19:51:21 +02:00
db.update_hash_db()
log.progress('', True)
result.write()
2021-07-16 21:26:42 +02:00
@click.command('verify')
2021-06-20 19:51:21 +02:00
@click.option('--path', type=click.Path(file_okay=False),
required=True, help='Path of your photo library.')
@click.option('--debug', default=False, is_flag=True,
help='Override the value in constants.py with True.')
2021-06-20 19:51:21 +02:00
def _verify(path, debug):
constants.debug = debug
result = Result()
db = Db(path)
for checksum, file_path in db.all():
if not os.path.isfile(file_path):
result.append((file_path, False))
log.progress('x')
continue
actual_checksum = db.checksum(file_path)
if checksum == actual_checksum:
result.append((file_path, True))
log.progress()
else:
result.append((file_path, False))
log.progress('x')
log.progress('', True)
result.write()
def update_location(media, file_path, location_name, db):
2015-12-30 23:52:01 +01:00
"""Update location exif metadata of media.
"""
location_coords = geolocation.coordinates_by_name(location_name, db)
2015-12-30 23:52:01 +01:00
if location_coords and 'latitude' in location_coords and \
'longitude' in location_coords:
location_status = media.set_location(location_coords[
2021-06-20 19:51:21 +02:00
'latitude'], location_coords['longitude'], file_path)
2015-12-30 23:52:01 +01:00
if not location_status:
2016-11-10 05:33:52 +01:00
log.error('Failed to update location')
log.all(('{"source":"%s",' % file_path,
'"error_msg":"Failed to update location"}'))
2015-12-30 23:52:01 +01:00
sys.exit(1)
return True
def update_time(media, file_path, time_string):
"""Update time exif metadata of media.
"""
time_format = '%Y-%m-%d %H:%M:%S'
if re.match(r'^\d{4}-\d{2}-\d{2}$', time_string):
time_string = '%s 00:00:00' % time_string
elif re.match(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}\d{2}$', time_string):
msg = ('Invalid time format. Use YYYY-mm-dd hh:ii:ss or YYYY-mm-dd')
2016-11-10 05:33:52 +01:00
log.error(msg)
log.all('{"source":"%s", "error_msg":"%s"}' % (file_path, msg))
2015-12-30 23:52:01 +01:00
sys.exit(1)
time = datetime.strptime(time_string, time_format)
media.set_date_original(time, file_path)
2015-12-30 23:52:01 +01:00
return True
@click.command('update')
@click.option('--album', help='Update the image album.')
@click.option('--location', help=('Update the image location. Location '
'should be the name of a place, like "Las '
'Vegas, NV".'))
@click.option('--time', help=('Update the image time. Time should be in '
'YYYY-mm-dd hh:ii:ss or YYYY-mm-dd format.'))
@click.option('--title', help='Update the image title.')
@click.option('--debug', default=False, is_flag=True,
help='Override the value in constants.py with True.')
@click.argument('paths', nargs=-1,
required=True)
def _update(album, location, time, title, paths, debug):
"""Update a file's EXIF. Automatically modifies the file's location and file name accordingly.
2015-12-30 23:52:01 +01:00
"""
constants.debug = debug
has_errors = False
result = Result()
files = set()
for path in paths:
path = os.path.expanduser(path)
if os.path.isdir(path):
2021-06-27 07:18:35 +02:00
files.update(FILESYSTEM.get_all_files(path, False))
else:
files.add(path)
for current_file in files:
if not os.path.exists(current_file):
has_errors = True
result.append((current_file, False))
log.warn('Could not find %s' % current_file)
log.all('{"source":"%s", "error_msg":"Could not find %s"}' %
(current_file, current_file))
continue
current_file = os.path.expanduser(current_file)
# The destination folder structure could contain any number of levels
# So we calculate that and traverse up the tree.
# '/path/to/file/photo.jpg' -> '/path/to/file' ->
# ['path','to','file'] -> ['path','to'] -> '/path/to'
current_directory = os.path.dirname(current_file)
destination_depth = -1 * len(FILESYSTEM.get_folder_path_definition())
destination = os.sep.join(
os.path.normpath(
current_directory
).split(os.sep)[:destination_depth]
)
2015-12-02 23:07:23 +01:00
# Initialize Db
db = Db(destination)
media = Media.get_class_by_file(current_file, get_all_subclasses())
2021-07-16 21:26:42 +02:00
if media is None:
continue
updated = False
if location:
update_location(media, current_file, location, db)
updated = True
if time:
update_time(media, current_file, time)
updated = True
if album:
2021-06-20 19:51:21 +02:00
media.set_album(album, current_file)
updated = True
# Updating a title can be problematic when doing it 2+ times on a file.
# You would end up with img_001.jpg -> img_001-first-title.jpg ->
# img_001-first-title-second-title.jpg.
# To resolve that we have to track the prior title (if there was one.
# Then we massage the updated_media's metadata['base_name'] to remove
# the old title.
# Since FileSystem.get_file_name() relies on base_name it will properly
# rename the file by updating the title instead of appending it.
remove_old_title_from_name = False
if title:
# We call get_metadata() to cache it before making any changes
metadata = media.get_metadata()
title_update_status = media.set_title(title)
original_title = metadata['title']
if title_update_status and original_title:
2015-12-31 00:29:13 +01:00
# @TODO: We should move this to a shared method since
# FileSystem.get_file_name() does it too.
original_title = re.sub(r'\W+', '-', original_title.lower())
original_base_name = metadata['base_name']
remove_old_title_from_name = True
updated = True
if updated:
updated_media = Media.get_class_by_file(current_file,
get_all_subclasses())
2015-12-31 00:29:13 +01:00
# See comments above on why we have to do this when titles
# get updated.
if remove_old_title_from_name and len(original_title) > 0:
updated_media.get_metadata()
updated_media.set_metadata_basename(
original_base_name.replace('-%s' % original_title, ''))
dest_path = FILESYSTEM.process_file(current_file, destination, db,
2021-07-16 21:26:42 +02:00
updated_media, False, mode='move', allowDuplicate=True)
log.info(u'%s -> %s' % (current_file, dest_path))
log.all('{"source":"%s", "destination":"%s"}' % (current_file,
dest_path))
2015-12-31 00:29:13 +01:00
# If the folder we moved the file out of or its parent are empty
# we delete it.
FILESYSTEM.delete_directory_if_empty(os.path.dirname(current_file))
2015-12-31 00:29:13 +01:00
FILESYSTEM.delete_directory_if_empty(
os.path.dirname(os.path.dirname(current_file)))
result.append((current_file, dest_path))
# Trip has_errors to False if it's already False or dest_path is.
has_errors = has_errors is True or not dest_path
else:
has_errors = False
result.append((current_file, False))
result.write()
2021-06-27 07:28:51 +02:00
if has_errors:
sys.exit(1)
2021-07-26 20:50:51 +02:00
@click.command('compare')
@click.option('--debug', default=False, is_flag=True,
help='Override the value in constants.py with True.')
@click.option('--dry-run', default=False, is_flag=True,
help='Dry run only, no change made to the filesystem.')
@click.option('--find-duplicates', '-f', default=False, is_flag=True)
@click.option('--output-dir', '-o', default=False, is_flag=True, help='output\
dir')
@click.option('--remove-duplicates', '-r', default=False, is_flag=True)
@click.option('--revert-compare', '-R', default=False, is_flag=True, help='Revert\
compare')
@click.option('--similar-to', '-s', default=False, help='Similar to given\
image')
@click.option('--similarity', '-S', default=80, help='Similarity level for\
images')
@click.option('--verbose', '-v', default=False, is_flag=True,
help='True if you want to see details of file processing')
@click.argument('path', nargs=1, required=True)
def _compare(debug, dry_run, find_duplicates, output_dir, remove_duplicates,
revert_compare, similar_to, similarity, verbose, path):
'''Compare files in directories'''
logger = logging.getLogger('elodie')
if debug:
logger.setLevel(logging.DEBUG)
elif verbose:
logger.setLevel(logging.INFO)
else:
logger.setLevel(logging.ERROR)
# Initialize Db
db = Db(path)
filesystem = FileSystem(mode='move', dry_run=dry_run, logger=logger)
if revert_compare:
summary, has_errors = filesystem.revert_compare(path, db, dry_run)
else:
summary, has_errors = filesystem.sort_similar_images(path, db,
similarity, dry_run)
if verbose or debug:
summary.write()
if has_errors:
sys.exit(1)
@click.group()
def main():
pass
2021-07-26 20:50:51 +02:00
main.add_command(_compare)
main.add_command(_import)
2021-07-16 21:26:42 +02:00
main.add_command(_sort)
main.add_command(_update)
main.add_command(_generate_db)
main.add_command(_verify)
main.add_command(_batch)
2015-12-31 00:29:13 +01:00
if __name__ == '__main__':
#Initialize ExifTool Subprocess
exiftool_addedargs = [
u'-config',
u'"{}"'.format(constants.exiftool_config)
]
with ExifTool(executable_=get_exiftool(), addedargs=exiftool_addedargs) as et:
main()