From d8cee15f32d3853070111c972ecf987a05a55549 Mon Sep 17 00:00:00 2001 From: Arian Maleki <6453083+amaleki@users.noreply.github.com> Date: Mon, 13 Jan 2020 23:04:33 -0800 Subject: [PATCH] Optimize exiftool calls by adding an ExifTool singleton in pyexiftool library (#352) This fix results in a 10x performance improvement [1] enabling a single exiftool subprocess to elimate spawing exiftool for each image. Closes #350 #347 [1] https://github.com/jmathai/elodie/issues/350#issuecomment-573412006 --- elodie.py | 13 ++++++-- elodie/external/pyexiftool.py | 12 ++++++- elodie/filesystem.py | 4 +-- elodie/media/media.py | 31 ++++++------------- elodie/tests/filesystem_test.py | 14 ++++++++- elodie/tests/helper.py | 13 ++++++++ elodie/tests/media/audio_test.py | 13 ++++++++ elodie/tests/media/base_test.py | 2 ++ elodie/tests/media/media_test.py | 4 ++- elodie/tests/media/photo_test.py | 3 ++ .../plugins/googlephotos/googlephotos_test.py | 11 +++++-- 11 files changed, 87 insertions(+), 33 deletions(-) diff --git a/elodie.py b/elodie.py index 0f292af..521548c 100755 --- a/elodie.py +++ b/elodie.py @@ -30,11 +30,12 @@ from elodie.media.photo import Photo from elodie.media.video import Video from elodie.plugins.plugins import Plugins from elodie.result import Result - +from elodie.external.pyexiftool import ExifTool +from elodie.dependencies import get_exiftool +from elodie import constants FILESYSTEM = FileSystem() - def import_file(_file, destination, album_from_folder, trash, allow_duplicates): _file = _decode(_file) @@ -368,4 +369,10 @@ main.add_command(_batch) if __name__ == '__main__': - main() + #Initialize ExifTool Subprocess + exiftool_addedargs = [ + u'-config', + u'"{}"'.format(constants.exiftool_config) + ] + with ExifTool(executable_=get_exiftool(), addedargs=exiftool_addedargs) as et: + main() diff --git a/elodie/external/pyexiftool.py b/elodie/external/pyexiftool.py index 0ba18bb..d09aa2a 100644 --- a/elodie/external/pyexiftool.py +++ b/elodie/external/pyexiftool.py @@ -65,6 +65,8 @@ import warnings import logging import codecs +from future.utils import with_metaclass + try: # Py3k compatibility basestring except NameError: @@ -151,8 +153,16 @@ def format_error (result): else: return 'exiftool finished with error: "%s"' % strip_nl(result) +class Singleton(type): + """Metaclass to use the singleton [anti-]pattern""" + instance = None -class ExifTool(object): + def __call__(cls, *args, **kwargs): + if cls.instance is None: + cls.instance = super(Singleton, cls).__call__(*args, **kwargs) + return cls.instance + +class ExifTool(object, with_metaclass(Singleton)): """Run the `exiftool` command-line tool and communicate to it. You can pass two arguments to the constructor: diff --git a/elodie/filesystem.py b/elodie/filesystem.py index c97082b..b585eb0 100644 --- a/elodie/filesystem.py +++ b/elodie/filesystem.py @@ -19,7 +19,6 @@ from elodie.localstorage import Db from elodie.media.base import Base, get_all_subclasses from elodie.plugins.plugins import Plugins - class FileSystem(object): """A class for interacting with the file system.""" @@ -48,7 +47,6 @@ class FileSystem(object): # Instantiate a plugins object self.plugins = Plugins() - def create_directory(self, directory_path): """Create a directory if it does not already exist. @@ -644,4 +642,4 @@ class FileSystem(object): compiled_list.append(re.compile(regex)) regex_list = compiled_list - return any(regex.search(path) for regex in regex_list) + return any(regex.search(path) for regex in regex_list) \ No newline at end of file diff --git a/elodie/media/media.py b/elodie/media/media.py index 1eca732..dce4ef3 100644 --- a/elodie/media/media.py +++ b/elodie/media/media.py @@ -13,12 +13,9 @@ from __future__ import print_function import os # load modules -from elodie import constants -from elodie.dependencies import get_exiftool from elodie.external.pyexiftool import ExifTool from elodie.media.base import Base - class Media(Base): """The base class for all media objects. @@ -52,10 +49,7 @@ class Media(Base): self.longitude_ref_key = 'EXIF:GPSLongitudeRef' self.original_name_key = 'XMP:OriginalFileName' self.set_gps_ref = True - self.exiftool_addedargs = [ - u'-config', - u'"{}"'.format(constants.exiftool_config) - ] + self.exif_metadata = None def get_album(self): """Get album from EXIF @@ -122,16 +116,15 @@ class Media(Base): :returns: dict, or False if exiftool was not available. """ source = self.source - exiftool = get_exiftool() - if(exiftool is None): + + #Cache exif metadata results and use if already exists for media + if(self.exif_metadata is None): + self.exif_metadata = ExifTool().get_metadata(source) + + if not self.exif_metadata: return False - with ExifTool(executable_=exiftool, addedargs=self.exiftool_addedargs) as et: - metadata = et.get_metadata(source) - if not metadata: - return False - - return metadata + return self.exif_metadata def get_camera_make(self): """Get the camera make stored in EXIF. @@ -211,6 +204,7 @@ class Media(Base): """Resets any internal cache """ self.exiftool_attributes = None + self.exif_metadata = None super(Media, self).reset_cache() def set_album(self, album): @@ -318,13 +312,8 @@ class Media(Base): return None source = self.source - exiftool = get_exiftool() - if(exiftool is None): - return False - status = '' - with ExifTool(executable_=exiftool, addedargs=self.exiftool_addedargs) as et: - status = et.set_tags(tags, source) + status = ExifTool().set_tags(tags,source) return status != '' diff --git a/elodie/tests/filesystem_test.py b/elodie/tests/filesystem_test.py index b48444a..601da11 100644 --- a/elodie/tests/filesystem_test.py +++ b/elodie/tests/filesystem_test.py @@ -20,9 +20,21 @@ from elodie.media.media import Media from elodie.media.photo import Photo from elodie.media.video import Video from nose.plugins.skip import SkipTest +from elodie.external.pyexiftool import ExifTool +from elodie.dependencies import get_exiftool +from elodie import constants os.environ['TZ'] = 'GMT' +def setup_module(): + exiftool_addedargs = [ + u'-config', + u'"{}"'.format(constants.exiftool_config) + ] + ExifTool(executable_=get_exiftool(), addedargs=exiftool_addedargs).start() + +def teardown_module(): + ExifTool().terminate def test_create_directory_success(): filesystem = FileSystem() @@ -575,7 +587,7 @@ full_path=%year/%month/%location path = filesystem.get_folder_path(media.get_metadata()) if hasattr(load_config, 'config'): del load_config.config - + assert path == os.path.join('2015','12','Sunnyvale, California'), path @mock.patch('elodie.config.config_file', '%s/config.ini-location-date' % gettempdir()) diff --git a/elodie/tests/helper.py b/elodie/tests/helper.py index adf6530..01ed000 100644 --- a/elodie/tests/helper.py +++ b/elodie/tests/helper.py @@ -15,6 +15,8 @@ from datetime import datetime from datetime import timedelta from elodie.compatability import _rename +from elodie.external.pyexiftool import ExifTool +from elodie.dependencies import get_exiftool from elodie import constants def checksum(file_path, blocksize=65536): @@ -159,3 +161,14 @@ def restore_dbs(): # This is no longer needed. See gh-322 # https://github.com/jmathai/elodie/issues/322 pass + + +def setup_module(): + exiftool_addedargs = [ + u'-config', + u'"{}"'.format(constants.exiftool_config) + ] + ExifTool(executable_=get_exiftool(), addedargs=exiftool_addedargs).start() + +def teardown_module(): + ExifTool().terminate diff --git a/elodie/tests/media/audio_test.py b/elodie/tests/media/audio_test.py index c66a0e9..e46fea6 100644 --- a/elodie/tests/media/audio_test.py +++ b/elodie/tests/media/audio_test.py @@ -16,9 +16,22 @@ import helper from elodie.media.media import Media from elodie.media.video import Video from elodie.media.audio import Audio +from elodie.external.pyexiftool import ExifTool +from elodie.dependencies import get_exiftool +from elodie import constants os.environ['TZ'] = 'GMT' +def setup_module(): + exiftool_addedargs = [ + u'-config', + u'"{}"'.format(constants.exiftool_config) + ] + ExifTool(executable_=get_exiftool(), addedargs=exiftool_addedargs).start() + +def teardown_module(): + ExifTool().terminate + def test_audio_extensions(): audio = Audio() extensions = audio.extensions diff --git a/elodie/tests/media/base_test.py b/elodie/tests/media/base_test.py index bc9dfbf..9838c80 100644 --- a/elodie/tests/media/base_test.py +++ b/elodie/tests/media/base_test.py @@ -23,6 +23,8 @@ from elodie.media.video import Video os.environ['TZ'] = 'GMT' +setup_module = helper.setup_module +teardown_module = helper.teardown_module def test_get_all_subclasses(): subclasses = get_all_subclasses(Base) diff --git a/elodie/tests/media/media_test.py b/elodie/tests/media/media_test.py index 068a8b9..f435bb4 100644 --- a/elodie/tests/media/media_test.py +++ b/elodie/tests/media/media_test.py @@ -21,6 +21,8 @@ from elodie.media.video import Video os.environ['TZ'] = 'GMT' +setup_module = helper.setup_module +teardown_module = helper.teardown_module def test_get_file_path(): media = Media(helper.get_file('plain.jpg')) @@ -86,7 +88,7 @@ def test_get_original_name_invalid_file(): original_name = media.get_original_name() assert original_name is None, original_name - + def test_set_original_name_when_exists(): temporary_folder, folder = helper.create_working_folder() diff --git a/elodie/tests/media/photo_test.py b/elodie/tests/media/photo_test.py index 110bed6..8ca389b 100644 --- a/elodie/tests/media/photo_test.py +++ b/elodie/tests/media/photo_test.py @@ -20,6 +20,9 @@ from elodie.media.photo import Photo os.environ['TZ'] = 'GMT' +setup_module = helper.setup_module +teardown_module = helper.teardown_module + def test_photo_extensions(): photo = Photo() extensions = photo.extensions diff --git a/elodie/tests/plugins/googlephotos/googlephotos_test.py b/elodie/tests/plugins/googlephotos/googlephotos_test.py index 16098f1..a29ec3c 100644 --- a/elodie/tests/plugins/googlephotos/googlephotos_test.py +++ b/elodie/tests/plugins/googlephotos/googlephotos_test.py @@ -30,9 +30,8 @@ config_string_fmt = config_string.format( secrets_file ) -sample_photo = Photo(helper.get_file('plain.jpg')) -sample_metadata = sample_photo.get_metadata() -sample_metadata['original_name'] = 'foobar' +setup_module = helper.setup_module +teardown_module = helper.teardown_module @mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-set-session' % gettempdir()) def test_googlephotos_set_session(): @@ -57,6 +56,9 @@ def test_googlephotos_after_supported(): if hasattr(load_config, 'config'): del load_config.config + sample_photo = Photo(helper.get_file('plain.jpg')) + sample_metadata = sample_photo.get_metadata() + sample_metadata['original_name'] = 'foobar' final_file_path = helper.get_file('plain.jpg') gp = GooglePhotos() gp.after('', '', final_file_path, sample_metadata) @@ -162,6 +164,9 @@ def test_googlephotos_batch(): if hasattr(load_config, 'config'): del load_config.config + sample_photo = Photo(helper.get_file('plain.jpg')) + sample_metadata = sample_photo.get_metadata() + sample_metadata['original_name'] = 'foobar' final_file_path = helper.get_file('plain.jpg') gp = GooglePhotos() gp.after('', '', final_file_path, sample_metadata)