Optimize exiftool calls by adding an ExifTool singleton in pyexiftool library (#352)

This fix results in a 10x performance improvement [1] enabling a single exiftool subprocess to elimate spawing exiftool for each image.

Closes #350 #347 

[1] https://github.com/jmathai/elodie/issues/350#issuecomment-573412006

elodie.py

@@ -30,11 +30,12 @@ from elodie.media.photo import Photo
 from elodie.media.video import Video
 from elodie.plugins.plugins import Plugins
 from elodie.result import Result
-
+from elodie.external.pyexiftool import ExifTool
+from elodie.dependencies import get_exiftool
+from elodie import constants
 
 FILESYSTEM = FileSystem()
 
-
 def import_file(_file, destination, album_from_folder, trash, allow_duplicates):
     
     _file = _decode(_file)
@@ -368,4 +369,10 @@ main.add_command(_batch)
 
 
 if __name__ == '__main__':
-    main()
+    #Initialize ExifTool Subprocess
+    exiftool_addedargs = [
+       u'-config',
+        u'"{}"'.format(constants.exiftool_config)
+    ]
+    with ExifTool(executable_=get_exiftool(), addedargs=exiftool_addedargs) as et:
+        main()

elodie/external/pyexiftool.py

@@ -65,6 +65,8 @@ import warnings
 import logging
 import codecs
 
+from future.utils import with_metaclass
+
 try:        # Py3k compatibility
     basestring
 except NameError:
@@ -151,8 +153,16 @@ def format_error (result):
         else:
             return 'exiftool finished with error: "%s"' % strip_nl(result) 
 
+class Singleton(type):
+    """Metaclass to use the singleton [anti-]pattern"""
+    instance = None
 
-class ExifTool(object):
+    def __call__(cls, *args, **kwargs):
+        if cls.instance is None:
+            cls.instance = super(Singleton, cls).__call__(*args, **kwargs)
+        return cls.instance
+
+class ExifTool(object, with_metaclass(Singleton)):
     """Run the `exiftool` command-line tool and communicate to it.
 
     You can pass two arguments to the constructor:

elodie/filesystem.py

@@ -19,7 +19,6 @@ from elodie.localstorage import Db
 from elodie.media.base import Base, get_all_subclasses
 from elodie.plugins.plugins import Plugins
 
-
 class FileSystem(object):
     """A class for interacting with the file system."""
 
@@ -48,7 +47,6 @@ class FileSystem(object):
         # Instantiate a plugins object
         self.plugins = Plugins()
 
-
     def create_directory(self, directory_path):
         """Create a directory if it does not already exist.
 
@@ -644,4 +642,4 @@ class FileSystem(object):
                 compiled_list.append(re.compile(regex))
             regex_list = compiled_list
 
-        return any(regex.search(path) for regex in regex_list)
+        return any(regex.search(path) for regex in regex_list)

elodie/media/media.py

@@ -13,12 +13,9 @@ from __future__ import print_function
 import os
 
 # load modules
-from elodie import constants
-from elodie.dependencies import get_exiftool
 from elodie.external.pyexiftool import ExifTool
 from elodie.media.base import Base
 
-
 class Media(Base):
 
     """The base class for all media objects.
@@ -52,10 +49,7 @@ class Media(Base):
         self.longitude_ref_key = 'EXIF:GPSLongitudeRef'
         self.original_name_key = 'XMP:OriginalFileName'
         self.set_gps_ref = True
-        self.exiftool_addedargs = [
+        self.exif_metadata = None
-            u'-config',
-            u'"{}"'.format(constants.exiftool_config)
-        ]
 
     def get_album(self):
         """Get album from EXIF
@@ -122,16 +116,15 @@ class Media(Base):
         :returns: dict, or False if exiftool was not available.
         """
         source = self.source
-        exiftool = get_exiftool()
+
-        if(exiftool is None):
+        #Cache exif metadata results and use if already exists for media
+        if(self.exif_metadata is None):
+            self.exif_metadata = ExifTool().get_metadata(source)
+
+        if not self.exif_metadata:
             return False
 
-        with ExifTool(executable_=exiftool, addedargs=self.exiftool_addedargs) as et:
+        return self.exif_metadata
-            metadata = et.get_metadata(source)
-            if not metadata:
-                return False
-
-        return metadata
 
     def get_camera_make(self):
         """Get the camera make stored in EXIF.
@@ -211,6 +204,7 @@ class Media(Base):
         """Resets any internal cache
         """
         self.exiftool_attributes = None
+        self.exif_metadata = None
         super(Media, self).reset_cache()
 
     def set_album(self, album):
@@ -318,13 +312,8 @@ class Media(Base):
             return None
 
         source = self.source
-        exiftool = get_exiftool()
-        if(exiftool is None):
-            return False
-
 
         status = ''
-        with ExifTool(executable_=exiftool, addedargs=self.exiftool_addedargs) as et:
+        status = ExifTool().set_tags(tags,source)
-            status = et.set_tags(tags, source)
 
         return status != ''

elodie/tests/filesystem_test.py

@@ -20,9 +20,21 @@ from elodie.media.media import Media
 from elodie.media.photo import Photo
 from elodie.media.video import Video
 from nose.plugins.skip import SkipTest
+from elodie.external.pyexiftool import ExifTool
+from elodie.dependencies import get_exiftool
+from elodie import constants
 
 os.environ['TZ'] = 'GMT'
 
+def setup_module():
+    exiftool_addedargs = [
+            u'-config',
+            u'"{}"'.format(constants.exiftool_config)
+        ]
+    ExifTool(executable_=get_exiftool(), addedargs=exiftool_addedargs).start()
+
+def teardown_module():
+    ExifTool().terminate
 
 def test_create_directory_success():
     filesystem = FileSystem()
@@ -575,7 +587,7 @@ full_path=%year/%month/%location
     path = filesystem.get_folder_path(media.get_metadata())
     if hasattr(load_config, 'config'):
         del load_config.config
-
+        
     assert path == os.path.join('2015','12','Sunnyvale, California'), path
 
 @mock.patch('elodie.config.config_file', '%s/config.ini-location-date' % gettempdir())

elodie/tests/helper.py

@@ -15,6 +15,8 @@ from datetime import datetime
 from datetime import timedelta
 
 from elodie.compatability import _rename
+from elodie.external.pyexiftool import ExifTool
+from elodie.dependencies import get_exiftool
 from elodie import constants
 
 def checksum(file_path, blocksize=65536):
@@ -159,3 +161,14 @@ def restore_dbs():
     # This is no longer needed. See gh-322
     # https://github.com/jmathai/elodie/issues/322
     pass
+
+
+def setup_module():
+    exiftool_addedargs = [
+            u'-config',
+            u'"{}"'.format(constants.exiftool_config)
+        ]
+    ExifTool(executable_=get_exiftool(), addedargs=exiftool_addedargs).start()
+
+def teardown_module():
+    ExifTool().terminate

elodie/tests/media/audio_test.py

@@ -16,9 +16,22 @@ import helper
 from elodie.media.media import Media
 from elodie.media.video import Video
 from elodie.media.audio import Audio
+from elodie.external.pyexiftool import ExifTool
+from elodie.dependencies import get_exiftool
+from elodie import constants
 
 os.environ['TZ'] = 'GMT'
 
+def setup_module():
+    exiftool_addedargs = [
+            u'-config',
+            u'"{}"'.format(constants.exiftool_config)
+        ]
+    ExifTool(executable_=get_exiftool(), addedargs=exiftool_addedargs).start()
+
+def teardown_module():
+    ExifTool().terminate
+
 def test_audio_extensions():
     audio = Audio()
     extensions = audio.extensions

elodie/tests/media/base_test.py

@@ -23,6 +23,8 @@ from elodie.media.video import Video
 
 os.environ['TZ'] = 'GMT'
 
+setup_module = helper.setup_module
+teardown_module = helper.teardown_module
 
 def test_get_all_subclasses():
     subclasses = get_all_subclasses(Base)

elodie/tests/media/media_test.py

@@ -21,6 +21,8 @@ from elodie.media.video import Video
 
 os.environ['TZ'] = 'GMT'
 
+setup_module = helper.setup_module
+teardown_module = helper.teardown_module
 
 def test_get_file_path():
     media = Media(helper.get_file('plain.jpg'))
@@ -86,7 +88,7 @@ def test_get_original_name_invalid_file():
     original_name = media.get_original_name()
 
     assert original_name is None, original_name
-    
+
 def test_set_original_name_when_exists():
     temporary_folder, folder = helper.create_working_folder()
 

elodie/tests/media/photo_test.py

@@ -20,6 +20,9 @@ from elodie.media.photo import Photo
 
 os.environ['TZ'] = 'GMT'
 
+setup_module = helper.setup_module
+teardown_module = helper.teardown_module
+
 def test_photo_extensions():
     photo = Photo()
     extensions = photo.extensions

elodie/tests/plugins/googlephotos/googlephotos_test.py

@@ -30,9 +30,8 @@ config_string_fmt = config_string.format(
     secrets_file
 )
 
-sample_photo = Photo(helper.get_file('plain.jpg'))
+setup_module = helper.setup_module
-sample_metadata = sample_photo.get_metadata()
+teardown_module = helper.teardown_module
-sample_metadata['original_name'] = 'foobar'
 
 @mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-set-session' % gettempdir())
 def test_googlephotos_set_session():
@@ -57,6 +56,9 @@ def test_googlephotos_after_supported():
     if hasattr(load_config, 'config'):
         del load_config.config
 
+    sample_photo = Photo(helper.get_file('plain.jpg'))
+    sample_metadata = sample_photo.get_metadata()
+    sample_metadata['original_name'] = 'foobar'
     final_file_path = helper.get_file('plain.jpg')
     gp = GooglePhotos()
     gp.after('', '', final_file_path, sample_metadata)
@@ -162,6 +164,9 @@ def test_googlephotos_batch():
     if hasattr(load_config, 'config'):
         del load_config.config
 
+    sample_photo = Photo(helper.get_file('plain.jpg'))
+    sample_metadata = sample_photo.get_metadata()
+    sample_metadata['original_name'] = 'foobar'
     final_file_path = helper.get_file('plain.jpg')
     gp = GooglePhotos()
     gp.after('', '', final_file_path, sample_metadata)