Create an extendable plugin framework #315 #316 (#318)

This commit introduces the initial version of a plugin framework with tests and sample plugins. This helps address #315 and closes #316.

elodie/config.py

@@ -17,3 +17,13 @@ def load_config():
     load_config.config = RawConfigParser()
     load_config.config.read(config_file)
     return load_config.config
+
+def load_plugin_config():
+    config = load_config()
+
+    # If plugins are defined in the config we return them as a list
+    # Else we return an empty list
+    if 'Plugins' in config and 'plugins' in config['Plugins']:
+        return config['Plugins']['plugins'].split(',')
+
+    return []

elodie/filesystem.py

@@ -17,6 +17,7 @@ from elodie import log
 from elodie.config import load_config
 from elodie.localstorage import Db
 from elodie.media.base import Base, get_all_subclasses
+from elodie.plugins.plugins import Plugins
 
 
 class FileSystem(object):
@@ -44,6 +45,10 @@ class FileSystem(object):
         #  https://travis-ci.org/jmathai/elodie/builds/483012902
         self.whitespace_regex = '[ \t\n\r\f\v]+'
 
+        # Instantiate a plugins object
+        self.plugins = Plugins()
+
+
     def create_directory(self, directory_path):
         """Create a directory if it does not already exist.
 
@@ -505,7 +510,6 @@ class FileSystem(object):
         return checksum
 
     def process_file(self, _file, destination, media, **kwargs):
-
         move = False
         if('move' in kwargs):
             move = kwargs['move']
@@ -526,6 +530,13 @@ class FileSystem(object):
                      _file)
             return
 
+        # Run `before()` for every loaded plugin and if any of them raise an exception
+        #  then we skip importing the file and log a message.
+        plugins_run_before_status = self.plugins.run_all_before(_file, destination, media)
+        if(plugins_run_before_status == False):
+            log.warn('At least one plugin pre-run failed for %s' % _file)
+            return
+
         media.set_original_name()
         metadata = media.get_metadata()
 

elodie/log.py

@@ -5,6 +5,8 @@ General file system methods.
 """
 from __future__ import print_function
 
+import sys
+
 from json import dumps
 
 from elodie import constants
@@ -45,6 +47,11 @@ def error_json(payload):
 
 
 def _print_debug(string):
+    # Print if debug == True or if running with nosetests
+    # Commenting out because this causes failures in other tests
+    #  which verify that output is correct.
+    # Use the line below if you want output printed during tests.
+    # if(constants.debug is True or 'nose' in sys.modules.keys()):
     if(constants.debug is True):
         _print(string)
 
@@ -56,4 +63,4 @@ def _print(s):
             try:
                 print(c, end='')
             except UnicodeEncodeError:
-                print('?', end='')
+                print('?', end='')

elodie/plugins/dummy/dummy.py

@@ -0,0 +1,21 @@
+"""
+Dummy plugin object used for tests.
+
+.. moduleauthor:: Jaisen Mathai <jaisen@jmathai.com>
+"""
+from __future__ import print_function
+from builtins import object
+
+from elodie.plugins.plugins import PluginBase
+
+class Dummy(PluginBase):
+
+    __name__ = 'Dummy'
+
+    """A dummy class to execute plugin actions for tests."""
+    def __init__(self):
+        self.before_ran = False
+
+    def before(self, file_path, destination_path, media):
+        self.before_ran = True
+

elodie/plugins/plugins.py

@@ -0,0 +1,85 @@
+"""
+Plugin object.
+
+.. moduleauthor:: Jaisen Mathai <jaisen@jmathai.com>
+"""
+from __future__ import print_function
+from builtins import object
+
+from importlib import import_module
+from sys import exc_info
+from traceback import format_exc
+
+from elodie.config import load_plugin_config
+from elodie import log
+
+class ElodiePluginError(Exception):
+    pass
+
+
+class PluginBase(object):
+
+    __name__ = 'PluginBase'
+
+    def log(self, msg):
+        log.info(msg)
+
+
+
+class Plugins(object):
+    """A class to execute plugin actions."""
+
+    def __init__(self):
+        self.plugins = []
+        self.classes = {}
+        self.loaded = False
+
+    def load(self):
+        """Load plugins from config file.
+        """
+        # If plugins have been loaded then return
+        if self.loaded == True:
+            return
+
+        plugin_list = load_plugin_config()
+        for plugin in plugin_list:
+            plugin_lower = plugin.lower()
+            try:
+                # We attempt to do the following.
+                #  1. Load the module of the plugin.
+                #  2. Instantiate an object of the plugin's class.
+                #  3. Add the plugin to the list of plugins.
+                #  
+                #  #3 should only happen if #2 doesn't throw an error
+                this_module = import_module('elodie.plugins.{}.{}'.format(plugin_lower, plugin_lower))
+                self.classes[plugin] = getattr(this_module, plugin)()
+                # We only append to self.plugins if we're able to load the class
+                self.plugins.append(plugin)
+            except:
+                log.error('An error occurred initiating plugin {}'.format(plugin))
+                log.error(format_exc())
+
+        self.loaded = True
+
+
+    def run_all_before(self, file_path, destination_path, media):
+        self.load()
+        """Process `before` methods of each plugin that was loaded.
+        """
+        pass_status = True
+        for cls in self.classes:
+            this_method = getattr(self.classes[cls], 'before')
+            # We try to call the plugin's `before()` method.
+            # If the method explicitly raises an ElodiePluginError we'll fail the import
+            #  by setting pass_status to False.
+            # If any other error occurs we log the message and proceed as usual.
+            # By default, plugins don't change behavior.
+            try:
+                this_method(file_path, destination_path, media)
+            except ElodiePluginError as err:
+                log.warn('Plugin {} raised an exception: {}'.format(cls, err))
+                log.error(format_exc())
+                pass_status = False
+            except:
+                log.error(format_exc())
+        return pass_status

elodie/plugins/runtimeerror/runtimeerror.py

@@ -0,0 +1,21 @@
+"""
+RuntimeError plugin object used for tests.
+
+.. moduleauthor:: Jaisen Mathai <jaisen@jmathai.com>
+"""
+from __future__ import print_function
+from builtins import object
+
+from elodie.plugins.plugins import PluginBase
+
+class RuntimeError(PluginBase):
+
+    __name__ = 'ThrowError'
+
+    """A dummy class to execute plugin actions for tests."""
+    def __init__(self):
+        pass
+
+    def before(self, file_path, destination_path, media):
+        print(does_not_exist)
+

elodie/plugins/throwerror/throwerror.py

@@ -0,0 +1,20 @@
+"""
+ThrowError plugin object used for tests.
+
+.. moduleauthor:: Jaisen Mathai <jaisen@jmathai.com>
+"""
+from __future__ import print_function
+from builtins import object
+
+from elodie.plugins.plugins import PluginBase, ElodiePluginError
+
+class ThrowError(PluginBase):
+
+    __name__ = 'ThrowError'
+
+    """A dummy class to execute plugin actions for tests."""
+    def __init__(self):
+        pass
+
+    def before(self, file_path, destination_path, media):
+        raise ElodiePluginError('Sample plugin error')

elodie/tests/config_test.py

@@ -5,27 +5,127 @@ import os
 import sys
 import unittest 
 
+
 from mock import patch
+from tempfile import gettempdir
 
 sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))
 
 from elodie import constants
-from elodie.config import load_config
+from elodie.config import load_config, load_plugin_config
 
-BASE_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
-
-@patch('elodie.config.config_file', '%s/config.ini-sample' % BASE_PATH)
+@patch('elodie.config.config_file', '%s/config.ini-singleton-success' % gettempdir())
 def test_load_config_singleton_success():
+    with open('%s/config.ini-singleton-success' % gettempdir(), 'w') as f:
+        f.write("""
+[MapQuest]
+key=your-api-key-goes-here
+prefer_english_names=False
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
     config = load_config()
     assert config['MapQuest']['key'] == 'your-api-key-goes-here', config.get('MapQuest', 'key')
     config.set('MapQuest', 'key', 'new-value')
 
     config = load_config()
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
     assert config['MapQuest']['key'] == 'new-value', config.get('MapQuest', 'key')
 
-    del load_config.config
-
-@patch('elodie.config.config_file', '%s/config.ini-does-not-exist' % BASE_PATH)
+@patch('elodie.config.config_file', '%s/config.ini-does-not-exist' % gettempdir())
 def test_load_config_singleton_no_file():
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
     config = load_config()
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
     assert config == {}, config
+
+@patch('elodie.config.config_file', '%s/config.ini-load-plugin-config-unset-backwards-compat' % gettempdir())
+def test_load_plugin_config_unset_backwards_compat():
+    with open('%s/config.ini-load-plugin-config-unset-backwards-compat' % gettempdir(), 'w') as f:
+        f.write("""
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    plugins = load_plugin_config()
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert plugins == [], plugins
+
+@patch('elodie.config.config_file', '%s/config.ini-load-plugin-config-exists-not-set' % gettempdir())
+def test_load_plugin_config_exists_not_set():
+    with open('%s/config.ini-load-plugin-config-exists-not-set' % gettempdir(), 'w') as f:
+        f.write("""
+[Plugins]
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    plugins = load_plugin_config()
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert plugins == [], plugins
+
+@patch('elodie.config.config_file', '%s/config.ini-load-plugin-config-one' % gettempdir())
+def test_load_plugin_config_one():
+    with open('%s/config.ini-load-plugin-config-one' % gettempdir(), 'w') as f:
+        f.write("""
+[Plugins]
+plugins=Dummy
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    plugins = load_plugin_config()
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert plugins == ['Dummy'], plugins
+
+@patch('elodie.config.config_file', '%s/config.ini-load-plugin-config-one-with-invalid' % gettempdir())
+def test_load_plugin_config_one_with_invalid():
+    with open('%s/config.ini-load-plugin-config-one' % gettempdir(), 'w') as f:
+        f.write("""
+[Plugins]
+plugins=DNE
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    plugins = load_plugin_config()
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert plugins == [], plugins
+
+@patch('elodie.config.config_file', '%s/config.ini-load-plugin-config-many' % gettempdir())
+def test_load_plugin_config_many():
+    with open('%s/config.ini-load-plugin-config-many' % gettempdir(), 'w') as f:
+        f.write("""
+[Plugins]
+plugins=GooglePhotos,Dummy
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    plugins = load_plugin_config()
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert plugins == ['GooglePhotos','Dummy'], plugins

elodie/tests/filesystem_test.py

@@ -946,6 +946,7 @@ full_path=%year/%month/%day
 
     if hasattr(load_config, 'config'):
         del load_config.config
+
     filesystem = FileSystem()
     temporary_folder, folder = helper.create_working_folder()
 
@@ -961,9 +962,11 @@ full_path=%year/%month/%day
 
     if hasattr(load_config, 'config'):
         del load_config.config
+
     media_second = Photo(destination)
     media_second.set_title('foo')
     destination_second = filesystem.process_file(destination, temporary_folder, media_second, allowDuplicate=True)
+
     if hasattr(load_config, 'config'):
         del load_config.config
 
@@ -993,6 +996,55 @@ def test_process_existing_file_without_changes():
     shutil.rmtree(folder)
     shutil.rmtree(os.path.dirname(os.path.dirname(destination)))
 
+@mock.patch('elodie.config.config_file', '%s/config.ini-plugin-throw-error' % gettempdir())
+def test_process_file_with_plugin_throw_error():
+    with open('%s/config.ini-plugin-throw-error' % gettempdir(), 'w') as f:
+        f.write("""
+[Plugins]
+plugins=ThrowError
+        """)
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    filesystem = FileSystem()
+    temporary_folder, folder = helper.create_working_folder()
+
+    origin = os.path.join(folder,'plain.jpg')
+    shutil.copyfile(helper.get_file('plain.jpg'), origin)
+
+    media = Photo(origin)
+    destination = filesystem.process_file(origin, temporary_folder, media, allowDuplicate=True)
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert destination is None, destination
+
+@mock.patch('elodie.config.config_file', '%s/config.ini-plugin-runtime-error' % gettempdir())
+def test_process_file_with_plugin_runtime_error():
+    with open('%s/config.ini-plugin-runtime-error' % gettempdir(), 'w') as f:
+        f.write("""
+[Plugins]
+plugins=RuntimeError
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    filesystem = FileSystem()
+    temporary_folder, folder = helper.create_working_folder()
+
+    origin = os.path.join(folder,'plain.jpg')
+    shutil.copyfile(helper.get_file('plain.jpg'), origin)
+
+    media = Photo(origin)
+    destination = filesystem.process_file(origin, temporary_folder, media, allowDuplicate=True)
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert '2015-12-Dec/Unknown Location/2015-12-05_00-59-26-plain.jpg' in destination, destination
+
 def test_set_utime_with_exif_date():
     filesystem = FileSystem()
     temporary_folder, folder = helper.create_working_folder()

elodie/tests/plugins_test.py

@@ -0,0 +1,201 @@
+from __future__ import absolute_import
+# Project imports
+import mock
+import os
+import sys
+from tempfile import gettempdir
+
+sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))
+
+from . import helper
+from elodie.config import load_config
+from elodie.plugins.plugins import Plugins
+
+@mock.patch('elodie.config.config_file', '%s/config.ini-load-plugins-unset-backwards-compat' % gettempdir())
+def test_load_plugins_unset_backwards_compat():
+    with open('%s/config.ini-load-plugins-unset-backwards-compat' % gettempdir(), 'w') as f:
+        f.write("""
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    plugins = Plugins()
+    plugins.load()
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert plugins.plugins == [], plugins.plugins
+
+@mock.patch('elodie.config.config_file', '%s/config.ini-load-plugins-exists-not-set' % gettempdir())
+def test_load_plugins_exists_not_set():
+    with open('%s/config.ini-load-plugins-exists-not-set' % gettempdir(), 'w') as f:
+        f.write("""
+[Plugins]
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    plugins = Plugins()
+    plugins.load()
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert plugins.plugins == [], plugins.plugins
+
+@mock.patch('elodie.config.config_file', '%s/config.ini-load-plugins-one' % gettempdir())
+def test_load_plugins_one():
+    with open('%s/config.ini-load-plugins-one' % gettempdir(), 'w') as f:
+        f.write("""
+[Plugins]
+plugins=Dummy
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    plugins = Plugins()
+    plugins.load()
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert plugins.plugins == ['Dummy'], plugins.plugins
+    assert len(plugins.classes) == 1, len(plugins.classes)
+
+@mock.patch('elodie.config.config_file', '%s/config.ini-load-plugins-one-with-invalid' % gettempdir())
+def test_load_plugins_one_with_invalid():
+    with open('%s/config.ini-load-plugins-one' % gettempdir(), 'w') as f:
+        f.write("""
+[Plugins]
+plugins=DNE
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    plugins = Plugins()
+    plugins.load()
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert plugins.plugins == [], plugins.plugins
+    assert len(plugins.classes) == 0, len(plugins.classes)
+
+@mock.patch('elodie.config.config_file', '%s/config.ini-load-plugins-many' % gettempdir())
+def test_load_plugins_many():
+    with open('%s/config.ini-load-plugins-many' % gettempdir(), 'w') as f:
+        f.write("""
+[Plugins]
+plugins=ThrowError,Dummy
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    plugins = Plugins()
+    plugins.load()
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert plugins.plugins == ['ThrowError','Dummy'], plugins.plugins
+    assert plugins.classes['ThrowError'].__name__ == 'ThrowError', plugins.classes['ThrowError'].__name__
+    assert plugins.classes['Dummy'].__name__ == 'Dummy', plugins.classes['Dummy'].__name__
+    assert len(plugins.classes) == 2, len(plugins.classes)
+
+@mock.patch('elodie.config.config_file', '%s/config.ini-load-plugins-many-with-invalid' % gettempdir())
+def test_load_plugins_set_many_with_invalid():
+    with open('%s/config.ini-load-plugins-many-with-invalid' % gettempdir(), 'w') as f:
+        f.write("""
+[Plugins]
+plugins=ThrowError,Dummy,DNE
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    plugins = Plugins()
+    plugins.load()
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert plugins.plugins == ['ThrowError','Dummy'], plugins.plugins
+
+@mock.patch('elodie.config.config_file', '%s/config.ini-run-before' % gettempdir())
+def test_run_before():
+    with open('%s/config.ini-run-before' % gettempdir(), 'w') as f:
+        f.write("""
+[Plugins]
+plugins=Dummy
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    plugins = Plugins()
+    plugins.load()
+    before_ran_1 = plugins.classes['Dummy'].before_ran
+    plugins.run_all_before('', '', '')
+    before_ran_2 = plugins.classes['Dummy'].before_ran
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert before_ran_1 == False, before_ran_1
+    assert before_ran_2 == True, before_ran_2
+
+@mock.patch('elodie.config.config_file', '%s/config.ini-throw-error' % gettempdir())
+def test_throw_error():
+    with open('%s/config.ini-throw-error' % gettempdir(), 'w') as f:
+        f.write("""
+[Plugins]
+plugins=ThrowError
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    plugins = Plugins()
+    plugins.load()
+    status = plugins.run_all_before('', '', '')
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert status == False, status
+
+@mock.patch('elodie.config.config_file', '%s/config.ini-throw-error-one-of-many' % gettempdir())
+def test_throw_error_one_of_many():
+    with open('%s/config.ini-throw-error-one-of-many' % gettempdir(), 'w') as f:
+        f.write("""
+[Plugins]
+plugins=Dummy,ThrowError
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    plugins = Plugins()
+    plugins.load()
+    status = plugins.run_all_before('', '', '')
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert status == False, status
+
+@mock.patch('elodie.config.config_file', '%s/config.ini-throw-runtime-error' % gettempdir())
+def test_throw_error_runtime_error():
+    with open('%s/config.ini-throw-runtime-error' % gettempdir(), 'w') as f:
+        f.write("""
+[Plugins]
+plugins=RuntimeError
+        """)
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    plugins = Plugins()
+    plugins.load()
+    status = plugins.run_all_before('', '', '')
+
+    if hasattr(load_config, 'config'):
+        del load_config.config
+
+    assert status == True, status

requirements-google.txt

@@ -0,0 +1,3 @@
+google-api-python-client==1.7.9
+google-auth-oauthlib==0.4.0
+oauth2client==4.1.3