From 7c3ea1e1d7a054f0cfe4f7c4801de8caf65d0a45 Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Thu, 4 Jul 2019 02:57:10 -0700 Subject: [PATCH] Create an extendable plugin framework #315 #316 (#318) This commit introduces the initial version of a plugin framework with tests and sample plugins. This helps address #315 and closes #316. --- elodie/config.py | 10 + elodie/filesystem.py | 13 +- elodie/log.py | 9 +- elodie/plugins/__init__.py | 0 elodie/plugins/dummy/__init__.py | 0 elodie/plugins/dummy/dummy.py | 21 ++ elodie/plugins/plugins.py | 85 +++++++++ elodie/plugins/runtimeerror/__init__.py | 0 elodie/plugins/runtimeerror/runtimeerror.py | 21 ++ elodie/plugins/throwerror/__init__.py | 0 elodie/plugins/throwerror/throwerror.py | 20 ++ elodie/tests/config_test.py | 114 ++++++++++- elodie/tests/filesystem_test.py | 52 +++++ elodie/tests/plugins_test.py | 201 ++++++++++++++++++++ requirements-google.txt | 3 + 15 files changed, 540 insertions(+), 9 deletions(-) create mode 100644 elodie/plugins/__init__.py create mode 100644 elodie/plugins/dummy/__init__.py create mode 100644 elodie/plugins/dummy/dummy.py create mode 100644 elodie/plugins/plugins.py create mode 100644 elodie/plugins/runtimeerror/__init__.py create mode 100644 elodie/plugins/runtimeerror/runtimeerror.py create mode 100644 elodie/plugins/throwerror/__init__.py create mode 100644 elodie/plugins/throwerror/throwerror.py create mode 100644 elodie/tests/plugins_test.py create mode 100644 requirements-google.txt diff --git a/elodie/config.py b/elodie/config.py index d54b12d..4bfb6ef 100644 --- a/elodie/config.py +++ b/elodie/config.py @@ -17,3 +17,13 @@ def load_config(): load_config.config = RawConfigParser() load_config.config.read(config_file) return load_config.config + +def load_plugin_config(): + config = load_config() + + # If plugins are defined in the config we return them as a list + # Else we return an empty list + if 'Plugins' in config and 'plugins' in config['Plugins']: + return config['Plugins']['plugins'].split(',') + + return [] diff --git a/elodie/filesystem.py b/elodie/filesystem.py index f8c0155..f243c20 100644 --- a/elodie/filesystem.py +++ b/elodie/filesystem.py @@ -17,6 +17,7 @@ from elodie import log from elodie.config import load_config from elodie.localstorage import Db from elodie.media.base import Base, get_all_subclasses +from elodie.plugins.plugins import Plugins class FileSystem(object): @@ -44,6 +45,10 @@ class FileSystem(object): # https://travis-ci.org/jmathai/elodie/builds/483012902 self.whitespace_regex = '[ \t\n\r\f\v]+' + # Instantiate a plugins object + self.plugins = Plugins() + + def create_directory(self, directory_path): """Create a directory if it does not already exist. @@ -505,7 +510,6 @@ class FileSystem(object): return checksum def process_file(self, _file, destination, media, **kwargs): - move = False if('move' in kwargs): move = kwargs['move'] @@ -526,6 +530,13 @@ class FileSystem(object): _file) return + # Run `before()` for every loaded plugin and if any of them raise an exception + # then we skip importing the file and log a message. + plugins_run_before_status = self.plugins.run_all_before(_file, destination, media) + if(plugins_run_before_status == False): + log.warn('At least one plugin pre-run failed for %s' % _file) + return + media.set_original_name() metadata = media.get_metadata() diff --git a/elodie/log.py b/elodie/log.py index 694a9fb..c1f980b 100644 --- a/elodie/log.py +++ b/elodie/log.py @@ -5,6 +5,8 @@ General file system methods. """ from __future__ import print_function +import sys + from json import dumps from elodie import constants @@ -45,6 +47,11 @@ def error_json(payload): def _print_debug(string): + # Print if debug == True or if running with nosetests + # Commenting out because this causes failures in other tests + # which verify that output is correct. + # Use the line below if you want output printed during tests. + # if(constants.debug is True or 'nose' in sys.modules.keys()): if(constants.debug is True): _print(string) @@ -56,4 +63,4 @@ def _print(s): try: print(c, end='') except UnicodeEncodeError: - print('?', end='') \ No newline at end of file + print('?', end='') diff --git a/elodie/plugins/__init__.py b/elodie/plugins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/elodie/plugins/dummy/__init__.py b/elodie/plugins/dummy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/elodie/plugins/dummy/dummy.py b/elodie/plugins/dummy/dummy.py new file mode 100644 index 0000000..6c5eff1 --- /dev/null +++ b/elodie/plugins/dummy/dummy.py @@ -0,0 +1,21 @@ +""" +Dummy plugin object used for tests. + +.. moduleauthor:: Jaisen Mathai +""" +from __future__ import print_function +from builtins import object + +from elodie.plugins.plugins import PluginBase + +class Dummy(PluginBase): + + __name__ = 'Dummy' + + """A dummy class to execute plugin actions for tests.""" + def __init__(self): + self.before_ran = False + + def before(self, file_path, destination_path, media): + self.before_ran = True + diff --git a/elodie/plugins/plugins.py b/elodie/plugins/plugins.py new file mode 100644 index 0000000..5720355 --- /dev/null +++ b/elodie/plugins/plugins.py @@ -0,0 +1,85 @@ +""" +Plugin object. + +.. moduleauthor:: Jaisen Mathai +""" +from __future__ import print_function +from builtins import object + +from importlib import import_module +from sys import exc_info +from traceback import format_exc + +from elodie.config import load_plugin_config +from elodie import log + +class ElodiePluginError(Exception): + pass + + +class PluginBase(object): + + __name__ = 'PluginBase' + + def log(self, msg): + log.info(msg) + + + +class Plugins(object): + """A class to execute plugin actions.""" + + def __init__(self): + self.plugins = [] + self.classes = {} + self.loaded = False + + def load(self): + """Load plugins from config file. + """ + # If plugins have been loaded then return + if self.loaded == True: + return + + plugin_list = load_plugin_config() + for plugin in plugin_list: + plugin_lower = plugin.lower() + try: + # We attempt to do the following. + # 1. Load the module of the plugin. + # 2. Instantiate an object of the plugin's class. + # 3. Add the plugin to the list of plugins. + # + # #3 should only happen if #2 doesn't throw an error + this_module = import_module('elodie.plugins.{}.{}'.format(plugin_lower, plugin_lower)) + self.classes[plugin] = getattr(this_module, plugin)() + # We only append to self.plugins if we're able to load the class + self.plugins.append(plugin) + except: + log.error('An error occurred initiating plugin {}'.format(plugin)) + log.error(format_exc()) + + self.loaded = True + + + def run_all_before(self, file_path, destination_path, media): + self.load() + """Process `before` methods of each plugin that was loaded. + """ + pass_status = True + for cls in self.classes: + this_method = getattr(self.classes[cls], 'before') + # We try to call the plugin's `before()` method. + # If the method explicitly raises an ElodiePluginError we'll fail the import + # by setting pass_status to False. + # If any other error occurs we log the message and proceed as usual. + # By default, plugins don't change behavior. + try: + this_method(file_path, destination_path, media) + except ElodiePluginError as err: + log.warn('Plugin {} raised an exception: {}'.format(cls, err)) + log.error(format_exc()) + pass_status = False + except: + log.error(format_exc()) + return pass_status diff --git a/elodie/plugins/runtimeerror/__init__.py b/elodie/plugins/runtimeerror/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/elodie/plugins/runtimeerror/runtimeerror.py b/elodie/plugins/runtimeerror/runtimeerror.py new file mode 100644 index 0000000..8468b1c --- /dev/null +++ b/elodie/plugins/runtimeerror/runtimeerror.py @@ -0,0 +1,21 @@ +""" +RuntimeError plugin object used for tests. + +.. moduleauthor:: Jaisen Mathai +""" +from __future__ import print_function +from builtins import object + +from elodie.plugins.plugins import PluginBase + +class RuntimeError(PluginBase): + + __name__ = 'ThrowError' + + """A dummy class to execute plugin actions for tests.""" + def __init__(self): + pass + + def before(self, file_path, destination_path, media): + print(does_not_exist) + diff --git a/elodie/plugins/throwerror/__init__.py b/elodie/plugins/throwerror/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/elodie/plugins/throwerror/throwerror.py b/elodie/plugins/throwerror/throwerror.py new file mode 100644 index 0000000..7b44f1d --- /dev/null +++ b/elodie/plugins/throwerror/throwerror.py @@ -0,0 +1,20 @@ +""" +ThrowError plugin object used for tests. + +.. moduleauthor:: Jaisen Mathai +""" +from __future__ import print_function +from builtins import object + +from elodie.plugins.plugins import PluginBase, ElodiePluginError + +class ThrowError(PluginBase): + + __name__ = 'ThrowError' + + """A dummy class to execute plugin actions for tests.""" + def __init__(self): + pass + + def before(self, file_path, destination_path, media): + raise ElodiePluginError('Sample plugin error') diff --git a/elodie/tests/config_test.py b/elodie/tests/config_test.py index ee073f7..b8c37c2 100644 --- a/elodie/tests/config_test.py +++ b/elodie/tests/config_test.py @@ -5,27 +5,127 @@ import os import sys import unittest + from mock import patch +from tempfile import gettempdir sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) from elodie import constants -from elodie.config import load_config +from elodie.config import load_config, load_plugin_config -BASE_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) - -@patch('elodie.config.config_file', '%s/config.ini-sample' % BASE_PATH) +@patch('elodie.config.config_file', '%s/config.ini-singleton-success' % gettempdir()) def test_load_config_singleton_success(): + with open('%s/config.ini-singleton-success' % gettempdir(), 'w') as f: + f.write(""" +[MapQuest] +key=your-api-key-goes-here +prefer_english_names=False + """) + if hasattr(load_config, 'config'): + del load_config.config + config = load_config() assert config['MapQuest']['key'] == 'your-api-key-goes-here', config.get('MapQuest', 'key') config.set('MapQuest', 'key', 'new-value') config = load_config() + + if hasattr(load_config, 'config'): + del load_config.config + assert config['MapQuest']['key'] == 'new-value', config.get('MapQuest', 'key') - del load_config.config - -@patch('elodie.config.config_file', '%s/config.ini-does-not-exist' % BASE_PATH) +@patch('elodie.config.config_file', '%s/config.ini-does-not-exist' % gettempdir()) def test_load_config_singleton_no_file(): + if hasattr(load_config, 'config'): + del load_config.config + config = load_config() + + if hasattr(load_config, 'config'): + del load_config.config + assert config == {}, config + +@patch('elodie.config.config_file', '%s/config.ini-load-plugin-config-unset-backwards-compat' % gettempdir()) +def test_load_plugin_config_unset_backwards_compat(): + with open('%s/config.ini-load-plugin-config-unset-backwards-compat' % gettempdir(), 'w') as f: + f.write(""" + """) + if hasattr(load_config, 'config'): + del load_config.config + + plugins = load_plugin_config() + + if hasattr(load_config, 'config'): + del load_config.config + + assert plugins == [], plugins + +@patch('elodie.config.config_file', '%s/config.ini-load-plugin-config-exists-not-set' % gettempdir()) +def test_load_plugin_config_exists_not_set(): + with open('%s/config.ini-load-plugin-config-exists-not-set' % gettempdir(), 'w') as f: + f.write(""" +[Plugins] + """) + if hasattr(load_config, 'config'): + del load_config.config + + plugins = load_plugin_config() + + if hasattr(load_config, 'config'): + del load_config.config + + assert plugins == [], plugins + +@patch('elodie.config.config_file', '%s/config.ini-load-plugin-config-one' % gettempdir()) +def test_load_plugin_config_one(): + with open('%s/config.ini-load-plugin-config-one' % gettempdir(), 'w') as f: + f.write(""" +[Plugins] +plugins=Dummy + """) + if hasattr(load_config, 'config'): + del load_config.config + + plugins = load_plugin_config() + + if hasattr(load_config, 'config'): + del load_config.config + + assert plugins == ['Dummy'], plugins + +@patch('elodie.config.config_file', '%s/config.ini-load-plugin-config-one-with-invalid' % gettempdir()) +def test_load_plugin_config_one_with_invalid(): + with open('%s/config.ini-load-plugin-config-one' % gettempdir(), 'w') as f: + f.write(""" +[Plugins] +plugins=DNE + """) + if hasattr(load_config, 'config'): + del load_config.config + + plugins = load_plugin_config() + + if hasattr(load_config, 'config'): + del load_config.config + + assert plugins == [], plugins + +@patch('elodie.config.config_file', '%s/config.ini-load-plugin-config-many' % gettempdir()) +def test_load_plugin_config_many(): + with open('%s/config.ini-load-plugin-config-many' % gettempdir(), 'w') as f: + f.write(""" +[Plugins] +plugins=GooglePhotos,Dummy + """) + if hasattr(load_config, 'config'): + del load_config.config + + plugins = load_plugin_config() + + if hasattr(load_config, 'config'): + del load_config.config + + assert plugins == ['GooglePhotos','Dummy'], plugins diff --git a/elodie/tests/filesystem_test.py b/elodie/tests/filesystem_test.py index 8460aa7..2a16379 100644 --- a/elodie/tests/filesystem_test.py +++ b/elodie/tests/filesystem_test.py @@ -946,6 +946,7 @@ full_path=%year/%month/%day if hasattr(load_config, 'config'): del load_config.config + filesystem = FileSystem() temporary_folder, folder = helper.create_working_folder() @@ -961,9 +962,11 @@ full_path=%year/%month/%day if hasattr(load_config, 'config'): del load_config.config + media_second = Photo(destination) media_second.set_title('foo') destination_second = filesystem.process_file(destination, temporary_folder, media_second, allowDuplicate=True) + if hasattr(load_config, 'config'): del load_config.config @@ -993,6 +996,55 @@ def test_process_existing_file_without_changes(): shutil.rmtree(folder) shutil.rmtree(os.path.dirname(os.path.dirname(destination))) +@mock.patch('elodie.config.config_file', '%s/config.ini-plugin-throw-error' % gettempdir()) +def test_process_file_with_plugin_throw_error(): + with open('%s/config.ini-plugin-throw-error' % gettempdir(), 'w') as f: + f.write(""" +[Plugins] +plugins=ThrowError + """) + + if hasattr(load_config, 'config'): + del load_config.config + + filesystem = FileSystem() + temporary_folder, folder = helper.create_working_folder() + + origin = os.path.join(folder,'plain.jpg') + shutil.copyfile(helper.get_file('plain.jpg'), origin) + + media = Photo(origin) + destination = filesystem.process_file(origin, temporary_folder, media, allowDuplicate=True) + + if hasattr(load_config, 'config'): + del load_config.config + + assert destination is None, destination + +@mock.patch('elodie.config.config_file', '%s/config.ini-plugin-runtime-error' % gettempdir()) +def test_process_file_with_plugin_runtime_error(): + with open('%s/config.ini-plugin-runtime-error' % gettempdir(), 'w') as f: + f.write(""" +[Plugins] +plugins=RuntimeError + """) + if hasattr(load_config, 'config'): + del load_config.config + + filesystem = FileSystem() + temporary_folder, folder = helper.create_working_folder() + + origin = os.path.join(folder,'plain.jpg') + shutil.copyfile(helper.get_file('plain.jpg'), origin) + + media = Photo(origin) + destination = filesystem.process_file(origin, temporary_folder, media, allowDuplicate=True) + + if hasattr(load_config, 'config'): + del load_config.config + + assert '2015-12-Dec/Unknown Location/2015-12-05_00-59-26-plain.jpg' in destination, destination + def test_set_utime_with_exif_date(): filesystem = FileSystem() temporary_folder, folder = helper.create_working_folder() diff --git a/elodie/tests/plugins_test.py b/elodie/tests/plugins_test.py new file mode 100644 index 0000000..15cfd70 --- /dev/null +++ b/elodie/tests/plugins_test.py @@ -0,0 +1,201 @@ +from __future__ import absolute_import +# Project imports +import mock +import os +import sys +from tempfile import gettempdir + +sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) + +from . import helper +from elodie.config import load_config +from elodie.plugins.plugins import Plugins + +@mock.patch('elodie.config.config_file', '%s/config.ini-load-plugins-unset-backwards-compat' % gettempdir()) +def test_load_plugins_unset_backwards_compat(): + with open('%s/config.ini-load-plugins-unset-backwards-compat' % gettempdir(), 'w') as f: + f.write(""" + """) + if hasattr(load_config, 'config'): + del load_config.config + + plugins = Plugins() + plugins.load() + + if hasattr(load_config, 'config'): + del load_config.config + + assert plugins.plugins == [], plugins.plugins + +@mock.patch('elodie.config.config_file', '%s/config.ini-load-plugins-exists-not-set' % gettempdir()) +def test_load_plugins_exists_not_set(): + with open('%s/config.ini-load-plugins-exists-not-set' % gettempdir(), 'w') as f: + f.write(""" +[Plugins] + """) + if hasattr(load_config, 'config'): + del load_config.config + + plugins = Plugins() + plugins.load() + + if hasattr(load_config, 'config'): + del load_config.config + + assert plugins.plugins == [], plugins.plugins + +@mock.patch('elodie.config.config_file', '%s/config.ini-load-plugins-one' % gettempdir()) +def test_load_plugins_one(): + with open('%s/config.ini-load-plugins-one' % gettempdir(), 'w') as f: + f.write(""" +[Plugins] +plugins=Dummy + """) + if hasattr(load_config, 'config'): + del load_config.config + + plugins = Plugins() + plugins.load() + + if hasattr(load_config, 'config'): + del load_config.config + + assert plugins.plugins == ['Dummy'], plugins.plugins + assert len(plugins.classes) == 1, len(plugins.classes) + +@mock.patch('elodie.config.config_file', '%s/config.ini-load-plugins-one-with-invalid' % gettempdir()) +def test_load_plugins_one_with_invalid(): + with open('%s/config.ini-load-plugins-one' % gettempdir(), 'w') as f: + f.write(""" +[Plugins] +plugins=DNE + """) + if hasattr(load_config, 'config'): + del load_config.config + + plugins = Plugins() + plugins.load() + + if hasattr(load_config, 'config'): + del load_config.config + + assert plugins.plugins == [], plugins.plugins + assert len(plugins.classes) == 0, len(plugins.classes) + +@mock.patch('elodie.config.config_file', '%s/config.ini-load-plugins-many' % gettempdir()) +def test_load_plugins_many(): + with open('%s/config.ini-load-plugins-many' % gettempdir(), 'w') as f: + f.write(""" +[Plugins] +plugins=ThrowError,Dummy + """) + if hasattr(load_config, 'config'): + del load_config.config + + plugins = Plugins() + plugins.load() + + if hasattr(load_config, 'config'): + del load_config.config + + assert plugins.plugins == ['ThrowError','Dummy'], plugins.plugins + assert plugins.classes['ThrowError'].__name__ == 'ThrowError', plugins.classes['ThrowError'].__name__ + assert plugins.classes['Dummy'].__name__ == 'Dummy', plugins.classes['Dummy'].__name__ + assert len(plugins.classes) == 2, len(plugins.classes) + +@mock.patch('elodie.config.config_file', '%s/config.ini-load-plugins-many-with-invalid' % gettempdir()) +def test_load_plugins_set_many_with_invalid(): + with open('%s/config.ini-load-plugins-many-with-invalid' % gettempdir(), 'w') as f: + f.write(""" +[Plugins] +plugins=ThrowError,Dummy,DNE + """) + if hasattr(load_config, 'config'): + del load_config.config + + plugins = Plugins() + plugins.load() + + if hasattr(load_config, 'config'): + del load_config.config + + assert plugins.plugins == ['ThrowError','Dummy'], plugins.plugins + +@mock.patch('elodie.config.config_file', '%s/config.ini-run-before' % gettempdir()) +def test_run_before(): + with open('%s/config.ini-run-before' % gettempdir(), 'w') as f: + f.write(""" +[Plugins] +plugins=Dummy + """) + if hasattr(load_config, 'config'): + del load_config.config + + plugins = Plugins() + plugins.load() + before_ran_1 = plugins.classes['Dummy'].before_ran + plugins.run_all_before('', '', '') + before_ran_2 = plugins.classes['Dummy'].before_ran + + if hasattr(load_config, 'config'): + del load_config.config + + assert before_ran_1 == False, before_ran_1 + assert before_ran_2 == True, before_ran_2 + +@mock.patch('elodie.config.config_file', '%s/config.ini-throw-error' % gettempdir()) +def test_throw_error(): + with open('%s/config.ini-throw-error' % gettempdir(), 'w') as f: + f.write(""" +[Plugins] +plugins=ThrowError + """) + if hasattr(load_config, 'config'): + del load_config.config + + plugins = Plugins() + plugins.load() + status = plugins.run_all_before('', '', '') + + if hasattr(load_config, 'config'): + del load_config.config + + assert status == False, status + +@mock.patch('elodie.config.config_file', '%s/config.ini-throw-error-one-of-many' % gettempdir()) +def test_throw_error_one_of_many(): + with open('%s/config.ini-throw-error-one-of-many' % gettempdir(), 'w') as f: + f.write(""" +[Plugins] +plugins=Dummy,ThrowError + """) + if hasattr(load_config, 'config'): + del load_config.config + + plugins = Plugins() + plugins.load() + status = plugins.run_all_before('', '', '') + + if hasattr(load_config, 'config'): + del load_config.config + + assert status == False, status + +@mock.patch('elodie.config.config_file', '%s/config.ini-throw-runtime-error' % gettempdir()) +def test_throw_error_runtime_error(): + with open('%s/config.ini-throw-runtime-error' % gettempdir(), 'w') as f: + f.write(""" +[Plugins] +plugins=RuntimeError + """) + if hasattr(load_config, 'config'): + del load_config.config + + plugins = Plugins() + plugins.load() + status = plugins.run_all_before('', '', '') + + if hasattr(load_config, 'config'): + del load_config.config + + assert status == True, status diff --git a/requirements-google.txt b/requirements-google.txt new file mode 100644 index 0000000..2fb79be --- /dev/null +++ b/requirements-google.txt @@ -0,0 +1,3 @@ +google-api-python-client==1.7.9 +google-auth-oauthlib==0.4.0 +oauth2client==4.1.3