Add a plugin to upload photos to Google Photos (#319)
Fixes #315. This PR aims to address the [recent changes](https://www.blog.google/products/photos/simplifying-google-photos-and-google-drive/) in Google Photos + Google Drive where syncing between the two is no longer supported. It works by uploading photos as part of the import process to add a copy of every photo in your library to Google Photos. Google Drive is not required for this plugin to work. This plugin lets you have all your photos in Google Photos without relying on Google Drive. You can use another cloud storage service like iCloud or Dropbox or no cloud storage at all. - [x] Add tests for `after()` plugin methods. - [x] Add support for storage/async support. - [x] Include plugins into code coverage. - [x] Sweep code and clean up and add comments.
This commit is contained in:
Jaisen Mathai
parent
9e42edfa76
commit
12c17c9cac
|
|
@ -10,6 +10,7 @@ before_install:
|
|||
- "sudo apt-get install python-dev python-pip -y"
|
||||
install:
|
||||
- "pip install -r elodie/tests/requirements.txt"
|
||||
- "pip install -r elodie/plugins/googlephotos/requirements.txt"
|
||||
- "pip install coveralls"
|
||||
before_script:
|
||||
- "mkdir ~/.elodie"
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ You can configure how Elodie names your files using placeholders. This works sim
|
|||
|
||||
If you'd like to specify your own naming convention it's recommended you include something that's mostly unique like the time including seconds. You'll need to include a `[File]` section in your `config.ini` file with a name attribute. If a placeholder doesn't have a value then it plus any preceding characters which are not alphabetic are removed.
|
||||
|
||||
By default the resulting filename is all lowercased. To change this behavior to upppercasing add capitalization=upper.
|
||||
By default the resulting filename is all lowercased. To change this behavior to uppercasing add capitalization=upper.
|
||||
|
||||
```
|
||||
[File]
|
||||
|
|
@ -270,7 +270,7 @@ name=%date-%original_name-%title.jpg
|
|||
|
||||
date=%Y-%m-%b-%H-%M-%S
|
||||
name=%date-%original_name-%album.jpg
|
||||
capitalization=uppper
|
||||
capitalization=upper
|
||||
# -> 2012-05-MAR-12-59-30-DSC_1234-MY-ALBUM.JPG
|
||||
```
|
||||
|
||||
|
|
|
|||
12
elodie.py
12
elodie.py
|
|
@ -27,6 +27,7 @@ from elodie.media.text import Text
|
|||
from elodie.media.audio import Audio
|
||||
from elodie.media.photo import Photo
|
||||
from elodie.media.video import Video
|
||||
from elodie.plugins.plugins import Plugins
|
||||
from elodie.result import Result
|
||||
|
||||
|
||||
|
|
@ -70,6 +71,16 @@ def import_file(_file, destination, album_from_folder, trash, allow_duplicates):
|
|||
|
||||
return dest_path or None
|
||||
|
||||
@click.command('batch')
|
||||
@click.option('--debug', default=False, is_flag=True,
|
||||
help='Override the value in constants.py with True.')
|
||||
def _batch(debug):
|
||||
"""Run batch() for all plugins.
|
||||
"""
|
||||
constants.debug = debug
|
||||
plugins = Plugins()
|
||||
plugins.run_batch()
|
||||
|
||||
|
||||
@click.command('import')
|
||||
@click.option('--destination', type=click.Path(file_okay=False),
|
||||
|
|
@ -340,6 +351,7 @@ main.add_command(_import)
|
|||
main.add_command(_update)
|
||||
main.add_command(_generate_db)
|
||||
main.add_command(_verify)
|
||||
main.add_command(_batch)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -24,6 +24,11 @@ def _decode(string, encoding=sys.getfilesystemencoding()):
|
|||
|
||||
return string
|
||||
|
||||
def _bytes(string):
|
||||
if constants.python_version == 3:
|
||||
return bytes(string, 'utf8')
|
||||
else:
|
||||
return bytes(string)
|
||||
|
||||
def _copyfile(src, dst):
|
||||
# shutil.copy seems slow, changing to streaming according to
|
||||
|
|
|
|||
|
|
@ -27,3 +27,13 @@ def load_plugin_config():
|
|||
return config['Plugins']['plugins'].split(',')
|
||||
|
||||
return []
|
||||
|
||||
def load_config_for_plugin(name):
|
||||
# Plugins store data using Plugin%PluginName% format.
|
||||
key = 'Plugin{}'.format(name)
|
||||
config = load_config()
|
||||
|
||||
if key in config:
|
||||
return config[key]
|
||||
|
||||
return {}
|
||||
|
|
|
|||
|
|
@ -532,7 +532,7 @@ class FileSystem(object):
|
|||
|
||||
# Run `before()` for every loaded plugin and if any of them raise an exception
|
||||
# then we skip importing the file and log a message.
|
||||
plugins_run_before_status = self.plugins.run_all_before(_file, destination, media)
|
||||
plugins_run_before_status = self.plugins.run_all_before(_file, destination)
|
||||
if(plugins_run_before_status == False):
|
||||
log.warn('At least one plugin pre-run failed for %s' % _file)
|
||||
return
|
||||
|
|
@ -594,6 +594,14 @@ class FileSystem(object):
|
|||
db.add_hash(checksum, dest_path)
|
||||
db.update_hash_db()
|
||||
|
||||
# Run `after()` for every loaded plugin and if any of them raise an exception
|
||||
# then we skip importing the file and log a message.
|
||||
plugins_run_after_status = self.plugins.run_all_after(_file, destination, dest_path, metadata)
|
||||
if(plugins_run_after_status == False):
|
||||
log.warn('At least one plugin pre-run failed for %s' % _file)
|
||||
return
|
||||
|
||||
|
||||
return dest_path
|
||||
|
||||
def set_utime_from_metadata(self, metadata, file_path):
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ Dummy plugin object used for tests.
|
|||
.. moduleauthor:: Jaisen Mathai <jaisen@jmathai.com>
|
||||
"""
|
||||
from __future__ import print_function
|
||||
from builtins import object
|
||||
|
||||
from elodie.plugins.plugins import PluginBase
|
||||
|
||||
|
|
@ -16,6 +15,6 @@ class Dummy(PluginBase):
|
|||
def __init__(self):
|
||||
self.before_ran = False
|
||||
|
||||
def before(self, file_path, destination_path, media):
|
||||
def before(self, file_path, destination_folder):
|
||||
self.before_ran = True
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,63 @@
|
|||
# Google Photos Plugin for Elodie
|
||||
|
||||
[](https://travis-ci.org/jmathai/elodie) [](https://coveralls.io/github/jmathai/elodie?branch=master) [](https://scrutinizer-ci.com/g/jmathai/elodie/?branch=master)
|
||||
|
||||
This plugin uploads all photos imported using Elodie to Google Photos. It was created after [Google Photos and Google Drive synchronization was deprecated](https://www.blog.google/products/photos/simplifying-google-photos-and-google-drive/). It aims to replicate my [workflow using Google Photos, Google Drive and Elodie](https://artplusmarketing.com/one-year-of-using-an-automated-photo-organization-and-archiving-workflow-89cf9ad7bddf).
|
||||
|
||||
I didn't intend on it, but it turned out that with this plugin you can use Google Photos with Google Drive, iCloud Drive, Dropbox or no cloud storage service while still using Google Photos for viewing and experiencing your photo library.
|
||||
|
||||
The hardest part of using this plugin is setting it up. Let's get started.
|
||||
|
||||
# Installation and Setup
|
||||
|
||||
## Google Photos
|
||||
Let's start by making sure you have a Google Photos account. If you don't, you should start by [creating your Google Photos account](https://photos.google.com/login).
|
||||
|
||||
## Google APIs
|
||||
Once you've got your Google Photos account created we can enable Google Photos' APIs for your account.
|
||||
|
||||
In order to enable Google APIs you need what's called a project. Don't worry about what it is, just create one so you can enable the Google Photos API for it.
|
||||
1. Go to [Google's developer console](https://console.developers.google.com).
|
||||
2. If you have a project already then you can skip this step.
|
||||
|
||||
If you don't already have a project or would like to create one just for this purpose then you should create it now. In the top bar there's a **project selector** which will open a dialog with a button to create a new project.
|
||||
3. Now you'll need to [enable the Google Photos API for your project](https://console.developers.google.com/apis/library/photoslibrary.googleapis.com). You should be able to follow that link and click the **Enable API** button. Make sure the project from the prior step is selected.
|
||||
4. Once you've enabled the Google Photos API you will need to [create an OAuth client ID](https://console.developers.google.com/apis/credentials).
|
||||
1. Select **other** as the type of client.
|
||||
2. Set up a consent screen if needed. Only you'll be seeing this so put whatever you want into the required fields. Most everything can be left blank.
|
||||
3. Download the credentials when prompted or click the download icon on the [credentials page](https://console.developers.google.com/apis/credentials).
|
||||
|
||||
## Configure the Google Photos Plugin for Elodie
|
||||
Now that you're set up with your Google Photos account, have enabled the APIs and configured your OAuth client we're ready to enable this plugin for Elodie.
|
||||
|
||||
1. Move the credentials file you downloaded to a permanent location and update your `config.ini` file. You'll need to add a `[Plugins]` section.
|
||||
|
||||
[Plugins]
|
||||
plugins=GooglePhotos
|
||||
|
||||
[PluginGooglePhotos]
|
||||
secrets_file=/full/path/to/saved/secrets_file.json
|
||||
auth_file=/full/path/to/save/auth_file.json
|
||||
|
||||
I put `secrets_file.json` (the one you downloaded) in my `~/.elodie` directory. `auth_file.json` will be automatically created so make sure the path is writable by the user running `./elodie.py`.
|
||||
2. If you did everything exactly correct you should be able to authenticate Elodie to start uploading to Google Photos.
|
||||
1. Start by importing a new photo by running `./elodie.py import`.
|
||||
2. Run `./elodie.py batch` which should open your browser.
|
||||
3. Login and tell Google Photos to allow Elodie the requested permissions to your Google Photos account.
|
||||
4. At some point you'll likely see a scary warning screen. This is because your OAuth client is not approved but go ahead and click on **Advanced** and **Go to {Your OAuth client name (unsafe)**.
|
||||
5. Return to your terminal and close your browser tab if you'd like.
|
||||
|
||||
Assuming you did not see any errors you can go back to your browser and load up Google Photos. If your photos show up in Google Photos then you got everything to work *a lot* easier than I did.
|
||||
|
||||
## Automating It All
|
||||
I'm not going to go into how you can automate this process but much of it is covered by various blog posts I've done in the past.
|
||||
|
||||
* [Understanding My Need for an Automated Photo Workflow](https://medium.com/vantage/understanding-my-need-for-an-automated-photo-workflow-a2ff95b46f8f#.dmwyjlc57)
|
||||
* [Introducing Elodie; Your Personal EXIF-based Photo and Video Assistant](https://medium.com/@jmathai/introducing-elodie-your-personal-exif-based-photo-and-video-assistant-d92868f302ec)
|
||||
* [My Automated Photo Workflow using Google Photos and Elodie](https://medium.com/swlh/my-automated-photo-workflow-using-google-photos-and-elodie-afb753b8c724)
|
||||
* [One Year of Using an Automated Photo Organization and Archiving Workflow](https://artplusmarketing.com/one-year-of-using-an-automated-photo-organization-and-archiving-workflow-89cf9ad7bddf)
|
||||
|
||||
## Credits
|
||||
Elodie is an open source project with many [contributors](https://github.com/jmathai/elodie/graphs/contributors) and [users](https://github.com/jmathai/elodie/stargazers) who have reported lots of [bugs and feature requests](https://github.com/jmathai/elodie/issues?utf8=%E2%9C%93&q=).
|
||||
|
||||
Google Photos is an amazing product. Kudos to the team for making it so magical.
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
"""
|
||||
Google Photos plugin object.
|
||||
This plugin will queue imported photos into the plugin's database file.
|
||||
Using this plugin should have no impact on performance of importing photos.
|
||||
|
||||
In order to upload the photos to Google Photos you need to run the following command.
|
||||
|
||||
```
|
||||
./elodie.py batch
|
||||
```
|
||||
|
||||
That command will execute the batch() method on all plugins, including this one.
|
||||
This plugin's batch() function reads all files from the database file and attempts to
|
||||
upload them to Google Photos.
|
||||
This plugin does not aim to keep Google Photos in sync.
|
||||
Once a photo is uploaded it's removed from the database and no records are kept thereafter.
|
||||
|
||||
Upload code adapted from https://github.com/eshmu/gphotos-upload
|
||||
|
||||
.. moduleauthor:: Jaisen Mathai <jaisen@jmathai.com>
|
||||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
import json
|
||||
|
||||
from os.path import basename, isfile
|
||||
|
||||
from google_auth_oauthlib.flow import InstalledAppFlow
|
||||
from google.auth.transport.requests import AuthorizedSession
|
||||
from google.oauth2.credentials import Credentials
|
||||
|
||||
from elodie.media.photo import Photo
|
||||
from elodie.media.video import Video
|
||||
from elodie.plugins.plugins import PluginBase
|
||||
|
||||
class GooglePhotos(PluginBase):
|
||||
"""A class to execute plugin actions.
|
||||
|
||||
Requires a config file with the following configurations set.
|
||||
secrets_file:
|
||||
The full file path where to find the downloaded secrets.
|
||||
auth_file:
|
||||
The full file path where to store authenticated tokens.
|
||||
|
||||
"""
|
||||
|
||||
__name__ = 'GooglePhotos'
|
||||
|
||||
def __init__(self):
|
||||
super(GooglePhotos, self).__init__()
|
||||
self.upload_url = 'https://photoslibrary.googleapis.com/v1/uploads'
|
||||
self.media_create_url = 'https://photoslibrary.googleapis.com/v1/mediaItems:batchCreate'
|
||||
self.scopes = ['https://www.googleapis.com/auth/photoslibrary.appendonly']
|
||||
|
||||
self.secrets_file = None
|
||||
if('secrets_file' in self.config_for_plugin):
|
||||
self.secrets_file = self.config_for_plugin['secrets_file']
|
||||
# 'client_id.json'
|
||||
self.auth_file = None
|
||||
if('auth_file' in self.config_for_plugin):
|
||||
self.auth_file = self.config_for_plugin['auth_file']
|
||||
self.session = None
|
||||
|
||||
def after(self, file_path, destination_folder, final_file_path, metadata):
|
||||
extension = metadata['extension']
|
||||
if(extension in Photo.extensions or extension in Video.extensions):
|
||||
self.log(u'Added {} to db.'.format(final_file_path))
|
||||
self.db.set(final_file_path, metadata['original_name'])
|
||||
else:
|
||||
self.log(u'Skipping {} which is not a supported media type.'.format(final_file_path))
|
||||
|
||||
def batch(self):
|
||||
queue = self.db.get_all()
|
||||
status = True
|
||||
count = 0
|
||||
for key in queue:
|
||||
this_status = self.upload(key)
|
||||
if(this_status):
|
||||
# Remove from queue if successful then increment count
|
||||
self.db.delete(key)
|
||||
count = count + 1
|
||||
self.display('{} uploaded successfully.'.format(key))
|
||||
else:
|
||||
status = False
|
||||
self.display('{} failed to upload.'.format(key))
|
||||
return (status, count)
|
||||
|
||||
def before(self, file_path, destination_folder):
|
||||
pass
|
||||
|
||||
def set_session(self):
|
||||
# Try to load credentials from an auth file.
|
||||
# If it doesn't exist or is not valid then catch the
|
||||
# exception and reauthenticate.
|
||||
try:
|
||||
creds = Credentials.from_authorized_user_file(self.auth_file, self.scopes)
|
||||
except:
|
||||
try:
|
||||
flow = InstalledAppFlow.from_client_secrets_file(self.secrets_file, self.scopes)
|
||||
creds = flow.run_local_server()
|
||||
cred_dict = {
|
||||
'token': creds.token,
|
||||
'refresh_token': creds.refresh_token,
|
||||
'id_token': creds.id_token,
|
||||
'scopes': creds.scopes,
|
||||
'token_uri': creds.token_uri,
|
||||
'client_id': creds.client_id,
|
||||
'client_secret': creds.client_secret
|
||||
}
|
||||
|
||||
# Store the returned authentication tokens to the auth_file.
|
||||
with open(self.auth_file, 'w') as f:
|
||||
f.write(json.dumps(cred_dict))
|
||||
except:
|
||||
return
|
||||
|
||||
self.session = AuthorizedSession(creds)
|
||||
self.session.headers["Content-type"] = "application/octet-stream"
|
||||
self.session.headers["X-Goog-Upload-Protocol"] = "raw"
|
||||
|
||||
def upload(self, path_to_photo):
|
||||
self.set_session()
|
||||
if(self.session is None):
|
||||
self.log('Could not initialize session')
|
||||
return None
|
||||
|
||||
self.session.headers["X-Goog-Upload-File-Name"] = basename(path_to_photo)
|
||||
if(not isfile(path_to_photo)):
|
||||
self.log('Could not find file: {}'.format(path_to_photo))
|
||||
return None
|
||||
|
||||
with open(path_to_photo, 'rb') as f:
|
||||
photo_bytes = f.read()
|
||||
|
||||
upload_token = self.session.post(self.upload_url, photo_bytes)
|
||||
if(upload_token.status_code != 200 or not upload_token.content):
|
||||
self.log('Uploading media failed: ({}) {}'.format(upload_token.status_code, upload_token.content))
|
||||
return None
|
||||
|
||||
create_body = json.dumps({'newMediaItems':[{'description':'','simpleMediaItem':{'uploadToken':upload_token.content.decode()}}]}, indent=4)
|
||||
resp = self.session.post(self.media_create_url, create_body).json()
|
||||
if(
|
||||
'newMediaItemResults' not in resp or
|
||||
'status' not in resp['newMediaItemResults'][0] or
|
||||
'message' not in resp['newMediaItemResults'][0]['status'] or
|
||||
(
|
||||
resp['newMediaItemResults'][0]['status']['message'] != 'Success' and # photos
|
||||
resp['newMediaItemResults'][0]['status']['message'] != 'OK' # videos
|
||||
)
|
||||
|
||||
):
|
||||
self.log('Creating new media item failed: {}'.format(resp['newMediaItemResults'][0]['status']))
|
||||
return None
|
||||
|
||||
return resp['newMediaItemResults'][0]
|
||||
|
|
@ -6,28 +6,118 @@ Plugin object.
|
|||
from __future__ import print_function
|
||||
from builtins import object
|
||||
|
||||
import io
|
||||
|
||||
from json import dumps, loads
|
||||
from importlib import import_module
|
||||
from os.path import dirname, dirname, isdir, isfile
|
||||
from os import mkdir
|
||||
from sys import exc_info
|
||||
from traceback import format_exc
|
||||
|
||||
from elodie.config import load_plugin_config
|
||||
from elodie.compatability import _bytes
|
||||
from elodie.config import load_config_for_plugin, load_plugin_config
|
||||
from elodie.constants import application_directory
|
||||
from elodie import log
|
||||
|
||||
|
||||
class ElodiePluginError(Exception):
|
||||
"""Exception which can be thrown by plugins to return failures.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class PluginBase(object):
|
||||
|
||||
"""Base class which all plugins should inherit from.
|
||||
Defines stubs for all methods and exposes logging and database functionality
|
||||
"""
|
||||
__name__ = 'PluginBase'
|
||||
|
||||
def log(self, msg):
|
||||
log.info(msg)
|
||||
def __init__(self):
|
||||
# Loads the config for the plugin from config.ini
|
||||
self.config_for_plugin = load_config_for_plugin(self.__name__)
|
||||
self.db = PluginDb(self.__name__)
|
||||
|
||||
def after(self, file_path, destination_folder, final_file_path, metadata):
|
||||
pass
|
||||
|
||||
def batch(self):
|
||||
pass
|
||||
|
||||
def before(self, file_path, destination_folder):
|
||||
pass
|
||||
|
||||
def log(self, msg):
|
||||
# Writes an info log not shown unless being run in --debug mode.
|
||||
log.info(dumps(
|
||||
{self.__name__: msg}
|
||||
))
|
||||
|
||||
def display(self, msg):
|
||||
# Writes a log for all modes and will be displayed.
|
||||
log.all(dumps(
|
||||
{self.__name__: msg}
|
||||
))
|
||||
|
||||
class PluginDb(object):
|
||||
"""A database module which provides a simple key/value database.
|
||||
The database is a JSON file located at %application_directory%/plugins/%pluginname.lower()%.json
|
||||
"""
|
||||
def __init__(self, plugin_name):
|
||||
self.db_file = '{}/plugins/{}.json'.format(
|
||||
application_directory,
|
||||
plugin_name.lower()
|
||||
)
|
||||
|
||||
# If the plugin db directory does not exist, create it
|
||||
if(not isdir(dirname(self.db_file))):
|
||||
mkdir(dirname(self.db_file))
|
||||
|
||||
# If the db file does not exist we initialize it
|
||||
if(not isfile(self.db_file)):
|
||||
with io.open(self.db_file, 'wb') as f:
|
||||
f.write(_bytes(dumps({})))
|
||||
|
||||
|
||||
def get(self, key):
|
||||
with io.open(self.db_file, 'r') as f:
|
||||
db = loads(f.read())
|
||||
|
||||
if(key not in db):
|
||||
return None
|
||||
|
||||
return db[key]
|
||||
|
||||
def set(self, key, value):
|
||||
with io.open(self.db_file, 'r') as f:
|
||||
data = f.read()
|
||||
db = loads(data)
|
||||
|
||||
db[key] = value
|
||||
new_content = dumps(db, ensure_ascii=False).encode('utf8')
|
||||
with io.open(self.db_file, 'wb') as f:
|
||||
f.write(new_content)
|
||||
|
||||
def get_all(self):
|
||||
with io.open(self.db_file, 'r') as f:
|
||||
db = loads(f.read())
|
||||
return db
|
||||
|
||||
def delete(self, key):
|
||||
with io.open(self.db_file, 'r') as f:
|
||||
db = loads(f.read())
|
||||
|
||||
# delete key without throwing an exception
|
||||
db.pop(key, None)
|
||||
new_content = dumps(db, ensure_ascii=False).encode('utf8')
|
||||
with io.open(self.db_file, 'wb') as f:
|
||||
f.write(new_content)
|
||||
|
||||
|
||||
class Plugins(object):
|
||||
"""A class to execute plugin actions."""
|
||||
"""Plugin object which manages all interaction with plugins.
|
||||
Exposes methods to load plugins and execute their methods.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.plugins = []
|
||||
|
|
@ -61,11 +151,55 @@ class Plugins(object):
|
|||
|
||||
self.loaded = True
|
||||
|
||||
|
||||
def run_all_before(self, file_path, destination_path, media):
|
||||
self.load()
|
||||
def run_all_after(self, file_path, destination_folder, final_file_path, metadata):
|
||||
"""Process `before` methods of each plugin that was loaded.
|
||||
"""
|
||||
self.load()
|
||||
pass_status = True
|
||||
for cls in self.classes:
|
||||
this_method = getattr(self.classes[cls], 'after')
|
||||
# We try to call the plugin's `before()` method.
|
||||
# If the method explicitly raises an ElodiePluginError we'll fail the import
|
||||
# by setting pass_status to False.
|
||||
# If any other error occurs we log the message and proceed as usual.
|
||||
# By default, plugins don't change behavior.
|
||||
try:
|
||||
this_method(file_path, destination_folder, final_file_path, metadata)
|
||||
log.info('Called after() for {}'.format(cls))
|
||||
except ElodiePluginError as err:
|
||||
log.warn('Plugin {} raised an exception in run_all_before: {}'.format(cls, err))
|
||||
log.error(format_exc())
|
||||
log.error('false')
|
||||
pass_status = False
|
||||
except:
|
||||
log.error(format_exc())
|
||||
return pass_status
|
||||
|
||||
def run_batch(self):
|
||||
self.load()
|
||||
pass_status = True
|
||||
for cls in self.classes:
|
||||
this_method = getattr(self.classes[cls], 'batch')
|
||||
# We try to call the plugin's `before()` method.
|
||||
# If the method explicitly raises an ElodiePluginError we'll fail the import
|
||||
# by setting pass_status to False.
|
||||
# If any other error occurs we log the message and proceed as usual.
|
||||
# By default, plugins don't change behavior.
|
||||
try:
|
||||
this_method()
|
||||
log.info('Called batch() for {}'.format(cls))
|
||||
except ElodiePluginError as err:
|
||||
log.warn('Plugin {} raised an exception in run_batch: {}'.format(cls, err))
|
||||
log.error(format_exc())
|
||||
pass_status = False
|
||||
except:
|
||||
log.error(format_exc())
|
||||
return pass_status
|
||||
|
||||
def run_all_before(self, file_path, destination_folder):
|
||||
"""Process `before` methods of each plugin that was loaded.
|
||||
"""
|
||||
self.load()
|
||||
pass_status = True
|
||||
for cls in self.classes:
|
||||
this_method = getattr(self.classes[cls], 'before')
|
||||
|
|
@ -75,9 +209,10 @@ class Plugins(object):
|
|||
# If any other error occurs we log the message and proceed as usual.
|
||||
# By default, plugins don't change behavior.
|
||||
try:
|
||||
this_method(file_path, destination_path, media)
|
||||
this_method(file_path, destination_folder)
|
||||
log.info('Called before() for {}'.format(cls))
|
||||
except ElodiePluginError as err:
|
||||
log.warn('Plugin {} raised an exception: {}'.format(cls, err))
|
||||
log.warn('Plugin {} raised an exception in run_all_after: {}'.format(cls, err))
|
||||
log.error(format_exc())
|
||||
pass_status = False
|
||||
except:
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ RuntimeError plugin object used for tests.
|
|||
.. moduleauthor:: Jaisen Mathai <jaisen@jmathai.com>
|
||||
"""
|
||||
from __future__ import print_function
|
||||
from builtins import object
|
||||
|
||||
from elodie.plugins.plugins import PluginBase
|
||||
|
||||
|
|
@ -16,6 +15,11 @@ class RuntimeError(PluginBase):
|
|||
def __init__(self):
|
||||
pass
|
||||
|
||||
def before(self, file_path, destination_path, media):
|
||||
def after(self, file_path, destination_folder, final_file_path, metadata):
|
||||
print(does_not_exist)
|
||||
|
||||
def batch(self):
|
||||
print(does_not_exist)
|
||||
|
||||
def before(self, file_path, destination_folder):
|
||||
print(does_not_exist)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ ThrowError plugin object used for tests.
|
|||
.. moduleauthor:: Jaisen Mathai <jaisen@jmathai.com>
|
||||
"""
|
||||
from __future__ import print_function
|
||||
from builtins import object
|
||||
|
||||
from elodie.plugins.plugins import PluginBase, ElodiePluginError
|
||||
|
||||
|
|
@ -16,5 +15,11 @@ class ThrowError(PluginBase):
|
|||
def __init__(self):
|
||||
pass
|
||||
|
||||
def before(self, file_path, destination_path, media):
|
||||
raise ElodiePluginError('Sample plugin error')
|
||||
def after(self, file_path, destination_folder, final_file_path, metadata):
|
||||
raise ElodiePluginError('Sample plugin error for after')
|
||||
|
||||
def batch(self):
|
||||
raise ElodiePluginError('Sample plugin error for batch')
|
||||
|
||||
def before(self, file_path, destination_folder):
|
||||
raise ElodiePluginError('Sample plugin error for before')
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@ from elodie.media.audio import Audio
|
|||
from elodie.media.photo import Photo
|
||||
from elodie.media.text import Text
|
||||
from elodie.media.video import Video
|
||||
from elodie.plugins.plugins import Plugins
|
||||
from elodie.plugins.googlephotos.googlephotos import GooglePhotos
|
||||
|
||||
os.environ['TZ'] = 'GMT'
|
||||
|
||||
|
|
@ -605,43 +607,43 @@ def test_verify_error():
|
|||
assert origin in result.output, result.output
|
||||
assert 'Error 1' in result.output, result.output
|
||||
|
||||
# @mock.patch('elodie.config.config_file', '%s/config.ini-cli-batch-plugin-googlephotos' % gettempdir())
|
||||
# def test_cli_batch_plugin_googlephotos():
|
||||
# auth_file = helper.get_file('plugins/googlephotos/auth_file.json')
|
||||
# secrets_file = helper.get_file('plugins/googlephotos/secrets_file.json')
|
||||
# config_string = """
|
||||
# [Plugins]
|
||||
# plugins=GooglePhotos
|
||||
@mock.patch('elodie.config.config_file', '%s/config.ini-cli-batch-plugin-googlephotos' % gettempdir())
|
||||
def test_cli_batch_plugin_googlephotos():
|
||||
auth_file = helper.get_file('plugins/googlephotos/auth_file.json')
|
||||
secrets_file = helper.get_file('plugins/googlephotos/secrets_file.json')
|
||||
config_string = """
|
||||
[Plugins]
|
||||
plugins=GooglePhotos
|
||||
|
||||
# [PluginGooglePhotos]
|
||||
# auth_file={}
|
||||
# secrets_file={}
|
||||
# """
|
||||
# config_string_fmt = config_string.format(
|
||||
# auth_file,
|
||||
# secrets_file
|
||||
# )
|
||||
# with open('%s/config.ini-cli-batch-plugin-googlephotos' % gettempdir(), 'w') as f:
|
||||
# f.write(config_string_fmt)
|
||||
[PluginGooglePhotos]
|
||||
auth_file={}
|
||||
secrets_file={}
|
||||
"""
|
||||
config_string_fmt = config_string.format(
|
||||
auth_file,
|
||||
secrets_file
|
||||
)
|
||||
with open('%s/config.ini-cli-batch-plugin-googlephotos' % gettempdir(), 'w') as f:
|
||||
f.write(config_string_fmt)
|
||||
|
||||
# if hasattr(load_config, 'config'):
|
||||
# del load_config.config
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
# final_file_path_1 = helper.get_file('plain.jpg')
|
||||
# final_file_path_2 = helper.get_file('no-exif.jpg')
|
||||
# sample_metadata_1 = Photo(final_file_path_1).get_metadata()
|
||||
# sample_metadata_2 = Photo(final_file_path_2).get_metadata()
|
||||
# gp = GooglePhotos()
|
||||
# gp.after('', '', final_file_path_1, sample_metadata_1)
|
||||
# gp.after('', '', final_file_path_2, sample_metadata_1)
|
||||
final_file_path_1 = helper.get_file('plain.jpg')
|
||||
final_file_path_2 = helper.get_file('no-exif.jpg')
|
||||
sample_metadata_1 = Photo(final_file_path_1).get_metadata()
|
||||
sample_metadata_2 = Photo(final_file_path_2).get_metadata()
|
||||
gp = GooglePhotos()
|
||||
gp.after('', '', final_file_path_1, sample_metadata_1)
|
||||
gp.after('', '', final_file_path_2, sample_metadata_1)
|
||||
|
||||
# if hasattr(load_config, 'config'):
|
||||
# del load_config.config
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
# runner = CliRunner()
|
||||
# result = runner.invoke(elodie._batch)
|
||||
# assert "elodie/elodie/tests/files/plain.jpg uploaded successfully.\"}\n" in result.output, result.output
|
||||
# assert "elodie/elodie/tests/files/no-exif.jpg uploaded successfully.\"}\n" in result.output, result.output
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(elodie._batch)
|
||||
assert "elodie/elodie/tests/files/plain.jpg uploaded successfully.\"}\n" in result.output, result.output
|
||||
assert "elodie/elodie/tests/files/no-exif.jpg uploaded successfully.\"}\n" in result.output, result.output
|
||||
|
||||
def test_cli_debug_import():
|
||||
runner = CliRunner()
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
{"scopes": ["https://www.googleapis.com/auth/photoslibrary", "https://www.googleapis.com/auth/photoslibrary.appendonly", "https://www.googleapis.com/auth/photoslibrary.sharing"], "id_token": null, "token": "ya29.Gls1B3ymBdURb3tBLAUJgxQtfTzKry0eaUqplbkHxfIYJH9sWvkalLwXprfAW-Ku0Fz8aP3jz7NrncJ2h58idSEgrYXPQ14iVSkwgUGE2gnsxZM6w4TbHz8ny8Yf", "client_id": "1004259275591-ogsk179e96cs0h126qj590mofk86gdqo.apps.googleusercontent.com", "token_uri": "https://oauth2.googleapis.com/token", "client_secret": "p84GOD_i7_PwDGbAmpnbwKiH", "refresh_token": "1/iHiK9Vbq9i5ebXScyQPDZf9_GJrDOWMvu-2zG9wRsDA"}
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"installed":{"client_id":"1004259275591-ogsk179e96cs0h126qj590mofk86gdqo.apps.googleusercontent.com","project_id":"elodietestphotos-1561522235041","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"p84GOD_i7_PwDGbAmpnbwKiH","redirect_uris":["urn:ietf:wg:oauth:2.0:oob","http://localhost"]}}
|
||||
|
|
@ -0,0 +1,185 @@
|
|||
from __future__ import absolute_import
|
||||
# Project imports
|
||||
import mock
|
||||
import os
|
||||
import sys
|
||||
from tempfile import gettempdir
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))
|
||||
sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))
|
||||
|
||||
import helper
|
||||
from elodie.config import load_config
|
||||
from elodie.plugins.googlephotos.googlephotos import GooglePhotos
|
||||
from elodie.media.audio import Audio
|
||||
from elodie.media.photo import Photo
|
||||
|
||||
# Globals to simplify mocking configs
|
||||
auth_file = helper.get_file('plugins/googlephotos/auth_file.json')
|
||||
secrets_file = helper.get_file('plugins/googlephotos/secrets_file.json')
|
||||
config_string = """
|
||||
[Plugins]
|
||||
plugins=GooglePhotos
|
||||
|
||||
[PluginGooglePhotos]
|
||||
auth_file={}
|
||||
secrets_file={}
|
||||
"""
|
||||
config_string_fmt = config_string.format(
|
||||
auth_file,
|
||||
secrets_file
|
||||
)
|
||||
|
||||
sample_photo = Photo(helper.get_file('plain.jpg'))
|
||||
sample_metadata = sample_photo.get_metadata()
|
||||
sample_metadata['original_name'] = 'foobar'
|
||||
|
||||
@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-set-session' % gettempdir())
|
||||
def test_googlephotos_set_session():
|
||||
with open('%s/config.ini-googlephotos-set-session' % gettempdir(), 'w') as f:
|
||||
f.write(config_string_fmt)
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
gp = GooglePhotos()
|
||||
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
assert gp.session is None, gp.session
|
||||
gp.set_session()
|
||||
assert gp.session is not None, gp.session
|
||||
|
||||
@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-after-supported' % gettempdir())
|
||||
def test_googlephotos_after_supported():
|
||||
with open('%s/config.ini-googlephotos-after-supported' % gettempdir(), 'w') as f:
|
||||
f.write(config_string_fmt)
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
final_file_path = helper.get_file('plain.jpg')
|
||||
gp = GooglePhotos()
|
||||
gp.after('', '', final_file_path, sample_metadata)
|
||||
db_row = gp.db.get(final_file_path)
|
||||
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
assert db_row == 'foobar', db_row
|
||||
|
||||
@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-after-unsupported' % gettempdir())
|
||||
def test_googlephotos_after_unsupported():
|
||||
with open('%s/config.ini-googlephotos-after-unsupported' % gettempdir(), 'w') as f:
|
||||
f.write(config_string_fmt)
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
final_file_path = helper.get_file('audio.m4a')
|
||||
sample_photo = Audio(final_file_path)
|
||||
sample_metadata = sample_photo.get_metadata()
|
||||
sample_metadata['original_name'] = 'foobar'
|
||||
gp = GooglePhotos()
|
||||
gp.after('', '', final_file_path, sample_metadata)
|
||||
db_row = gp.db.get(final_file_path)
|
||||
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
assert db_row == None, db_row
|
||||
|
||||
@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-upload' % gettempdir())
|
||||
def test_googlephotos_upload():
|
||||
with open('%s/config.ini-googlephotos-upload' % gettempdir(), 'w') as f:
|
||||
f.write(config_string_fmt)
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
gp = GooglePhotos()
|
||||
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
gp.set_session()
|
||||
status = gp.upload(helper.get_file('plain.jpg'))
|
||||
|
||||
assert status is not None, status
|
||||
|
||||
@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-upload-session-fail' % gettempdir())
|
||||
def test_googlephotos_upload_session_fail():
|
||||
with open('%s/config.ini-googlephotos-upload-session-fail' % gettempdir(), 'w') as f:
|
||||
f.write(config_string)
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
gp = GooglePhotos()
|
||||
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
gp.set_session()
|
||||
status = gp.upload(helper.get_file('plain.jpg'))
|
||||
|
||||
assert status is None, status
|
||||
|
||||
@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-upload-invalid-empty' % gettempdir())
|
||||
def test_googlephotos_upload_invalid_empty():
|
||||
with open('%s/config.ini-googlephotos-upload-invalid-empty' % gettempdir(), 'w') as f:
|
||||
f.write(config_string_fmt)
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
gp = GooglePhotos()
|
||||
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
gp.set_session()
|
||||
status = gp.upload(helper.get_file('invalid.jpg'))
|
||||
|
||||
assert status is None, status
|
||||
|
||||
@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-upload-dne' % gettempdir())
|
||||
def test_googlephotos_upload_dne():
|
||||
with open('%s/config.ini-googlephotos-upload-dne' % gettempdir(), 'w') as f:
|
||||
f.write(config_string_fmt)
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
gp = GooglePhotos()
|
||||
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
gp.set_session()
|
||||
status = gp.upload('/file/does/not/exist')
|
||||
|
||||
assert status is None, status
|
||||
|
||||
@mock.patch('elodie.config.config_file', '%s/config.ini-googlephotos-batch' % gettempdir())
|
||||
def test_googlephotos_batch():
|
||||
with open('%s/config.ini-googlephotos-batch' % gettempdir(), 'w') as f:
|
||||
f.write(config_string_fmt)
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
final_file_path = helper.get_file('plain.jpg')
|
||||
gp = GooglePhotos()
|
||||
gp.after('', '', final_file_path, sample_metadata)
|
||||
db_row = gp.db.get(final_file_path)
|
||||
assert db_row == 'foobar', db_row
|
||||
|
||||
status, count = gp.batch()
|
||||
db_row_after = gp.db.get(final_file_path)
|
||||
assert status == True, status
|
||||
assert count == 1, count
|
||||
assert db_row_after is None, db_row_after
|
||||
|
||||
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
|
||||
gp.set_session()
|
||||
status = gp.upload(helper.get_file('invalid.jpg'))
|
||||
|
||||
assert status is None, status
|
||||
|
|
@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirna
|
|||
|
||||
from . import helper
|
||||
from elodie.config import load_config
|
||||
from elodie.plugins.plugins import Plugins
|
||||
from elodie.plugins.plugins import Plugins, PluginBase, PluginDb
|
||||
|
||||
@mock.patch('elodie.config.config_file', '%s/config.ini-load-plugins-unset-backwards-compat' % gettempdir())
|
||||
def test_load_plugins_unset_backwards_compat():
|
||||
|
|
@ -134,7 +134,7 @@ plugins=Dummy
|
|||
plugins = Plugins()
|
||||
plugins.load()
|
||||
before_ran_1 = plugins.classes['Dummy'].before_ran
|
||||
plugins.run_all_before('', '', '')
|
||||
plugins.run_all_before('', '')
|
||||
before_ran_2 = plugins.classes['Dummy'].before_ran
|
||||
|
||||
if hasattr(load_config, 'config'):
|
||||
|
|
@ -155,12 +155,16 @@ plugins=ThrowError
|
|||
|
||||
plugins = Plugins()
|
||||
plugins.load()
|
||||
status = plugins.run_all_before('', '', '')
|
||||
status_after = plugins.run_all_after('', '', '', '')
|
||||
status_batch = plugins.run_batch()
|
||||
status_before = plugins.run_all_before('', '')
|
||||
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
assert status == False, status
|
||||
assert status_after == False, status_after
|
||||
assert status_batch == False, status_batch
|
||||
assert status_before == False, status_before
|
||||
|
||||
@mock.patch('elodie.config.config_file', '%s/config.ini-throw-error-one-of-many' % gettempdir())
|
||||
def test_throw_error_one_of_many():
|
||||
|
|
@ -174,12 +178,16 @@ plugins=Dummy,ThrowError
|
|||
|
||||
plugins = Plugins()
|
||||
plugins.load()
|
||||
status = plugins.run_all_before('', '', '')
|
||||
status_after = plugins.run_all_after('', '', '', '')
|
||||
status_batch = plugins.run_batch()
|
||||
status_before = plugins.run_all_before('', '')
|
||||
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
assert status == False, status
|
||||
assert status_after == False, status_after
|
||||
assert status_batch == False, status_batch
|
||||
assert status_before == False, status_before
|
||||
|
||||
@mock.patch('elodie.config.config_file', '%s/config.ini-throw-runtime-error' % gettempdir())
|
||||
def test_throw_error_runtime_error():
|
||||
|
|
@ -193,9 +201,55 @@ plugins=RuntimeError
|
|||
|
||||
plugins = Plugins()
|
||||
plugins.load()
|
||||
status = plugins.run_all_before('', '', '')
|
||||
status_after = plugins.run_all_after('', '', '', '')
|
||||
status_batch = plugins.run_batch()
|
||||
status_before = plugins.run_all_before('', '')
|
||||
|
||||
if hasattr(load_config, 'config'):
|
||||
del load_config.config
|
||||
|
||||
assert status == True, status
|
||||
assert status_after == True, status_after
|
||||
assert status_batch == True, status_batch
|
||||
assert status_before == True, status_before
|
||||
|
||||
def test_plugin_base_inherits_db():
|
||||
plugin_base = PluginBase()
|
||||
assert hasattr(plugin_base.db, 'get')
|
||||
assert hasattr(plugin_base.db, 'set')
|
||||
assert hasattr(plugin_base.db, 'get_all')
|
||||
assert hasattr(plugin_base.db, 'delete')
|
||||
|
||||
def test_db_initialize_file():
|
||||
db = PluginDb('foobar')
|
||||
try:
|
||||
os.remove(db.db_file)
|
||||
except OSError:
|
||||
pass
|
||||
db = PluginDb('foobar')
|
||||
|
||||
def test_db_get_then_set_then_get_then_delete():
|
||||
db = PluginDb('foobar')
|
||||
foo = db.get('foo')
|
||||
assert foo is None, foo
|
||||
db.set('foo', 'bar')
|
||||
foo = db.get('foo')
|
||||
assert foo == 'bar', foo
|
||||
db.delete('foo')
|
||||
foo = db.get('foo')
|
||||
assert foo is None, foo
|
||||
|
||||
def test_db_get_all():
|
||||
# we initialize the db to get the file path to delete then reinitialize
|
||||
db = PluginDb('foobar')
|
||||
try:
|
||||
os.remove(db.db_file)
|
||||
except OSError:
|
||||
pass
|
||||
db = PluginDb('foobar')
|
||||
db.set('a', '1')
|
||||
db.set('b', '2')
|
||||
db.set('c', '3')
|
||||
db.set('d', '4')
|
||||
all_rows = db.get_all()
|
||||
|
||||
assert all_rows == {'a': '1', 'b': '2', 'c': '3', 'd': '4'}, all_rows
|
||||
|
|
|
|||
Loading…
Reference in New Issue