From ad62b0d61d54e01085453d0288b057bfe77a58df Mon Sep 17 00:00:00 2001 From: Jaisen Mathai Date: Tue, 13 Oct 2015 20:26:55 -0700 Subject: [PATCH] Added FileSystem.process_file and adjust.py --- README.md | 12 ++++ adjust.py | 106 ++++++++++++++++++++++++++++++++ elodie/arguments.py | 9 +-- elodie/filesystem.py | 54 +++++++++++++++++ elodie/geolocation.py | 123 +++++++++++++++++++++++++++++++++----- elodie/media/__init__.py | 3 + elodie/media/media.py | 10 ++++ elodie/media/photo.py | 4 +- import.py | 60 +++++++++++++------ tests/scripts/datetime.py | 32 ++++++++++ 10 files changed, 371 insertions(+), 42 deletions(-) create mode 100755 adjust.py create mode 100644 tests/scripts/datetime.py diff --git a/README.md b/README.md index ff43dd9..8434ae4 100644 --- a/README.md +++ b/README.md @@ -6,3 +6,15 @@ pip install requests brew install exiftool Need config.ini for reverse lookup + +Need pyexiv2. Here's how to install on OS X using homebrew...on your own to use other tools. + +Sourced from http://stackoverflow.com/a/18817419/1318758 + +May need to run these. +brew rm $(brew deps pyexiv2) +brew rm pyexiv2 + +Definietly need to run these +brew install boost --build-from-source +brew install pyexiv2 diff --git a/adjust.py b/adjust.py new file mode 100755 index 0000000..4f719e1 --- /dev/null +++ b/adjust.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python + +import os +import pyexiv2 +import re +import shutil +import sys +import time + +from datetime import datetime + +from elodie import arguments +from elodie import geolocation +from elodie.media.photo import Media +from elodie.media.photo import Photo +from elodie.media.video import Video +from elodie.filesystem import FileSystem +from elodie.localstorage import Db + +def parse_arguments(args): + config = { + 'time': None, + 'location': None, + 'process': 'yes' + } + + config.update(args) + return config + +def main(config, args): + for arg in args: + if(arg[:2] == '--'): + continue + elif(not os.path.exists(arg)): + print 'Could not find %s' % arg + continue + + file_path = arg + destination = os.path.dirname(os.path.dirname(os.path.dirname(file_path))) + + _class = None + extension = os.path.splitext(file_path)[1][1:].lower() + if(extension in Photo.get_valid_extensions()): + _class = Photo + elif(extension in Video.get_valid_extensions()): + _class = Video + + if(_class is None): + continue + + write = False + exif_metadata = pyexiv2.ImageMetadata(file_path) + exif_metadata.read() + if(config['location'] is not None): + location_coords = geolocation.coordinates_by_name(config['location']) + if(location_coords is not None and 'latitude' in location_coords and 'longitude' in location_coords): + print 'Queueing location to exif ...', + exif_metadata['Exif.GPSInfo.GPSLatitude'] = geolocation.decimal_to_dms(location_coords['latitude']) + exif_metadata['Exif.GPSInfo.GPSLatitudeRef'] = pyexiv2.ExifTag('Exif.GPSInfo.GPSLatitudeRef', 'N' if location_coords['latitude'] >= 0 else 'S') + exif_metadata['Exif.GPSInfo.GPSLongitude'] = geolocation.decimal_to_dms(location_coords['longitude']) + exif_metadata['Exif.GPSInfo.GPSLongitudeRef'] = pyexiv2.ExifTag('Exif.GPSInfo.GPSLongitudeRef', 'E' if location_coords['longitude'] >= 0 else 'W') + write = True + print 'OK' + + if(config['time'] is not None): + time_string = config['time'] + print '%r' % time_string + time_format = '%Y-%m-%d %H:%M:%S' + if(re.match('^\d{4}-\d{2}-\d{2}$', time_string)): + time_string = '%s 00:00:00' % time_string + + if(re.match('^\d{4}-\d{2}-\d{2}$', time_string) is None and re.match('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}\d{2}$', time_string)): + print 'Invalid time format. Use YYYY-mm-dd hh:ii:ss or YYYY-mm-dd' + sys.exit(1) + + if(time_format is not None): + print 'Queueing time to exif ...', + exif_metadata['Exif.Photo.DateTimeOriginal'].value = datetime.strptime(time_string, time_format) + exif_metadata['Exif.Image.DateTime'].value = datetime.strptime(time_string, time_format) + print '%r' % datetime.strptime(time_string, time_format) + write = True + print 'OK' + + if(write == True): + exif_metadata.write() + + exif_metadata = pyexiv2.ImageMetadata(file_path) + exif_metadata.read() + + media = _class(file_path) + dest_path = filesystem.process_file(file_path, destination, media, move=True, allowDuplicate=True) + print '%s ...' % dest_path, + print 'OK' + + # If the folder we moved the file out of or its parent are empty we delete it. + filesystem.delete_directory_if_empty(os.path.dirname(file_path)) + filesystem.delete_directory_if_empty(os.path.dirname(os.path.dirname(file_path))) + +db = Db() +filesystem = FileSystem() +args = arguments.parse(sys.argv[1:], None, ['time=','location=','process='], './adjust.py --time= --location= --process=no file1 file2...fileN') +config = parse_arguments(args) + +if __name__ == '__main__': + main(config, sys.argv) + sys.exit(0) diff --git a/elodie/arguments.py b/elodie/arguments.py index 49fe42b..d3b11dd 100644 --- a/elodie/arguments.py +++ b/elodie/arguments.py @@ -4,21 +4,18 @@ import sys, getopt from re import sub def parse(argv, options, long_options, usage): - def help(): - print 'Usage: %s' % usage - try: opts, args = getopt.getopt(argv, options, long_options) except getopt.GetoptError: - print 'Unknown arguments' - help() + print usage sys.exit(2) return_arguments = {} for opt, arg in opts: if opt == '-h': - help() + print usage sys.exit() else: return_arguments[sub('^-+', '', opt)] = arg + return return_arguments diff --git a/elodie/filesystem.py b/elodie/filesystem.py index 2a91859..9c1f90f 100644 --- a/elodie/filesystem.py +++ b/elodie/filesystem.py @@ -4,9 +4,11 @@ Video package that handles all video operations """ import os import re +import shutil import time from elodie import geolocation +from elodie.localstorage import Db """ General file system methods @@ -21,6 +23,18 @@ class FileSystem: if not os.path.exists(directory_path): os.makedirs(directory_path) + """ + Delete a directory only if it's empty. + Instead of checking first using `len([name for name in os.listdir(directory_path)]) == 0` we catch the OSError exception. + + @param, directory_name, string, A fully qualified path of the directory to delete. + """ + def delete_directory_if_empty(self, directory_path): + try: + os.rmdir(directory_path) + except OSError: + pass + """ Recursively get all files which match a path and extension. @@ -99,6 +113,46 @@ class FileSystem: return '/'.join(path) + def process_file(self, _file, destination, media, **kwargs): + move = False + if('move' in kwargs): + move = kwargs['move'] + + allowDuplicate = False + if('allowDuplicate' in kwargs): + allowDuplicate = kwargs['allowDuplicate'] + + metadata = media.get_metadata() + + directory_name = self.get_folder_path(date=metadata['date_taken'], latitude=metadata['latitude'], longitude=metadata['longitude']) + + dest_directory = '%s/%s' % (destination, directory_name) + file_name = self.get_file_name(media) + dest_path = '%s/%s' % (dest_directory, file_name) + + db = Db() + checksum = db.checksum(_file) + if(checksum == None): + print 'Could not get checksum for %s. Skipping...' % _file + return + + # If duplicates are not allowed and this hash exists in the db then we return + if(allowDuplicate == False and db.check_hash(checksum) == True): + print '%s already exists at %s. Skipping...' % (_file, db.get_hash(checksum)) + return + + self.create_directory(dest_directory) + + if(move == True): + shutil.move(_file, dest_path) + else: + shutil.copy2(_file, dest_path) + + db.add_hash(checksum, dest_path) + db.update_hash_db() + + return dest_path + """ Set the modification time on the file based on the file path. Noop if the path doesn't match the format YYYY-MM/DD-IMG_0001.JPG. diff --git a/elodie/geolocation.py b/elodie/geolocation.py index b0b6ee4..ca88cd3 100644 --- a/elodie/geolocation.py +++ b/elodie/geolocation.py @@ -1,12 +1,77 @@ from os import path from ConfigParser import ConfigParser +import fractions +import pyexiv2 + +import math import requests import sys +import urllib -def reverse_lookup(lat, lon): - if(lat is None or lon is None): - return None +class Fraction(fractions.Fraction): + """Only create Fractions from floats. + >>> Fraction(0.3) + Fraction(3, 10) + >>> Fraction(1.1) + Fraction(11, 10) + """ + + def __new__(cls, value, ignore=None): + """Should be compatible with Python 2.6, though untested.""" + return fractions.Fraction.from_float(value).limit_denominator(99999) + +def coordinates_by_name(name): + geolocation_info = lookup(name) + + if(geolocation_info is not None): + if('results' in geolocation_info and len(geolocation_info['results']) != 0 and + 'locations' in geolocation_info['results'][0] and len(geolocation_info['results'][0]['locations']) != 0): + + # By default we use the first entry unless we find one with geocodeQuality=city. + use_location = geolocation_info['results'][0]['locations'][0]['latLng'] + # Loop over the locations to see if we come accross a geocodeQuality=city. + # If we find a city we set that to the use_location and break + for location in geolocation_info['results'][0]['locations']: + if('latLng' in location and 'lat' in location['latLng'] and 'lng' in location['latLng'] and location['geocodeQuality'].lower() == 'city'): + use_location = location['latLng'] + break + + return { + 'latitude': use_location['lat'], + 'longitude': use_location['lng'] + } + + return None + +def decimal_to_dms(decimal): + """Convert decimal degrees into degrees, minutes, seconds. + + >>> decimal_to_dms(50.445891) + [Fraction(50, 1), Fraction(26, 1), Fraction(113019, 2500)] + >>> decimal_to_dms(-125.976893) + [Fraction(125, 1), Fraction(58, 1), Fraction(92037, 2500)] + """ + remainder, degrees = math.modf(abs(decimal)) + remainder, minutes = math.modf(remainder * 60) + # @TODO figure out a better and more proper way to do seconds + return (pyexiv2.Rational(degrees, 1), pyexiv2.Rational(minutes, 1), pyexiv2.Rational(int(remainder*1000000000), 1000000000)) + +def dms_to_decimal(degrees, minutes, seconds, sign=' '): + """Convert degrees, minutes, seconds into decimal degrees. + + >>> dms_to_decimal(10, 10, 10) + 10.169444444444444 + >>> dms_to_decimal(8, 9, 10, 'S') + -8.152777777777779 + """ + return (-1 if sign[0] in 'SWsw' else 1) * ( + float(degrees) + + float(minutes) / 60 + + float(seconds) / 3600 + ) + +def get_key(): config_file = '%s/config.ini' % path.dirname(path.dirname(path.abspath(__file__))) if not path.exists(config_file): return None @@ -16,19 +81,7 @@ def reverse_lookup(lat, lon): if('MapQuest' not in config.sections()): return None - key = config.get('MapQuest', 'key') - - try: - r = requests.get('https://open.mapquestapi.com/nominatim/v1/reverse.php?key=%s&lat=%s&lon=%s&format=json' % (key, lat, lon)) - return r.json() - except requests.exceptions.RequestException as e: - print e - return None - except ValueError as e: - print r.text - print e - return None - + return config.get('MapQuest', 'key') def place_name(lat, lon): geolocation_info = reverse_lookup(lat, lon) @@ -42,3 +95,41 @@ def place_name(lat, lon): elif('country' in address): return address['country'] return None + + +def reverse_lookup(lat, lon): + if(lat is None or lon is None): + return None + + key = get_key() + + try: + params = {'format': 'json', 'key': key, 'lat': lat, 'lon': lon} + r = requests.get('http://open.mapquestapi.com/nominatim/v1/reverse.php?%s' % urllib.urlencode(params)) + return r.json() + except requests.exceptions.RequestException as e: + print e + return None + except ValueError as e: + print r.text + print e + return None + +def lookup(name): + if(name is None or len(name) == 0): + return None + + key = get_key() + + try: + params = {'format': 'json', 'key': key, 'location': name} + print 'http://open.mapquestapi.com/geocoding/v1/address?%s' % urllib.urlencode(params) + r = requests.get('http://open.mapquestapi.com/geocoding/v1/address?%s' % urllib.urlencode(params)) + return r.json() + except requests.exceptions.RequestException as e: + print e + return None + except ValueError as e: + print r.text + print e + return None diff --git a/elodie/media/__init__.py b/elodie/media/__init__.py index e69de29..a430ebb 100644 --- a/elodie/media/__init__.py +++ b/elodie/media/__init__.py @@ -0,0 +1,3 @@ +from media import Media +from photo import Photo +from video import Video diff --git a/elodie/media/media.py b/elodie/media/media.py index fad862b..d54be1e 100644 --- a/elodie/media/media.py +++ b/elodie/media/media.py @@ -66,3 +66,13 @@ class Media(object): return None return mimetype[0] + + def get_class_by_file(Media, _file): + extension = os.path.splitext(_file)[1][1:].lower() + if(extension in Photo.get_valid_extensions()): + return Photo + elif(extension in Video.get_valid_extensions()): + return Video + else: + return None + diff --git a/elodie/media/photo.py b/elodie/media/photo.py index 8ff7fc1..b97c0a9 100644 --- a/elodie/media/photo.py +++ b/elodie/media/photo.py @@ -161,5 +161,5 @@ class Photo(Media): @returns, tuple """ @classmethod - def get_valid_extensions(Video): - return Video.__valid_extensions + def get_valid_extensions(Photo): + return Photo.__valid_extensions diff --git a/import.py b/import.py index f30965b..c45878a 100755 --- a/import.py +++ b/import.py @@ -10,8 +10,31 @@ from elodie.media.video import Video from elodie.filesystem import FileSystem from elodie.localstorage import Db -db = Db() -filesystem = FileSystem() +def help(): + return """ + usage: ./import.py --type=photo --source=/path/to/photos --destination=/path/to/destination + + --type Valid values are 'photo' or 'video'. Only files of *type* are imported. + --file Full path to a photo or video to be imported. The --type argument should match the file type of the file. + @TODO: Automatically determine *type* from *file* + --source Full path to a directory which will be recursively crawled for files of *type*. + --destination Full path to a directory where organized photos will be placed. + """ + +def parse_arguments(args): + config = { + 'type': 'photo', + 'file': None, + 'source': None, + 'destination': None + } + + if('destination' not in args): + help() + sys.exit(2) + + config.update(args) + return config def process_file(_file, destination, media): checksum = db.checksum(_file) @@ -35,25 +58,20 @@ def process_file(_file, destination, media): filesystem.create_directory(dest_directory) print '%s -> %s' % (_file, dest_path) - shutil.copy2(_file, dest_path) - #shutil.move(_file, dest_path) + #shutil.copy2(_file, dest_path) + shutil.move(_file, dest_path) db.add_hash(checksum, dest_path) def main(argv): - args = arguments.parse(argv, None, ['file=','type=','source=','destination='], './import.py --type= --source= -destination=') - if('destination' not in args): - print 'No destination passed in' - sys.exit(2) - - destination = args['destination'] - if('type' in args and args['type'] == 'photo'): + destination = config['destination'] + if(config['type'] == 'photo'): media_type = Photo else: media_type = Video - if('source' in args): - source = args['source'] + if(config['source'] is not None): + source = config['source'] write_counter = 0 for current_file in filesystem.get_all_files(source, media_type.get_valid_extensions()): @@ -71,15 +89,21 @@ def main(argv): # If there's anything we haven't written to the hash database then write it now if(write_counter % 10 != 10): db.update_hash_db() - elif('file' in args): - media = media_type(args['file']) + elif(config['file'] is not None): + media = media_type(config['file']) if(media_type.__name__ == 'Video'): filesystem.set_date_from_path_video(media) - process_file(args['file'], destination, media) + process_file(config['file'], destination, media) db.update_hash_db() + else: + help() + +db = Db() +filesystem = FileSystem() +args = arguments.parse(sys.argv[1:], None, ['file=','type=','source=','destination='], help()) +config = parse_arguments(args) if __name__ == '__main__': - main(sys.argv[1:]) + main(config) sys.exit(0) - diff --git a/tests/scripts/datetime.py b/tests/scripts/datetime.py new file mode 100644 index 0000000..166c368 --- /dev/null +++ b/tests/scripts/datetime.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +import os +import shutil +import sys + +from elodie import arguments +from elodie.media.photo import Photo +from elodie.media.video import Video + +def main(argv): + args = arguments.parse(argv, None, ['file=','type='], './import.py --type= --file=') + + if('file' not in args): + print 'No file specified' + sys.exit(1) + + if('type' in args and args['type'] == 'photo'): + media_type = Photo + else: + media_type = Video + + media = media_type(args['file']) + metadata = media.get_metadata() + + output = {'date_taken': metadata['date_taken']} + print '%r' % output + + +if __name__ == '__main__': + main(sys.argv[1:]) + sys.exit(0)