diff --git a/elodie/constants.py b/elodie/constants.py new file mode 100644 index 0000000..2e5d894 --- /dev/null +++ b/elodie/constants.py @@ -0,0 +1,4 @@ +from os import path + +application_directory = '{}/.elodie'.format(path.expanduser('~')) +hash_db = '{}/hash.json'.format(application_directory) diff --git a/elodie/geolocation.py b/elodie/geolocation.py index 23c9aa5..a7a25f6 100644 --- a/elodie/geolocation.py +++ b/elodie/geolocation.py @@ -17,8 +17,17 @@ def reverse_lookup(lat, lon): key = config.get('MapQuest', 'key') - r = requests.get('https://open.mapquestapi.com/nominatim/v1/reverse.php?key=%s&lat=%s&lon=%s&format=json' % (key, lat, lon)) - return r.json() + try: + r = requests.get('https://open.mapquestapi.com/nominatim/v1/reverse.php?key=%s&lat=%s&lon=%s&format=json' % (key, lat, lon)) + return r.json() + except requests.exceptions.RequestException as e: + print e + return None + except ValueError as e: + print r.text + print e + return None + def place_name(lat, lon): geolocation_info = reverse_lookup(lat, lon) diff --git a/elodie/localstorage.py b/elodie/localstorage.py new file mode 100644 index 0000000..3786b9f --- /dev/null +++ b/elodie/localstorage.py @@ -0,0 +1,50 @@ +import hashlib +import json +import os + +from elodie import constants + +class Db(object): + def __init__(self): + # verify that the application directory (~/.elodie) exists, else create it + if not os.path.exists(constants.application_directory): + os.makedirs(constants.application_directory) + + self.hash_db = {} + # we need to open in w incase the file doesn't exist + with open(constants.hash_db, 'rw') as f: + try: + self.hash_db = json.load(f) + except ValueError: + pass + + def add_hash(self, key, value, write=False): + self.hash_db[key] = value + if(write == True): + self.update_hash_db() + + def check_hash(self, key): + return key in self.hash_db + + def get_hash(self, key): + if(self.check_hash(key) == True): + return self.hash_db[key] + return None + + def update_hash_db(self): + with open(constants.hash_db, 'w') as f: + json.dump(self.hash_db, f) + + """ + http://stackoverflow.com/a/3431835/1318758 + """ + def checksum(self, file_path, blocksize=65536): + hasher = hashlib.sha256() + with open(file_path, 'r') as f: + buf = f.read(blocksize) + + while len(buf) > 0: + hasher.update(buf) + buf = f.read(blocksize) + return hasher.hexdigest() + return None diff --git a/elodie/media/photo.py b/elodie/media/photo.py index 96ec33b..c9e720f 100644 --- a/elodie/media/photo.py +++ b/elodie/media/photo.py @@ -51,12 +51,15 @@ class Photo(Media): # EXIF DateTime is already stored as a timestamp # Sourced from https://github.com/photo/frontend/blob/master/src/libraries/models/Photo.php#L500 exif = self.get_exif() - if('EXIF DateTimeOriginal' in exif): - seconds_since_epoch = time.mktime(datetime.strptime(str(exif['EXIF DateTimeOriginal']), '%Y:%m:%d %H:%M:%S').timetuple()) - elif('EXIF DateTime' in exif): - seconds_since_epoch = time.mktime(datetime.strptime(str(exif['EXIF DateTime']), '%Y:%m:%d %H:%M:%S').timetuple()) - elif('EXIF FileDateTime' in exif): - seconds_since_epoch = str(exif['EXIF DateTime']) + try: + if('EXIF DateTimeOriginal' in exif): + seconds_since_epoch = time.mktime(datetime.strptime(str(exif['EXIF DateTimeOriginal']), '%Y:%m:%d %H:%M:%S').timetuple()) + elif('EXIF DateTime' in exif): + seconds_since_epoch = time.mktime(datetime.strptime(str(exif['EXIF DateTime']), '%Y:%m:%d %H:%M:%S').timetuple()) + elif('EXIF FileDateTime' in exif): + seconds_since_epoch = str(exif['EXIF DateTime']) + except: + pass if(seconds_since_epoch == 0): return None diff --git a/import.py b/import.py index b27e7b6..dcd7da7 100755 --- a/import.py +++ b/import.py @@ -5,35 +5,67 @@ import shutil import sys from elodie import arguments +from elodie.media.photo import Photo from elodie.media.video import Video from elodie.filesystem import FileSystem +from elodie.localstorage import Db def main(argv): - - args = arguments.parse(argv, None, ['source=','destination='], './import.py --source= -destination=') + args = arguments.parse(argv, None, ['type=','source=','destination='], './import.py --type= --source= -destination=') + + db = Db() source = args['source'] destination = args['destination'] filesystem = FileSystem() + if(args['type'] == 'photo'): + media_type = Photo + else: + media_type = Video - for video_file in filesystem.get_all_files(source, Video.get_valid_extensions()): - video = Video(video_file) + write_counter = 0 + for current_file in filesystem.get_all_files(source, media_type.get_valid_extensions()): + checksum = db.checksum(current_file) + if(checksum == None): + print 'Could not get checksum for %s. Skipping...' % current_file + continue - filesystem.set_date_from_path_video(video) + if(db.check_hash(checksum) == True): + print '%s already exists at %s. Skipping...' % (current_file, db.get_hash(checksum)) + continue - metadata = video.get_metadata() + media = media_type(current_file) - directory_name = filesystem.get_folder_name_by_date(metadata['date_taken']) + if(media_type.__name__ == 'Video'): + filesystem.set_date_from_path_video(media) + + metadata = media.get_metadata() + + directory_name = filesystem.get_folder_path(date=metadata['date_taken'], latitude=metadata['latitude'], longitude=metadata['longitude']) + #directory_name = filesystem.get_folder_path(date=metadata['date_taken']) dest_directory = '%s/%s' % (destination, directory_name) # TODO remove the day prefix of the file that was there prior to the crawl - file_name = filesystem.get_file_name_for_video(video) + file_name = filesystem.get_file_name(media) dest_path = '%s/%s' % (dest_directory, file_name) filesystem.create_directory(dest_directory) - print '%s -> %s' % (video_file, dest_path) - shutil.copy2(video_file, dest_path) + print '%s -> %s' % (current_file, dest_path) + shutil.copy2(current_file, dest_path) + #shutil.move(current_file, dest_path) + db.add_hash(checksum, dest_path) + + # Write to the hash database every 10 iterations + write_counter += 1 + if(write_counter % 10 == 0): + db.update_hash_db() + + + + # If there's anything we haven't written to the hash database then write it now + if(write_counter % 10 != 10): + db.update_hash_db() if __name__ == '__main__': main(sys.argv[1:])