takino
/
ordigi
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add fallback folder support when configuring folder hierarchy. #199 (#209) 

* add test
This commit is contained in:
Jaisen Mathai 2017-04-12 23:33:18 -07:00 committed by GitHub
parent 6777c32588
commit e3123872c4
4 changed files with 195 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Readme.md
View File

@ -231,18 +231,30 @@ year=%Y
date=%year-%month date=%year-%month
full_path=%date/%location full_path=%date/%location
# -> 2015-12/Sunnyvale, California # -> 2015-12/Sunnyvale, California
full_path=%country/%state/%city
# -> US/California/Sunnyvale
``` ```
#### Using fallback folders
There are times when the EXIF needed to correctly name a folder doesn't exist on a photo. I came up with fallback folders to help you deal with situations such as this. Here's how it works.
You can specify a series of folder names by separating them with a `|`. That's a pipe, not an L. Let's look at an example.
```
month=%m
year=%Ykkkk
location=%city
full_path=%month/%year/%album|%location|%"Beats me"
```
What this asks me to do is to name the last folder the same as the album I find in EXIF. If I don't find an album in EXIF then I should use the location. If there's no GPS in the EXIf then I should name the last folder `Beats me`.
#### How folder customization works #### How folder customization works
You can construct your folder structure using a combination of the location and dates. Under the `Directory` section of your `config.ini` file you can define placeholder names and assign each a value. For example, `date=%Y-%m` would create a date placeholder with a value of YYYY-MM which would be filled in with the date from the EXIF on the photo. You can construct your folder structure using a combination of the location and dates. Under the `Directory` section of your `config.ini` file you can define placeholder names and assign each a value. For example, `date=%Y-%m` would create a date placeholder with a value of YYYY-MM which would be filled in with the date from the EXIF on the photo.
The placeholders can be used to define the folder structure you'd like to create. The example above happens to be the default structure and would look like `2015-07-Jul/Mountain View`. The placeholders can be used to define the folder structure you'd like to create. The example above happens to be the default structure and would look like `2015-07-Jul/Mountain View`.
I have a few built-in location placeholders you can use. I have a few built-in location placeholders you can use. Use this to construct the `%location` you use in `full_path`.
* `%city` the name of the city the photo was taken. Requires geolocation data in EXIF. * `%city` the name of the city the photo was taken. Requires geolocation data in EXIF.
* `%state` the name of the state the photo was taken. Requires geolocation data in EXIF. * `%state` the name of the state the photo was taken. Requires geolocation data in EXIF.

89
elodie/filesystem.py
View File

@ -24,10 +24,15 @@ class FileSystem(object):
def __init__(self): def __init__(self):
# The default folder path is along the lines of 2015-01-Jan/Chicago # The default folder path is along the lines of 2015-01-Jan/Chicago
self.default_folder_path_definition = [ self.default_folder_path_definition = {
('date', '%Y-%m-%b'), ('location', '%city') 'date': '%Y-%m-%b',
] 'location': '%city',
'full_path': '%date/%album|%location|"{}"'.format(
geolocation.__DEFAULT_LOCATION__
),
}
self.cached_folder_path_definition = None self.cached_folder_path_definition = None
self.default_parts = ['album', 'city', 'state', 'country']
def create_directory(self, directory_path): def create_directory(self, directory_path):
"""Create a directory if it does not already exist. """Create a directory if it does not already exist.
@ -149,6 +154,22 @@ class FileSystem(object):
return file_name.lower() return file_name.lower()
def get_folder_path_definition(self): def get_folder_path_definition(self):
"""Returns a list of folder definitions.
Each element in the list represents a folder.
Fallback folders are supported and are nested lists.
Return values take the following form.
[
('date', '%Y-%m-%d'),
[
('location', '%city'),
('album', ''),
('"Unknown Location", '')
]
]
:returns: list
"""
# If we've done this already then return it immediately without # If we've done this already then return it immediately without
# incurring any extra work # incurring any extra work
if self.cached_folder_path_definition is not None: if self.cached_folder_path_definition is not None:
@ -158,28 +179,46 @@ class FileSystem(object):
# If Directory is in the config we assume full_path and its # If Directory is in the config we assume full_path and its
# corresponding values (date, location) are also present # corresponding values (date, location) are also present
if('Directory' not in config): config_directory = self.default_folder_path_definition
return self.default_folder_path_definition if('Directory' in config):
config_directory = config['Directory'] config_directory = config['Directory']
# Find all subpatterns of full_path that map to directories. # Find all subpatterns of full_path that map to directories.
# I.e. %foo/%bar => ['foo', 'bar'] # I.e. %foo/%bar => ['foo', 'bar']
# I.e. %foo/%bar|%example|"something" => ['foo', 'bar|example|"something"']
path_parts = re.findall( path_parts = re.findall(
'\%([a-z]+)', '(\%[^/]+)',
config_directory['full_path'] config_directory['full_path']
) )
if not path_parts or len(path_parts) == 0: if not path_parts or len(path_parts) == 0:
return self.default_folder_path_definition return self.default_folder_path_definition
self.cached_folder_path_definition = [ self.cached_folder_path_definition = []
(part, config_directory[part]) for part in path_parts for part in path_parts:
] if part in config_directory:
part = part[1:]
self.cached_folder_path_definition.append(
[(part, config_directory[part])]
)
elif part in self.default_parts:
part = part[1:]
self.cached_folder_path_definition.append(
[(part, '')]
)
else:
this_part = []
for p in part.split('|'):
p = p[1:]
this_part.append(
(p, config_directory[p] if p in config_directory else '')
)
self.cached_folder_path_definition.append(this_part)
return self.cached_folder_path_definition return self.cached_folder_path_definition
def get_folder_path(self, metadata): def get_folder_path(self, metadata):
"""Get folder path by various parameters. """Given a media's metadata this function returns the folder path as a string.
:param metadata dict: Metadata dictionary. :param metadata dict: Metadata dictionary.
:returns: str :returns: str
@ -187,9 +226,19 @@ class FileSystem(object):
path_parts = self.get_folder_path_definition() path_parts = self.get_folder_path_definition()
path = [] path = []
for path_part in path_parts: for path_part in path_parts:
part, mask = path_part # We support fallback values so that
# 'album|city|"Unknown Location"
# %album|%city|"Unknown Location" results in
# My Album - when an album exists
# Sunnyvale - when no album exists but a city exists
# Unknown Location - when neither an album nor location exist
for this_part in path_part:
part, mask = this_part
if part in ('date', 'day', 'month', 'year'): if part in ('date', 'day', 'month', 'year'):
path.append(time.strftime(mask, metadata['date_taken'])) path.append(
time.strftime(mask, metadata['date_taken'])
)
break
elif part in ('location', 'city', 'state', 'country'): elif part in ('location', 'city', 'state', 'country'):
place_name = geolocation.place_name( place_name = geolocation.place_name(
metadata['latitude'], metadata['latitude'],
@ -203,15 +252,13 @@ class FileSystem(object):
place_name, place_name,
) )
path.append(parsed_folder_name) path.append(parsed_folder_name)
break
# For now we always make the leaf folder an album if it's in the EXIF. elif part in ('album'):
# This is to preserve backwards compatability until we figure out how if metadata['album']:
# to include %album in the config.ini syntax.
if(metadata['album'] is not None):
if(len(path) == 1):
path.append(metadata['album']) path.append(metadata['album'])
elif(len(path) == 2): break
path[1] = metadata['album'] elif part.startswith('"') and part.endswith('"'):
path.append(part[1:-1])
return os.path.join(*path) return os.path.join(*path)

32
elodie/tests/elodie_test.py
View File

@ -430,38 +430,6 @@ def test_update_time_on_video():
assert metadata['date_taken'] != metadata_processed['date_taken'] assert metadata['date_taken'] != metadata_processed['date_taken']
assert metadata_processed['date_taken'] == helper.time_convert((2000, 1, 1, 12, 0, 0, 5, 1, 0)), metadata_processed['date_taken'] assert metadata_processed['date_taken'] == helper.time_convert((2000, 1, 1, 12, 0, 0, 5, 1, 0)), metadata_processed['date_taken']
@mock.patch('elodie.config.config_file', '%s/config.ini-multiple-directories' % gettempdir())
def test_update_with_more_than_two_levels_of_directories():
with open('%s/config.ini-multiple-directories' % gettempdir(), 'w') as f:
f.write("""
[Directory]
year=%Y
month=%m
day=%d
full_path=%year/%month/%day
""")
temporary_folder, folder = helper.create_working_folder()
temporary_folder_destination, folder_destination = helper.create_working_folder()
origin = '%s/plain.jpg' % folder
shutil.copyfile(helper.get_file('plain.jpg'), origin)
if hasattr(load_config, 'config'):
del load_config.config
cfg = load_config()
helper.reset_dbs()
runner = CliRunner()
result = runner.invoke(elodie._import, ['--destination', folder_destination, folder])
runner2 = CliRunner()
result = runner2.invoke(elodie._update, ['--title', 'test title', folder_destination])
helper.restore_dbs()
if hasattr(load_config, 'config'):
del load_config.config
updated_file_path = '{}/2015/12/05/2015-12-05_00-59-26-plain-test-title.jpg'.format(folder_destination)
assert os.path.isfile(updated_file_path), updated_file_path
def test_update_with_directory_passed_in(): def test_update_with_directory_passed_in():
temporary_folder, folder = helper.create_working_folder() temporary_folder, folder = helper.create_working_folder()
temporary_folder_destination, folder_destination = helper.create_working_folder() temporary_folder_destination, folder_destination = helper.create_working_folder()

103
elodie/tests/filesystem_test.py
View File

@ -226,6 +226,20 @@ def test_get_folder_path_with_location():
assert path == os.path.join('2015-12-Dec','Sunnyvale'), path assert path == os.path.join('2015-12-Dec','Sunnyvale'), path
@mock.patch('elodie.config.config_file', '%s/config.ini-original-default-unknown-location' % gettempdir())
def test_get_folder_path_with_original_default_unknown_location():
with open('%s/config.ini-original-default-with-unknown-location' % gettempdir(), 'w') as f:
f.write('')
if hasattr(load_config, 'config'):
del load_config.config
filesystem = FileSystem()
media = Photo(helper.get_file('plain.jpg'))
path = filesystem.get_folder_path(media.get_metadata())
if hasattr(load_config, 'config'):
del load_config.config
assert path == os.path.join('2015-12-Dec','Unknown Location'), path
@mock.patch('elodie.config.config_file', '%s/config.ini-custom-path' % gettempdir()) @mock.patch('elodie.config.config_file', '%s/config.ini-custom-path' % gettempdir())
def test_get_folder_path_with_custom_path(): def test_get_folder_path_with_custom_path():
with open('%s/config.ini-custom-path' % gettempdir(), 'w') as f: with open('%s/config.ini-custom-path' % gettempdir(), 'w') as f:
@ -248,10 +262,33 @@ full_path=%date/%location
assert path == os.path.join('2015-12-05','United States of America-California-Sunnyvale'), path assert path == os.path.join('2015-12-05','United States of America-California-Sunnyvale'), path
@mock.patch('elodie.config.config_file', '%s/config.ini-fallback' % gettempdir())
def test_get_folder_path_with_fallback_folder():
with open('%s/config.ini-fallback' % gettempdir(), 'w') as f:
f.write("""
[Directory]
year=%Y
month=%m
full_path=%year/%month/%album|%"No Album Fool"/%month
""")
#full_path=%year/%album|"No Album"
if hasattr(load_config, 'config'):
del load_config.config
filesystem = FileSystem()
media = Photo(helper.get_file('plain.jpg'))
path = filesystem.get_folder_path(media.get_metadata())
if hasattr(load_config, 'config'):
del load_config.config
assert path == os.path.join('2015','12','No Album Fool','12'), path
@mock.patch('elodie.config.config_file', '%s/config.ini-location-date' % gettempdir()) @mock.patch('elodie.config.config_file', '%s/config.ini-location-date' % gettempdir())
def test_get_folder_path_with_with_more_than_two_levels(): def test_get_folder_path_with_with_more_than_two_levels():
with open('%s/config.ini-location-date' % gettempdir(), 'w') as f: with open('%s/config.ini-location-date' % gettempdir(), 'w') as f:
f.write(""" f.write("""
[MapQuest]
key=czjNKTtFjLydLteUBwdgKAIC8OAbGLUx
[Directory] [Directory]
year=%Y year=%Y
month=%m month=%m
@ -551,6 +588,37 @@ def test_process_video_with_album_then_title():
assert origin_checksum != destination_checksum, destination_checksum assert origin_checksum != destination_checksum, destination_checksum
assert helper.path_tz_fix(os.path.join('2015-01-Jan','test_album','2015-01-19_12-45-11-movie-test_title.mov')) in destination, destination assert helper.path_tz_fix(os.path.join('2015-01-Jan','test_album','2015-01-19_12-45-11-movie-test_title.mov')) in destination, destination
@mock.patch('elodie.config.config_file', '%s/config.ini-multiple-directories' % gettempdir())
def test_process_twice_more_than_two_levels_of_directories():
with open('%s/config.ini-multiple-directories' % gettempdir(), 'w') as f:
f.write("""
[Directory]
year=%Y
month=%m
day=%d
full_path=%year/%month/%day
""")
filesystem = FileSystem()
temporary_folder, folder = helper.create_working_folder()
origin = os.path.join(folder,'plain.jpg')
shutil.copyfile(helper.get_file('plain.jpg'), origin)
media = Photo(origin)
destination = filesystem.process_file(origin, temporary_folder, media, allowDuplicate=True)
assert helper.path_tz_fix(os.path.join('2015','12','05', '2015-12-05_00-59-26-plain.jpg')) in destination, destination
media_second = Photo(destination)
media_second.set_title('foo')
destination_second = filesystem.process_file(destination, temporary_folder, media_second, allowDuplicate=True)
assert destination.replace('.jpg', '-foo.jpg') == destination_second, destination_second
shutil.rmtree(folder)
shutil.rmtree(os.path.dirname(os.path.dirname(destination)))
def test_set_utime_with_exif_date(): def test_set_utime_with_exif_date():
filesystem = FileSystem() filesystem = FileSystem()
temporary_folder, folder = helper.create_working_folder() temporary_folder, folder = helper.create_working_folder()
@ -618,7 +686,7 @@ def test_get_folder_path_definition_default():
if hasattr(load_config, 'config'): if hasattr(load_config, 'config'):
del load_config.config del load_config.config
assert path_definition == filesystem.default_folder_path_definition, path_definition assert path_definition == [[('date', '%Y-%m-%b')], [('album', ''), ('location', '%city'), ('Unknown Location"', '')]], path_definition
@mock.patch('elodie.config.config_file', '%s/config.ini-date-location' % gettempdir()) @mock.patch('elodie.config.config_file', '%s/config.ini-date-location' % gettempdir())
def test_get_folder_path_definition_date_location(): def test_get_folder_path_definition_date_location():
@ -635,7 +703,7 @@ full_path=%date/%location
filesystem = FileSystem() filesystem = FileSystem()
path_definition = filesystem.get_folder_path_definition() path_definition = filesystem.get_folder_path_definition()
expected = [ expected = [
('date', '%Y-%m-%d'), ('location', '%country') [('date', '%Y-%m-%d')], [('location', '%country')]
] ]
if hasattr(load_config, 'config'): if hasattr(load_config, 'config'):
del load_config.config del load_config.config
@ -657,7 +725,7 @@ full_path=%location/%date
filesystem = FileSystem() filesystem = FileSystem()
path_definition = filesystem.get_folder_path_definition() path_definition = filesystem.get_folder_path_definition()
expected = [ expected = [
('location', '%country'), ('date', '%Y-%m-%d') [('location', '%country')], [('date', '%Y-%m-%d')]
] ]
if hasattr(load_config, 'config'): if hasattr(load_config, 'config'):
del load_config.config del load_config.config
@ -679,7 +747,7 @@ full_path=%date/%location
filesystem = FileSystem() filesystem = FileSystem()
path_definition = filesystem.get_folder_path_definition() path_definition = filesystem.get_folder_path_definition()
expected = [ expected = [
('date', '%Y-%m-%d'), ('location', '%country') [('date', '%Y-%m-%d')], [('location', '%country')]
] ]
assert path_definition == expected, path_definition assert path_definition == expected, path_definition
@ -696,7 +764,7 @@ full_path=%date/%location
filesystem = FileSystem() filesystem = FileSystem()
path_definition = filesystem.get_folder_path_definition() path_definition = filesystem.get_folder_path_definition()
expected = [ expected = [
('date', '%Y-%m-%d'), ('location', '%country') [('date', '%Y-%m-%d')], [('location', '%country')]
] ]
if hasattr(load_config, 'config'): if hasattr(load_config, 'config'):
del load_config.config del load_config.config
@ -717,7 +785,7 @@ full_path=%year/%month/%day
filesystem = FileSystem() filesystem = FileSystem()
path_definition = filesystem.get_folder_path_definition() path_definition = filesystem.get_folder_path_definition()
expected = [ expected = [
('year', '%Y'), ('month', '%m'), ('day', '%d') [('year', '%Y')], [('month', '%m')], [('day', '%d')]
] ]
if hasattr(load_config, 'config'): if hasattr(load_config, 'config'):
del load_config.config del load_config.config
@ -738,9 +806,30 @@ full_path=%year
filesystem = FileSystem() filesystem = FileSystem()
path_definition = filesystem.get_folder_path_definition() path_definition = filesystem.get_folder_path_definition()
expected = [ expected = [
('year', '%Y') [('year', '%Y')]
] ]
if hasattr(load_config, 'config'): if hasattr(load_config, 'config'):
del load_config.config del load_config.config
assert path_definition == expected, path_definition assert path_definition == expected, path_definition
@mock.patch('elodie.config.config_file', '%s/config.ini-multi-level-custom' % gettempdir())
def test_get_folder_path_definition_multi_level_custom():
with open('%s/config.ini-multi-level-custom' % gettempdir(), 'w') as f:
f.write("""
[Directory]
year=%Y
month=%M
full_path=%year/%album|%month|%"foo"/%month
""")
if hasattr(load_config, 'config'):
del load_config.config
filesystem = FileSystem()
path_definition = filesystem.get_folder_path_definition()
expected = [[('year', '%Y')], [('album', ''), ('month', '%M'), ('"foo"', '')], [('month', '%M')]]
if hasattr(load_config, 'config'):
del load_config.config
assert path_definition == expected, path_definition