ordigi/tests/test_collection.py

299 lines
11 KiB
Python
Raw Normal View History

2021-08-08 15:33:47 +02:00
from datetime import datetime
import shutil
2021-08-31 16:18:41 +02:00
import sqlite3
2021-08-08 15:33:47 +02:00
from pathlib import Path
import re
2021-10-27 00:06:38 +02:00
import pytest
import inquirer
2021-08-08 15:33:47 +02:00
2021-11-12 09:04:53 +01:00
from ordigi import LOG
2021-08-13 21:11:24 +02:00
from ordigi import constants
from ordigi import utils
from ordigi.summary import Summary
2021-11-06 16:35:35 +01:00
from ordigi.collection import Collection, FPath, Paths
from ordigi.exiftool import ExifTool, ExifToolCaching, exiftool_is_running, terminate_exiftool
from ordigi.geolocation import GeoLocation
2021-11-11 16:24:37 +01:00
from ordigi.media import Media, ReadExif
from ordigi.request import Input
2021-10-27 00:06:38 +02:00
from .conftest import randomize_files, randomize_db
2021-08-08 15:33:47 +02:00
2021-11-12 09:04:53 +01:00
LOG.setLevel(10)
2021-08-08 15:33:47 +02:00
2021-10-17 12:33:14 +02:00
class TestFPath:
2021-08-27 12:45:25 +02:00
@pytest.fixture(autouse=True)
def setup_class(cls, sample_files_paths):
2021-09-18 22:06:34 +02:00
cls.src_path, cls.file_paths = sample_files_paths
cls.path_format = constants.DEFAULT_PATH + '/' + constants.DEFAULT_NAME
2021-08-08 15:33:47 +02:00
2021-08-08 21:43:37 +02:00
def test_get_part(self, tmp_path):
2021-08-08 15:33:47 +02:00
"""
Test all parts
"""
2021-11-12 09:04:53 +01:00
fpath = FPath(self.path_format, 4)
2021-08-08 15:33:47 +02:00
# Item to search for:
2021-10-17 12:33:14 +02:00
items = fpath.get_items()
2021-08-08 15:33:47 +02:00
masks = [
2022-04-18 08:57:47 +02:00
'<album>',
'<basename>',
'<camera_make>',
'<camera_model>',
'<city>',
'<"custom">',
'<country>',
'<ext>',
'<folder>',
'<folders[1:3]>',
'<location>',
'<name>',
'<original_name>',
'<state>',
'<title>',
'<%Y-%m-%d>',
'<%Y-%m-%d_%H-%M-%S>',
'<%Y-%m-%b>'
2021-08-08 15:33:47 +02:00
]
2021-08-08 21:43:37 +02:00
for file_path in self.file_paths:
media = Media(file_path, self.src_path, use_date_filename=True,
use_file_dates=True)
2021-09-18 22:06:34 +02:00
subdirs = file_path.relative_to(self.src_path).parent
exif_tags = {}
for key in ('album', 'camera_make', 'camera_model', 'latitude',
'longitude', 'original_name', 'title'):
exif_tags[key] = media.tags_keys[key]
2021-08-08 15:33:47 +02:00
exif_data = ExifToolCaching(str(file_path)).asdict()
loc = GeoLocation()
2021-11-11 16:24:37 +01:00
media.get_metadata(self.src_path, loc)
2021-08-08 15:33:47 +02:00
for item, regex in items.items():
for mask in masks:
matched = re.search(regex, mask)
if matched:
2021-11-11 16:24:37 +01:00
part = fpath.get_part(item, mask[1:-1], media.metadata)
2021-08-08 15:33:47 +02:00
# check if part is correct
2021-08-08 21:43:37 +02:00
assert isinstance(part, str), file_path
2021-08-08 15:33:47 +02:00
if item == 'basename':
2021-08-08 21:43:37 +02:00
assert part == file_path.stem, file_path
2021-08-08 15:33:47 +02:00
elif item == 'date':
if part == '':
media.get_date_media()
2021-08-08 15:33:47 +02:00
assert datetime.strptime(part, mask[1:-1])
elif item == 'folder':
2021-08-08 21:43:37 +02:00
assert part == subdirs.name, file_path
2021-08-08 15:33:47 +02:00
elif item == 'folders':
2021-09-18 22:06:34 +02:00
assert part in str(subdirs)
2021-08-08 15:33:47 +02:00
elif item == 'ext':
2021-08-08 21:43:37 +02:00
assert part == file_path.suffix[1:], file_path
elif item == 'name':
2021-08-08 15:33:47 +02:00
expected_part = file_path.stem
for rx in utils.get_date_regex().values():
2021-08-08 21:43:37 +02:00
part = re.sub(rx, '', expected_part)
assert part == expected_part, file_path
2021-08-08 15:33:47 +02:00
elif item == 'custom':
2021-08-08 21:43:37 +02:00
assert part == mask[2:-2], file_path
2021-08-08 15:33:47 +02:00
elif item in ('city', 'country', 'location', 'state'):
2021-08-08 21:43:37 +02:00
pass
2021-08-08 15:33:47 +02:00
elif item in exif_tags.keys():
f = False
for key in exif_tags[item]:
if key in exif_data:
f = True
2021-08-08 21:43:37 +02:00
assert part == exif_data[key], file_path
2021-08-08 15:33:47 +02:00
break
if f == False:
2021-08-08 21:43:37 +02:00
assert part == '', file_path
else:
assert part == '', file_path
2021-08-08 15:33:47 +02:00
2021-10-17 12:33:14 +02:00
def test_get_early_morning_photos_date(self):
date = datetime(2021, 10, 16, 2, 20, 40)
2021-11-12 09:04:53 +01:00
fpath = FPath(self.path_format, 4)
2021-10-17 12:33:14 +02:00
part = fpath.get_early_morning_photos_date(date, '%Y-%m-%d')
assert part == '2021-10-15'
part = fpath.get_early_morning_photos_date(date, '%Y%m%d-%H%M%S')
assert part == '20211016-022040'
class TestCollection:
@pytest.fixture(autouse=True)
def setup_class(cls, sample_files_paths):
cls.src_path, cls.file_paths = sample_files_paths
cls.path_format = constants.DEFAULT_PATH + '/' + constants.DEFAULT_NAME
2021-10-17 12:33:14 +02:00
def teardown_class(self):
terminate_exiftool()
assert not exiftool_is_running()
2021-10-27 00:06:38 +02:00
def assert_import(self, summary, nb):
# Summary is created and there is no errors
assert summary.errors == 0
2021-11-01 11:42:01 +01:00
assert summary.success_table.sum('import') == nb
2021-10-27 00:06:38 +02:00
def assert_sort(self, summary, nb):
# Summary is created and there is no errors
assert summary.errors == 0
2021-11-01 11:42:01 +01:00
assert summary.success_table.sum('sort') == nb
2021-10-27 00:06:38 +02:00
2021-08-08 15:33:47 +02:00
def test_sort_files(self, tmp_path):
2022-07-23 20:12:03 +02:00
cli_options = {
'album_from_folder': True, 'cache': False, 'path_format': self.path_format
}
2021-11-19 18:24:35 +01:00
collection = Collection(tmp_path, cli_options=cli_options)
loc = GeoLocation()
2022-07-23 20:12:03 +02:00
summary = collection.sort_files([self.src_path], loc, imp='copy')
2021-08-08 15:33:47 +02:00
self.assert_import(summary, 29)
2021-08-08 15:33:47 +02:00
2021-10-27 00:06:38 +02:00
summary = collection.check_files()
assert summary.success_table.sum('import') == 29
2021-11-01 11:42:01 +01:00
assert summary.success_table.sum('update') == 0
2021-10-27 00:06:38 +02:00
assert not summary.errors
2021-10-15 06:41:22 +02:00
2021-10-27 00:06:38 +02:00
# check if album value are set
2021-11-19 18:24:35 +01:00
filters = {
'exclude': None,
'extensions': None,
'glob': '**/*',
'max_deep': None,
}
paths = Paths(filters).get_files(tmp_path)
for file_path in paths:
2021-09-12 07:39:37 +02:00
if '.db' not in str(file_path):
2021-11-11 16:24:37 +01:00
for value in ReadExif(file_path).get_key_values('album'):
2021-09-12 07:39:37 +02:00
assert value != '' or None
2021-11-19 18:24:35 +01:00
collection = Collection(tmp_path, cli_options=cli_options)
2021-10-27 00:06:38 +02:00
# Try to change path format and sort files again
2022-04-18 08:57:47 +02:00
path_format = 'test_exif/<city>/<%Y>-<name>.%l<ext>'
2022-07-23 20:12:03 +02:00
summary = collection.sort_files([tmp_path], loc)
2021-10-15 06:41:22 +02:00
2022-07-23 20:12:03 +02:00
self.assert_sort(summary, 23)
2021-10-15 06:41:22 +02:00
2021-10-27 00:06:38 +02:00
shutil.copytree(tmp_path / 'test_exif', tmp_path / 'test_exif_copy')
collection.summary = Summary(tmp_path)
2021-11-01 11:42:01 +01:00
assert collection.summary.success_table.sum() == 0
2021-10-27 00:06:38 +02:00
summary = collection.update(loc)
2022-07-23 20:12:03 +02:00
assert summary.success_table.sum('update') == 2
assert summary.success_table.sum() == 2
2021-10-27 00:06:38 +02:00
assert not summary.errors
collection.summary = Summary(tmp_path)
summary = collection.update(loc)
2021-11-01 11:42:01 +01:00
assert summary.success_table.sum() == 0
2021-10-27 00:06:38 +02:00
assert not summary.errors
2021-08-27 12:45:25 +02:00
2021-10-27 00:06:38 +02:00
# test with populated dest dir
randomize_files(tmp_path)
summary = collection.check_files()
assert summary.errors
2021-10-15 06:41:22 +02:00
2021-10-27 00:06:38 +02:00
# test summary update
collection.summary = Summary(tmp_path)
summary = collection.update(loc)
2021-11-01 11:42:01 +01:00
assert summary.success_table.sum('sort') == 0
assert summary.success_table.sum('update')
2021-10-27 00:06:38 +02:00
assert not summary.errors
2021-08-08 15:33:47 +02:00
2021-08-31 16:18:41 +02:00
def test_sort_files_invalid_db(self, tmp_path):
2022-07-23 20:12:03 +02:00
collection = Collection(tmp_path, {'path_format': self.path_format})
2021-08-31 16:18:41 +02:00
loc = GeoLocation()
randomize_db(tmp_path)
with pytest.raises(sqlite3.DatabaseError) as e:
2022-07-23 20:12:03 +02:00
summary = collection.sort_files([self.src_path], loc, imp='copy')
2021-08-27 12:45:25 +02:00
2021-08-08 15:33:47 +02:00
def test_sort_file(self, tmp_path):
2021-11-06 16:35:35 +01:00
for imp in ('copy', 'move', False):
2021-10-27 00:06:38 +02:00
collection = Collection(tmp_path)
2021-08-08 15:33:47 +02:00
# copy mode
2021-09-18 22:06:34 +02:00
src_path = Path(self.src_path, 'test_exif', 'photo.png')
2021-10-27 00:06:38 +02:00
media = Media(src_path, self.src_path)
2021-11-11 16:24:37 +01:00
media.get_metadata(tmp_path)
2021-11-06 16:35:35 +01:00
name = 'photo_' + str(imp) + '.png'
2021-11-07 07:13:56 +01:00
media.metadata['file_path'] = name
dest_path = Path(tmp_path, name)
src_checksum = utils.checksum(src_path)
2021-11-08 07:02:21 +01:00
summary = collection.sort_file(
src_path, dest_path, media.metadata, imp=imp
2021-12-05 13:39:02 +01:00
)
2021-10-27 00:06:38 +02:00
assert not summary.errors
2021-08-08 15:33:47 +02:00
# Ensure files remain the same
2022-04-23 20:04:17 +02:00
if not imp:
assert collection._checkcomp(dest_path, src_checksum)
2021-08-08 15:33:47 +02:00
2021-11-06 16:35:35 +01:00
if imp == 'copy':
2021-08-08 15:33:47 +02:00
assert src_path.exists()
else:
assert not src_path.exists()
shutil.copyfile(dest_path, src_path)
2021-08-08 15:33:47 +02:00
2021-11-06 16:35:35 +01:00
def test_get_files(self):
2021-11-19 18:24:35 +01:00
filters = {
'exclude': {'**/*.dng',},
'extensions': None,
'glob': '**/*',
'max_deep': 1,
}
paths = Paths(filters)
2021-11-06 16:35:35 +01:00
paths = list(paths.get_files(self.src_path))
assert len(paths) == 9
assert Path(self.src_path, 'test_exif/photo.dng') not in paths
2021-09-18 22:06:34 +02:00
for path in paths:
assert isinstance(path, Path)
def test_sort_similar_images(self, tmp_path):
path = tmp_path / 'collection'
shutil.copytree(self.src_path, path)
2021-11-12 09:04:53 +01:00
collection = Collection(path)
2021-10-15 06:41:22 +02:00
loc = GeoLocation()
summary = collection.init(loc)
2021-10-27 00:06:38 +02:00
summary = collection.sort_similar_images(path, similarity=60)
# Summary is created and there is no errors
2021-10-27 00:06:38 +02:00
assert not summary.errors
2022-04-23 13:01:55 +02:00
def test_edit_date_metadata(self, tmp_path, monkeypatch):
2021-10-27 00:06:38 +02:00
path = tmp_path / 'collection'
shutil.copytree(self.src_path, path)
2022-01-29 07:48:33 +01:00
collection = Collection(path, {'cache': False})
2021-10-27 00:06:38 +02:00
def mockreturn(self, message):
return '03-12-2021 08:12:35'
2021-09-18 22:06:34 +02:00
monkeypatch.setattr(Input, 'text', mockreturn)
2021-08-08 15:33:47 +02:00
2022-04-17 22:05:13 +02:00
collection.edit_metadata({path}, {'date_original'}, overwrite=True)
2022-01-29 07:48:33 +01:00
# check if db value is set
2022-04-23 13:01:55 +02:00
file_path = 'test_exif/photo.rw2'
date = collection.db.sqlite.get_metadata(file_path, 'DateOriginal')
2022-01-29 07:48:33 +01:00
assert date == '2021-03-12 08:12:35'
# Check if exif value is set
2022-04-23 13:01:55 +02:00
path_file = path.joinpath(file_path)
date = ExifTool(path_file).asdict()['EXIF:DateTimeOriginal']
assert date == '2021-03-12 08:12:35'
2022-04-23 13:01:55 +02:00
def test_edit_location_metadata(self, tmp_path, monkeypatch):
path = tmp_path / 'collection'
shutil.copytree(self.src_path, path)
collection = Collection(path, {'cache': False})
loc = GeoLocation()
def mockreturn(self, message):
return 'lyon'
2022-04-23 13:01:55 +02:00
monkeypatch.setattr(Input, 'text', mockreturn)
2022-04-23 13:01:55 +02:00
collection.edit_metadata({path}, {'location'}, loc, True)
# check if db value is set
file_path = 'test_exif/photo.rw2'
location_id = collection.db.sqlite.get_metadata(file_path, 'LocationId')
location = collection.db.sqlite.get_location_data(location_id, 'Location')
assert location_id, location == 'Lyon'
# Check if exif value is set
path_file = path.joinpath(file_path)
latitude = ExifTool(path_file).asdict()['EXIF:GPSLatitude']
longitude = ExifTool(path_file).asdict()['EXIF:GPSLongitude']
assert latitude == 45.7578136999889
assert longitude == 4.83201140001667