From 64018561f297e3d1e06c7f8aca8241346557af2c Mon Sep 17 00:00:00 2001 From: Cedric Leporcq Date: Sat, 14 Aug 2021 23:06:58 +0200 Subject: [PATCH] some stuff --- pytest.ini | 7 + todo.md | 612 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 619 insertions(+) create mode 100644 pytest.ini create mode 100755 todo.md diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..51527ed --- /dev/null +++ b/pytest.ini @@ -0,0 +1,7 @@ +[pytest] +addopts = --ignore=old_tests -s + +# collect_ignore = ["old_test"] + +[pycodestyle] +# ignore = old_test/* ALL diff --git a/todo.md b/todo.md new file mode 100755 index 0000000..e8f6c27 --- /dev/null +++ b/todo.md @@ -0,0 +1,612 @@ +# NOW + +# Media: +- rewrite set_date... + +# Test: +- finish filesystem +- date_taken +- geolocation + +move elodie to dozo + +check for early morning photos: add test + +add --folder-path option %Y-%d-%m/%city/%album +datetime.today().strftime('%Y-%m-%d') + +add %filename +add edit_exif command? + +Add update command + +# enhancement +- acccept Path in get_exiftool +- Use get_exiftool instead of get metadata: + try to do it in get_date_taken... +media class: +- Add self.file_path +- + +## Album form folder +- move to filesystem + # TODO implement album from folder here? + # folder = os.path.basename(os.path.dirname(source)) + # album = self.metadata['album'] + # if album_from_folder and (album is None or album == ''): + # album = folder +# Update +use pathlib instead of os.path + +Allow update in sort command in same dir if path is the dest dir + +ENhancement: swap hash db key value: for checking file integrity +https://github.com/JohannesBuchner/imagehash +https://github.com/cw-somil/Duplicate-Remover +https://leons.im/posts/a-python-implementation-of-simhash-algorithm/ + +Visualy check similar image +https://www.pluralsight.com/guides/importing-image-data-into-numpy-arrays +https://stackoverflow.com/questions/56056054/add-check-boxes-to-scrollable-image-in-python +https://wellsr.com/python/python-image-manipulation-with-pillow-library/ +kitty gird image? +https://fr.wikibooks.org/wiki/PyQt/PyQt_versus_wxPython +https://docs.python.org/3/faq/gui.html +https://docs.opencv.org/3.4/d3/df2/tutorial_py_basic_ops.html +https://stackoverflow.com/questions/52727332/python-tkinter-create-checkbox-list-from-listbox + + +Image gird method: +matplot +https://gist.github.com/lebedov/7018889ba47668c64bcf96aee82caec0 + +Tkinter +https://python-forum.io/thread-22700.html +https://stackoverflow.com/questions/43326282/how-can-i-use-images-in-a-tkinter-grid + +wxwidget +https://wxpython.org/Phoenix/docs/html/wx.lib.agw.thumbnailctrl.html + + +Ability to change metadata to selection + +Enhancement: Option to keep existing directory structure + + +Fix: change versvalidion number to 0.x99 +Fix: README + +Refactoring: elodie update: update metadata of destination + +Fix: update: fix move files... + +Refactoring: Move exiftool config + +Checksum: +FIX: test if checksum remain the same for all files (global check) +FIX: if dest file already here and checksum d'ont match change name to +prevent overwriting to file with same dest path + +Enhancement: media file, do not filter files, only to prevent error when copying +fix: Valid file: check for open file error + +Enhancement: Add %base_name string key + +Refactoring: class get_metadata +check if as exiF, check exif type... + +Interface: show error and warning +interface: less verbose when no error +interface: Move default setting to config? + +Behavior: Move only by defaut without changing metatdata and filename... + +Refactoring: check one time media is valid? +Refactoring: Unify source and path +Enhancement: allow nested dir +Fix: check exclusion for file +Refactoring: Import perl as submodule? + +Enhancement: # setup arguments to exiftool +https://github.com/andrewning/sortphotos/blob/master/src/sortphotos.py + +# AFTER +Enhancement: add walklevel function +Enhancement: change early morning date sort + +# TODO +Fix: date, make correction in filename if needed +Check: date from filename +Options: +--update-cache|-u +--date-from-filename +--location --time +# --date from folder +# --date from file +# -f overwrite metadata + +Add get tag function +Add --copy alternative +--auto|-a: a set of option: geolocalisation, best match date, rename, album +from folder... +defaut: only move +# --keep-folder option +# --rename +-- no cache mode!! +--confirm unsure operation +--interactive + + +# TEST + # lat='45.58339' + # lon='4.79823' + # coordinates ='53.480837, -2.244914' + # Alger + # coords=(36.752887, 3.042048) + +https://www.gitmemory.com/issue/pallets/click/843/634305917 +https://github.com/pallets/click/issues/843 + + # import unittest + # import pytest + + # from thing.__main__ import cli + + + # class TestCli(unittest.TestCase): + + # @pytest.fixture(autouse=True) + # def capsys(self, capsys): + # self.capsys = capsys + + # def test_cli(self): + # with pytest.raises(SystemExit) as ex: + # cli(["create", "--name", "test"]) + # self.assertEqual(ex.value.code, 0) + # out, err = self.capsys.readouterr() + # self.assertEqual(out, "Succesfully created test\n") + + +# dev +# mode ~/.elodie ~/.config/elodie +# location selection buggy + + +# TODO: +# /home/cedric/src/elodie/elodie/media/photo.py(86)get_date_taken() +# 85 # TODO potential bu for old photo below 1970... +# ---> 86 if(seconds_since_epoch == 0): +# 87 return None + + +import os + +def walklevel(some_dir, level=1): + some_dir = some_dir.rstrip(os.path.sep) + assert os.path.isdir(some_dir) + num_sep = some_dir.count(os.path.sep) + for root, dirs, files in os.walk(some_dir): + yield root, dirs, files + num_sep_this = root.count(os.path.sep) + if num_sep + level <= num_sep_this: + del dirs[:] +49/2: y=walklevel('/home/cedric', level=1) +49/3: next(y) +49/4: next(y) +49/5: next(y) +49/6: next(y) +49/7: next(y) +49/8: y=walklevel('/home/cedric', level=0) +49/9: next(y) +49/10: next(y) +49/11: y=walklevel('/home/cedric/.test/Nexcloud/', level=0) +49/12: +import os + +def walklevel(some_dir, level=1): + some_dir = some_dir.rstrip(os.path.sep) + assert os.path.isdir(some_dir) + num_sep = some_dir.count(os.path.sep) + for root, dirs, files in os.walk(some_dir): + yield root, dirs, files + num_sep_this = root.count(os.path.sep) + if num_sep + level <= num_sep_this: + print dirs, files +49/13: +import os + +def walklevel(some_dir, level=1): + some_dir = some_dir.rstrip(os.path.sep) + assert os.path.isdir(some_dir) + num_sep = some_dir.count(os.path.sep) + for root, dirs, files in os.walk(some_dir): + yield root, dirs, files + num_sep_this = root.count(os.path.sep) + if num_sep + level <= num_sep_this: + print(dirs, files) +49/14: y=walklevel('/home/cedric/.test/Nexcloud/', level=0) +49/15: next(y) +49/16: next(y) +49/17: y=walklevel('/home/cedric/.test/Nexcloud/', level=0) +49/18: +import os + +def walklevel(some_dir, level=1): + some_dir = some_dir.rstrip(os.path.sep) + assert os.path.isdir(some_dir) + num_sep = some_dir.count(os.path.sep) + for root, dirs, files in os.walk(some_dir): + yield root, dirs, files + num_sep_this = root.count(os.path.sep) +49/19: y=walklevel('/home/cedric/.test/Nexcloud/', level=0) +49/20: next(y) +49/21: next(y) +49/22: y=walklevel('/home/cedric/.test/Nexcloud/', level=2) +49/23: next(y) +49/24: next(y) +49/25: y=walklevel('/home/cedric/.test/las canarias 2012/', level=2) +49/26: next(y) +49/27: next(y) +49/28: next(y) +49/29: next(y) +49/30: y=walklevel('/home/cedric/.test/las canarias 2012/', level=0) +49/31: next(y) +49/32: next(y) +49/33: next(y) +49/34: +import os + +def walklevel(some_dir, level=1): + some_dir = some_dir.rstrip(os.path.sep) + assert os.path.isdir(some_dir) + num_sep = some_dir.count(os.path.sep) + for root, dirs, files in os.walk(some_dir): + yield root, dirs, files + num_sep_this = root.count(os.path.sep) + if num_sep + level <= num_sep_this: + print('fuck') +49/35: y=walklevel('/home/cedric/.test/las canarias 2012/', level=0) +49/36: next(y) +49/37: next(y) +49/38: next(y) +64/1: a=os.walk('/home/cedric/.test/las canarias 2012') +64/2: import os +64/3: a=os.walk('/home/cedric/.test/las canarias 2012') +64/4: next(a) +64/5: next(a) +64/6: os.path.sep +64/7: os.path.relpath('/home/cedric/.test/las canarias 2012/private', 'private') +64/8: os.path.relpath('/home/cedric/.test/las canarias 2012', 'private') +64/9: os.path.relpath('/home/cedric/.test/las canarias 2012/private', '/home/cedric/.test/las canarias 2012') +64/10: b='test' +64/11: a='private' +64/12: a+b +64/13: os.path.join(a,b,b) +64/14: !True +64/15: not True +64/16: a=TRue +64/17: a=True +64/18: not a +77/1: +import os +import requests + +def get_location(geotags): + coords = get_coordinates(geotags) + + uri = 'https://revgeocode.search.hereapi.com/v1/revgeocode' + headers = {} + params = { + 'apiKey': os.environ['API_KEY'], + 'at': "%s,%s" % coords, + 'lang': 'en-US', + 'limit': 1, + } + + response = requests.get(uri, headers=headers, params=params) + try: + response.raise_for_status() + return response.json() + + except requests.exceptions.HTTPError as e: + print(str(e)) + return {} +77/2: cd ~/.test/ +77/3: ls +77/4: cd 2021-02-Feb/ +77/5: ls +77/6: cd Villeurbanne/ +77/7: ls +77/8: ls -l +77/9: exif = get_exif('2021-02-24_09-33-29-20210305_081001_01.mp4') +77/10: +from PIL import Image + +def get_exif(filename): + image = Image.open(filename) + image.verify() + return image._getexif() +77/11: exif = get_exif('2021-02-24_09-33-29-20210305_081001_01.mp4') +77/12: .. +77/13: cd .. +77/14: ls +77/15: cd .. +77/16: ls +77/17: cd 2021-03-Mar/ +77/18: cd Villeurbanne/ +77/19: ls +77/20: exif = get_exif('2021-03-09_09-58-42-img_20210309_105842.jpg') +77/21: exif +77/22: +def get_geotagging(exif): + if not exif: + raise ValueError("No EXIF metadata found") + + geotagging = {} + for (idx, tag) in TAGS.items(): + if tag == 'GPSInfo': + if idx not in exif: + raise ValueError("No EXIF geotagging found") + + for (key, val) in GPSTAGS.items(): + if key in exif[idx]: + geotagging[val] = exif[idx][key] + + return geotagging +77/23: get_geotagging(exif) +77/24: from PIL.ExifTags import TAGS +77/25: +def get_labeled_exif(exif): + labeled = {} + for (key, val) in exif.items(): + labeled[TAGS.get(key)] = val + + return labeled +77/26: get_geotagging(exif) +77/27: from PIL.ExifTags import GPSTAGS +77/28: get_geotagging(exif) +77/29: geotags = get_geotagging(exif) +77/30: get_location(geotags) +77/31: +def get_decimal_from_dms(dms, ref): + + degrees = dms[0][0] / dms[0][1] + minutes = dms[1][0] / dms[1][1] / 60.0 + seconds = dms[2][0] / dms[2][1] / 3600.0 + + if ref in ['S', 'W']: + degrees = -degrees + minutes = -minutes + seconds = -seconds + + return round(degrees + minutes + seconds, 5) + +def get_coordinates(geotags): + lat = get_decimal_from_dms(geotags['GPSLatitude'], geotags['GPSLatitudeRef']) + + lon = get_decimal_from_dms(geotags['GPSLongitude'], geotags['GPSLongitudeRef']) + + return (lat,lon) +77/32: get_geotagging(exif) +77/33: get_location(geotags) +77/34: from geopy.geocoders import Here +78/1: from geopy.geocoders import Here +78/3: +78/4: get_exif +78/5: ls +78/6: cd ~/.test +78/7: ls +78/8: cd 2021-03-Mar/ +78/9: ls +78/10: cd Villeurbanne/ +78/11: get_exif('2021-03-04_11-50-32-img_20210304_125032.jpg') +78/12: exif=get_exif('2021-03-04_11-50-32-img_20210304_125032.jpg') +78/13: get_geotagging(exif) +78/14: +from PIL.ExifTags import GPSTAGS + +def get_geotagging(exif): + if not exif: + raise ValueError("No EXIF metadata found") + + geotagging = {} + for (idx, tag) in TAGS.items(): + if tag == 'GPSInfo': + if idx not in exif: + raise ValueError("No EXIF geotagging found") + + for (key, val) in GPSTAGS.items(): + if key in exif[idx]: + geotagging[val] = exif[idx][key] + + return geotagging +78/15: geotags = get_geotagging(exif) +78/17: geotags = get_geotagging(exif) +78/18: get_coordinates(geotags) +78/19: + +78/23: get_location(geotags) +78/24: +78/25: get_location(geotags) +78/26: +def get_decimal_from_dms(dms, ref): + + degrees = dms[0][0] / dms[0][1] + minutes = dms[1][0] / dms[1][1] / 60.0 + seconds = dms[2][0] / dms[2][1] / 3600.0 + + if ref in ['S', 'W']: + degrees = -degrees + minutes = -minutes + seconds = -seconds + + return round(degrees + minutes + seconds, 5) +78/27: get_location(geotags) +78/28: +def get_decimal_from_dms(dms, ref): + + degrees = dms[0] + minutes = dms[1] / 60.0 + seconds = dms[2] / 3600.0 + + if ref in ['S', 'W']: + degrees = -degrees + minutes = -minutes + seconds = -seconds + + return round(degrees + minutes + seconds, 5) +78/29: get_location(geotags) +78/30: exif +78/31: get_geotagging(exif) +78/32: geotags = get_geotagging(exif) +78/33: get_coordinates(geotags) +78/34: geotags = get_geotagging(exif) +78/35: get_location(geotags) +78/36: get_coordinates(geotags) +78/37: coords = get_coordinates(geotags) +78/38: coords +78/39: uri = 'https://revgeocode.search.hereapi.com/v1/revgeocode' +78/40: +headers = {} + params = { + 'apiKey': os.environ['API_KEY'], + 'at': "%s,%s" % coords, + 'lang': 'en-US', + 'limit': 1, + } +78/41: headers = {} +78/42: +params = { + 'apiKey': os.environ['API_KEY'], + 'at': "%s,%s" % coords, + 'lang': 'en-US', + 'limit': 1, + } +78/43: +params = { + 'apiKey': os.environ['API_KEY'], + 'at': "%s,%s" % coords, + 'lang': 'en-US', + 'limit': 1, + } +78/44: API_KEY=m5aGo8xGe4LLhxeKZYpHr2MPXGN2aDhe +78/45: API_KEY='m5aGo8xGe4LLhxeKZYpHr2MPXGN2aDhe' +78/46: +params = { + 'apiKey': os.environ['API_KEY'], + 'at': "%s,%s" % coords, + 'lang': 'en-US', + 'limit': 1, + } +78/47: API_KEY='m5aGo8xGe4LLhxeKZYpHr2MPXGN2aDhe' +78/48: +params = { + 'apiKey': os.environ['API_KEY'], + 'at': "%s,%s" % coords, + 'lang': 'en-US', + 'limit': 1, + } +78/49: +params = { + 'apiKey': os.environ['m5aGo8xGe4LLhxeKZYpHr2MPXGN2aDhe'], + 'at': "%s,%s" % coords, + 'lang': 'en-US', + 'limit': 1, + } +78/50: %load_ext autotime +78/51: +import pandas as pd +import geopandas as gpd +import geopy +from geopy.geocoders import Nominatim +from geopy.extra.rate_limiter import RateLimiterimport matplotlib.pyplot as plt +import plotly_express as pximport tqdm +from tqdm._tqdm_notebook import tqdm_notebook +78/52: +import pandas as pd +import geopandas as gpd +import geopy +from geopy.geocoders import Nominatim +from geopy.extra.rate_limiter import RateLimiterimport matplotlib.pyplot as plt +import plotly_express as px +import pandas as pd +import geopandas as gpd + + + + + + +from PIL import Image + +filename='2021-02-24_09-33-29-20210305_081001_01.mp4' +def get_exif(filename): + image = Image.open(filename) + image.verify() + return image._getexif() +exif=get_exif(filename) + +from PIL.ExifTags import TAGS +from PIL.ExifTags import GPSTAGS +def get_geotagging(exif): + if not exif: + raise ValueError("No EXIF metadata found") + + geotagging = {} + for (idx, tag) in TAGS.items(): + if tag == 'GPSInfo': + if idx not in exif: + raise ValueError("No EXIF geotagging found") + + for (key, val) in GPSTAGS.items(): + if key in exif[idx]: + geotagging[val] = exif[idx][key] + + return geotagging +geotags = get_geotagging(exif) +import os +import requests + +def get_location(geotags): + coords = get_coordinates(geotags) + + uri = 'https://revgeocode.search.hereapi.com/v1/revgeocode' + headers = {} + params = { + 'apiKey': os.environ['API_KEY'], + 'at': "%s,%s" % coords, + 'lang': 'en-US', + 'limit': 1, + } + + response = requests.get(uri, headers=headers, params=params) + try: + response.raise_for_status() + return response.json() + + except requests.exceptions.HTTPError as e: + print(str(e)) + return {} +def get_coordinates(geotags): + lat = get_decimal_from_dms(geotags['GPSLatitude'], geotags['GPSLatitudeRef']) + + lon = get_decimal_from_dms(geotags['GPSLongitude'], geotags['GPSLongitudeRef']) + + return (lat,lon) +coords = get_coordinates(geotags) +import geopy +from geopy.geocoders import Nominatim +locator = Nominatim(user_agent='myGeocoder') +# coordinates ='53.480837, -2.244914' +lat='45.58339' +lon='4.79823' +coords = lat + ',' + lon +locator.reverse(coords) +location =locator.reverse(coords) +location.address.split(',') +city=location.address.split(',')[1].strip() +country=location.address.split(',')[7].strip() +location.raw +rint +country=location.raw['address']['country'] +city=location.raw['address']['village']