From d2ecd0ed3d424bcd3e7d447d5510db8582ae212c Mon Sep 17 00:00:00 2001 From: Cedric Leporcq Date: Fri, 16 Apr 2021 20:22:27 +0200 Subject: [PATCH] use geopy lib and use Nominatum as default geocoder --- config.ini-sample | 3 +- elodie/geolocation.py | 122 ++++++++++++++++++++++--------- elodie/tests/config_test.py | 10 +-- elodie/tests/filesystem_test.py | 8 +- elodie/tests/geolocation_test.py | 26 +++---- 5 files changed, 111 insertions(+), 58 deletions(-) diff --git a/config.ini-sample b/config.ini-sample index 45a21fa..efd3d77 100644 --- a/config.ini-sample +++ b/config.ini-sample @@ -1,4 +1,5 @@ [Geolocation] +; geocoder: Nominatim or MapQuest geocoder=Nominatim -mapquest_key=m5aGo8xGe4LLhxeKZYpHr2MPXGN2aDhe +mapquest_key=None prefer_english_names=False diff --git a/elodie/geolocation.py b/elodie/geolocation.py index dd5e03e..d4ce5c5 100644 --- a/elodie/geolocation.py +++ b/elodie/geolocation.py @@ -12,6 +12,8 @@ import requests import urllib.request import urllib.parse import urllib.error +import geopy +from geopy.geocoders import Nominatim from elodie.config import load_config from elodie import constants @@ -34,37 +36,50 @@ def coordinates_by_name(name): } # If the name is not cached then we go ahead with an API lookup - geolocation_info = lookup(location=name) - - if(geolocation_info is not None): - if( - 'results' in geolocation_info and - len(geolocation_info['results']) != 0 and - 'locations' in geolocation_info['results'][0] and - len(geolocation_info['results'][0]['locations']) != 0 - ): - - # By default we use the first entry unless we find one with - # geocodeQuality=city. - geolocation_result = geolocation_info['results'][0] - use_location = geolocation_result['locations'][0]['latLng'] - # Loop over the locations to see if we come accross a - # geocodeQuality=city. - # If we find a city we set that to the use_location and break - for location in geolocation_result['locations']: - if( - 'latLng' in location and - 'lat' in location['latLng'] and - 'lng' in location['latLng'] and - location['geocodeQuality'].lower() == 'city' - ): - use_location = location['latLng'] - break - + geocoder = get_geocoder() + if geocoder == 'Nominatim': + locator = Nominatim(user_agent='myGeocoder') + geolocation_info = locator.geocode(name) + if geolocation_info is not None: return { - 'latitude': use_location['lat'], - 'longitude': use_location['lng'] + 'latitude': geolocation_info.latitude, + 'longitude': geolocation_info.longitude } + elif geocoder == 'MapQuest': + geolocation_info = lookup_mapquest(location=name) + + if(geolocation_info is not None): + if( + 'results' in geolocation_info and + len(geolocation_info['results']) != 0 and + 'locations' in geolocation_info['results'][0] and + len(geolocation_info['results'][0]['locations']) != 0 + ): + + # By default we use the first entry unless we find one with + # geocodeQuality=city. + geolocation_result = geolocation_info['results'][0] + use_location = geolocation_result['locations'][0]['latLng'] + # Loop over the locations to see if we come accross a + # geocodeQuality=city. + # If we find a city we set that to the use_location and break + for location in geolocation_result['locations']: + if( + 'latLng' in location and + 'lat' in location['latLng'] and + 'lng' in location['latLng'] and + location['geocodeQuality'].lower() == 'city' + ): + use_location = location['latLng'] + break + + return { + 'latitude': use_location['lat'], + 'longitude': use_location['lng'] + } + + else: + return None return None @@ -99,6 +114,17 @@ def dms_string(decimal, type='latitude'): return '{} deg {}\' {}" {}'.format(dms[0], dms[1], dms[2], direction) +def get_geocoder(): + config = load_config(constants.CONFIG_FILE) + try: + geocoder = config['Geolocation']['geocoder'] + except KeyError as e: + log.error(e) + return None + + return geocoder + + def get_key(): global __KEY__ if __KEY__ is not None: @@ -148,12 +174,19 @@ def place_name(lat, lon): return cached_place_name lookup_place_name = {} - geolocation_info = lookup(lat=lat, lon=lon) + geocoder = get_geocoder() + if geocoder == 'Nominatim': + geolocation_info = lookup_osm(lat, lon) + elif geocoder == 'MapQuest': + geolocation_info = lookup_mapquest(lat=lat, lon=lon) + else: + return None + if(geolocation_info is not None and 'address' in geolocation_info): address = geolocation_info['address'] # gh-386 adds support for town # taking precedence after city for backwards compatability - for loc in ['city', 'town', 'state', 'country']: + for loc in ['city', 'town', 'village', 'state', 'country']: if(loc in address): lookup_place_name[loc] = address[loc] # In many cases the desired key is not available so we @@ -171,8 +204,27 @@ def place_name(lat, lon): return lookup_place_name +def lookup_osm(lat, lon): -def lookup(**kwargs): + prefer_english_names = get_prefer_english_names() + from geopy.geocoders import Nominatim + try: + locator = Nominatim(user_agent='myGeocoder') + coords = (lat, lon) + if(prefer_english_names): + lang='en' + else: + lang='local' + return locator.reverse(coords, language=lang).raw + except geopy.exc.GeocoderUnavailable as e: + log.error(e) + return None + except ValueError as e: + log.error(e) + return None + + +def lookup_mapquest(**kwargs): if( 'location' not in kwargs and 'lat' not in kwargs and @@ -180,14 +232,14 @@ def lookup(**kwargs): ): return None - key = get_key() + mapquest_key = get_key() prefer_english_names = get_prefer_english_names() - if(key is None): + if(mapquest_key is None): return None try: - params = {'format': 'json', 'key': key} + params = {'format': 'json', 'key': mapquest_key} params.update(kwargs) path = '/geocoding/v1/address' if('lat' in kwargs and 'lon' in kwargs): diff --git a/elodie/tests/config_test.py b/elodie/tests/config_test.py index 94d4822..01f8acd 100755 --- a/elodie/tests/config_test.py +++ b/elodie/tests/config_test.py @@ -18,18 +18,18 @@ from elodie.config import load_config, load_plugin_config def test_load_config_singleton_success(): with open('%s/config.ini-singleton-success' % gettempdir(), 'w') as f: f.write(""" -[MapQuest] +[Geolocation] key=your-api-key-goes-here prefer_english_names=False """) if hasattr(load_config, 'config'): del load_config.config - config = load_config() - assert config['MapQuest']['key'] == 'your-api-key-goes-here', config.get('MapQuest', 'key') + config = load_config(constants.CONFIG_FILE) + assert config['Geolocation']['key'] == 'your-api-key-goes-here', config.get('MapQuest', 'key') config.set('MapQuest', 'key', 'new-value') - config = load_config() + config = load_config(constants.CONFIG_FILE) if hasattr(load_config, 'config'): del load_config.config @@ -41,7 +41,7 @@ def test_load_config_singleton_no_file(): if hasattr(load_config, 'config'): del load_config.config - config = load_config() + config = load_config(constants.CONFIG_FILE) if hasattr(load_config, 'config'): del load_config.config diff --git a/elodie/tests/filesystem_test.py b/elodie/tests/filesystem_test.py index ed6ada9..683807f 100644 --- a/elodie/tests/filesystem_test.py +++ b/elodie/tests/filesystem_test.py @@ -527,8 +527,8 @@ def test_get_folder_path_with_original_default_unknown_location(): def test_get_folder_path_with_custom_path(): with open('%s/config.ini-custom-path' % gettempdir(), 'w') as f: f.write(""" -[MapQuest] -key=czjNKTtFjLydLteUBwdgKAIC8OAbGLUx +[Geolocation] +mapquest_key=czjNKTtFjLydLteUBwdgKAIC8OAbGLUx [Directory] date=%Y-%m-%d @@ -569,8 +569,8 @@ full_path=%year/%month/%album|%"No Album Fool"/%month def test_get_folder_path_with_with_more_than_two_levels(): with open('%s/config.ini-location-date' % gettempdir(), 'w') as f: f.write(""" -[MapQuest] -key=czjNKTtFjLydLteUBwdgKAIC8OAbGLUx +[Geolocation] +mapquest_key=czjNKTtFjLydLteUBwdgKAIC8OAbGLUx [Directory] year=%Y diff --git a/elodie/tests/geolocation_test.py b/elodie/tests/geolocation_test.py index 5b75436..5f476c0 100644 --- a/elodie/tests/geolocation_test.py +++ b/elodie/tests/geolocation_test.py @@ -80,47 +80,47 @@ def test_dms_string_longitude(): assert str(dms[0]) in dms_string, '%s not in %s' % (dms[0], dms_string) def test_reverse_lookup_with_valid_key(): - res = geolocation.lookup(lat=37.368, lon=-122.03) + res = geolocation.lookup_osm(lat=37.368, lon=-122.03) assert res['address']['city'] == 'Sunnyvale', res def test_reverse_lookup_with_invalid_lat_lon(): - res = geolocation.lookup(lat=999, lon=999) + res = geolocation.lookup_osm(lat=999, lon=999) assert res is None, res @mock.patch('elodie.geolocation.__KEY__', 'invalid_key') def test_reverse_lookup_with_invalid_key(): - res = geolocation.lookup(lat=37.368, lon=-122.03) + res = geolocation.lookup_mapquest(lat=37.368, lon=-122.03) assert res is None, res def test_lookup_with_valid_key(): - res = geolocation.lookup(location='Sunnyvale, CA') + res = geolocation.lookup_mapquest(location='Sunnyvale, CA') latLng = res['results'][0]['locations'][0]['latLng'] assert latLng['lat'] == 37.36883, latLng assert latLng['lng'] == -122.03635, latLng def test_lookup_with_invalid_location(): - res = geolocation.lookup(location='foobar dne') + res = geolocation.lookup_mapquest(location='foobar dne') assert res is None, res def test_lookup_with_invalid_location(): - res = geolocation.lookup(location='foobar dne') + res = geolocation.lookup_mapquest(location='foobar dne') assert res is None, res def test_lookup_with_valid_key(): - res = geolocation.lookup(location='Sunnyvale, CA') + res = geolocation.lookup_mapquest(location='Sunnyvale, CA') latLng = res['results'][0]['locations'][0]['latLng'] assert latLng['lat'] == 37.36883, latLng assert latLng['lng'] == -122.03635, latLng @mock.patch('elodie.geolocation.__PREFER_ENGLISH_NAMES__', True) def test_lookup_with_prefer_english_names_true(): - res = geolocation.lookup(lat=55.66333, lon=37.61583) - assert res['address']['city'] == 'Nagorny District', res + res = geolocation.lookup_osm(lat=55.66333, lon=37.61583) + assert res['address']['city'] == 'Moscow', res @mock.patch('elodie.geolocation.__PREFER_ENGLISH_NAMES__', False) def test_lookup_with_prefer_english_names_false(): - res = geolocation.lookup(lat=55.66333, lon=37.61583) - assert res['address']['city'] == u'\u041d\u0430\u0433\u043e\u0440\u043d\u044b\u0439 \u0440\u0430\u0439\u043e\u043d', res + res = geolocation.lookup_osm(lat=55.66333, lon=37.61583) + assert res['address']['city'] == 'Москва', res @mock.patch('elodie.constants.location_db', '%s/location.json-cached' % gettempdir()) def test_place_name_deprecated_string_cached(): @@ -159,12 +159,12 @@ def test_place_name_no_default(): @mock.patch('elodie.geolocation.__KEY__', 'invalid_key') def test_lookup_with_invalid_key(): - res = geolocation.lookup(location='Sunnyvale, CA') + res = geolocation.lookup_mapquest(location='Sunnyvale, CA') assert res is None, res @mock.patch('elodie.geolocation.__KEY__', '') def test_lookup_with_no_key(): - res = geolocation.lookup(location='Sunnyvale, CA') + res = geolocation.lookup_mapquest(location='Sunnyvale, CA') assert res is None, res def test_parse_result_with_error():