Use PIL(pillow) as a fallback mechanism image format identification and add HEIC support (#320)

Fixes #281 and #269

When elodie imports images, imghdr.what is used to determine the image
type. imghdr implementation won't support all JPEG variants. Because
of this, even for a valid JPEG file, imghdr.what returns None causing
elodie to skip the image during import.

This commit adds a fallback mechanism which uses PIL(pillow) library to
identify image formats. When imghdr fails, elodie uses pillow to
identify image format. When pillow is unavailable, it will be treated as
an invalid media file. Pillow is imported only when imghdr fails.

.travis.yml

@@ -7,7 +7,7 @@ virtualenv:
   system_site_packages: true
 before_install:
   - "sudo apt-get update -qq"
-  - "sudo apt-get install python-dev python-pip libimage-exiftool-perl -y"
+  - "sudo apt-get install python-dev python-pip -y"
 install:
   - "pip install -r elodie/tests/requirements.txt"
   - "pip install coveralls"
@@ -19,9 +19,9 @@ before_script:
   - "sed -i.bak 's/debug = False/debug = True/g' elodie/constants.py"
   # Get exiftool 10.20 installed
   - "export ELODIE_DIR=${PWD}"
-  - "wget http://www.sno.phy.queensu.ca/~phil/exiftool/Image-ExifTool-10.20.tar.gz"
+  - "wget http://www.sno.phy.queensu.ca/~phil/exiftool/Image-ExifTool-11.50.tar.gz"
-  - "gzip -dc Image-ExifTool-10.20.tar.gz  | tar -xf -"
+  - "gzip -dc Image-ExifTool-11.50.tar.gz  | tar -xf -"
-  - "cd Image-ExifTool-10.20"
+  - "cd Image-ExifTool-11.50"
   - "perl Makefile.PL"
   - "sudo make install"
   - "cd ${ELODIE_DIR}"

Readme.md

@@ -13,7 +13,7 @@ Getting started takes just a few minutes.
 
 Elodie relies on the great [ExifTool library by Phil Harvey](http://www.sno.phy.queensu.ca/~phil/exiftool/). You'll need to install it for your platform.
 
-Some features for video files will only work with newer versions of ExifTool and have been tested on version 10.20 or higher. Check your version by typing `exiftool -ver` and see the [manual installation instructions for ExifTool](http://www.sno.phy.queensu.ca/~phil/exiftool/install.html#Unix) if needed.
+Some features for video files will only work with newer versions of ExifTool and have been tested on version 10.20 or higher. Support for HEIC files requires version 11.50 or higher. Check your version by typing `exiftool -ver` and see the [manual installation instructions for ExifTool](http://www.sno.phy.queensu.ca/~phil/exiftool/install.html#Unix) if needed.
 
 ```
 # OSX (uses homebrew, http://brew.sh/)

elodie/media/photo.py

@@ -12,6 +12,7 @@ import os
 import re
 import time
 from datetime import datetime
+from PIL import Image
 from re import compile
 
 
@@ -29,7 +30,7 @@ class Photo(Media):
     __name__ = 'Photo'
 
     #: Valid extensions for photo files.
-    extensions = ('arw', 'cr2', 'dng', 'gif', 'jpeg', 'jpg', 'nef', 'rw2')
+    extensions = ('arw', 'cr2', 'dng', 'gif', 'heic', 'jpeg', 'jpg', 'nef', 'rw2')
 
     def __init__(self, source=None):
         super(Photo, self).__init__(source)
@@ -91,9 +92,28 @@ class Photo(Media):
         """
         source = self.source
 
-        # gh-4 This checks if the source file is an image.
+        # HEIC is not well supported yet so we special case it.
-        # It doesn't validate against the list of supported types.
+        # https://github.com/python-pillow/Pillow/issues/2806
-        if(imghdr.what(source) is None):
+        extension = os.path.splitext(source)[1][1:].lower()
-            return False
+        if(extension != 'heic'):
+            # gh-4 This checks if the source file is an image.
+            # It doesn't validate against the list of supported types.
+            if(imghdr.what(source) is None):
+                # imghdr won't detect all variants of images (https://bugs.python.org/issue28591)
+                # see https://github.com/jmathai/elodie/issues/281
+                # before giving up, we use `pillow` imaging library to detect file type
+                #
+                # It is important to note that the library doesn't decode or load the
+                # raster data unless it really has to. When you open a file,
+                # the file header is read to determine the file format and extract
+                # things like mode, size, and other properties required to decode the file,
+                # but the rest of the file is not processed until later.
+                try:
+                    im = Image.open(source)
+                except IOError:
+                    return False
 
-        return os.path.splitext(source)[1][1:].lower() in self.extensions
+                if(im.format is None):
+                    return False
+
+        return extension in self.extensions

elodie/tests/media/photo_test.py

@@ -28,6 +28,7 @@ def test_photo_extensions():
     assert 'cr2' in extensions
     assert 'dng' in extensions
     assert 'gif' in extensions
+    assert 'heic' in extensions
     assert 'jpg' in extensions
     assert 'jpeg' in extensions
     assert 'nef' in extensions
@@ -159,6 +160,12 @@ def test_is_not_valid():
 
     assert not photo.is_valid()
 
+def test_is_valid_fallback_using_pillow():
+    photo = Photo(helper.get_file('imghdr-error.jpg'))
+
+    assert photo.is_valid()
+
+
 def test_set_album():
     temporary_folder, folder = helper.create_working_folder()
 
@@ -330,6 +337,7 @@ def test_various_types():
         'arw': (2007, 4, 8, 17, 41, 18, 6, 98, 0),
         'cr2': (2005, 10, 29, 16, 14, 44, 5, 302, 0),
         'dng': (2009, 10, 20, 9, 10, 46, 1, 293, 0),
+        'heic': (2019, 5, 26, 10, 33, 20, 6, 146, 0),
         'nef': (2008, 10, 24, 9, 12, 56, 4, 298, 0),
         'rw2': (2014, 11, 19, 23, 7, 44, 2, 323, 0)
     }

requirements.txt

@@ -4,3 +4,4 @@ Send2Trash==1.3.0
 future==0.16.0
 configparser==3.5.0
 tabulate==0.7.7
+Pillow==5.3.0