Add improved copy method for large files #173 (#189)

elodie/compatability.py

@@ -1,3 +1,8 @@
+import os
+import shutil
+
+from elodie import constants
+
 def _decode(string, encoding='utf8'):
     """Return a utf8 encoded unicode string.
 
@@ -17,4 +22,30 @@ def _decode(string, encoding='utf8'):
 
     return string
 
+def _copyfile(src, dst):
+    # Python 3 hangs using open/write method
+    if (constants.python_version == 3):
+        shutil.copy(src, dst)
+        return
 
+    try:
+        O_BINARY = os.O_BINARY
+    except:
+        O_BINARY = 0
+
+    READ_FLAGS = os.O_RDONLY | O_BINARY
+    WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | O_BINARY
+    TEN_MEGABYTES = 10485760
+    BUFFER_SIZE = min(TEN_MEGABYTES, os.path.getsize(src))
+        
+    try:
+        fin = os.open(src, READ_FLAGS)
+        stat = os.fstat(fin)
+        fout = os.open(dst, WRITE_FLAGS, stat.st_mode)
+        for x in iter(lambda: os.read(fin, BUFFER_SIZE), ""):
+            os.write(fout, x)
+    finally:
+        try: os.close(fin)
+        except: pass
+        try: os.close(fout)
+        except: pass

elodie/constants.py

@@ -3,6 +3,7 @@ Settings used by Elodie.
 """
 
 from os import path
+from sys import version_info
 
 #: If True, debug messages will be printed.
 debug = False
@@ -24,3 +25,6 @@ exiftool_config = path.join(script_directory, 'configs', 'ExifTool_config')
 
 #: Accepted language in responses from MapQuest
 accepted_language = 'en'
+
+# check python version, required in filesystem.py to trigger appropriate method
+python_version = version_info.major

elodie/filesystem.py

@@ -11,6 +11,7 @@ import re
 import shutil
 import time
 
+from elodie import compatability
 from elodie import geolocation
 from elodie import log
 from elodie.config import load_config
@@ -333,7 +334,10 @@ class FileSystem(object):
             # Do not use copy2(), will have an issue when copying to a
             # network/mounted drive using copy and manual
             # set_date_from_filename gets the job done
-            shutil.copy(_file, dest_path)
+            # Do not use copy2(), will have an issue when copying to a network/mounted drive
+            # shutil.copy seems slow, changing to streaming according to
+            # http://stackoverflow.com/questions/22078621/python-how-to-copy-files-fast
+            compatability._copyfile(_file, dest_path)
             self.set_utime(media)
 
         db.add_hash(checksum, dest_path)