Add files via upload

jannemus · Feb 28, 2019 · 37613c5 · 37613c5
1 parent c13bef2
commit 37613c5
Show file tree

Hide file tree

Showing 43 changed files with 528 additions and 0 deletions.
diff --git a/deepblind.py b/deepblind.py
@@ -0,0 +1,46 @@
+import os, os.path, errno
+import argparse
+from PIL import Image
+import numpy as np
+from keras.models import Model
+from keras.preprocessing.image import array_to_img
+from models import modelsClass
+
+# Parse input arguments
+desc = "DeepBlind - Blind deblurring method."
+parser = argparse.ArgumentParser(description=desc)
+parser.add_argument('-i', '--input', type=str, metavar='', required=True, help='input folder')
+args = parser.parse_args()
+inpath = args.input
+
+# Deblurred images will be saved to 'output' folder
+outpath = "output"
+try:
+    os.makedirs(outpath)
+except OSError as e:
+    if e.errno != errno.EEXIST:
+        raise
+
+blurred_path = inpath + "/blurred/"
+blurred_names = os.listdir(blurred_path)
+
+for fname in blurred_names:
+
+    print("Deblurring '%s' with DeepBlind" %(fname))
+
+    blurred_img = Image.open(blurred_path + fname)
+    blurred_np = (1./255)*np.array(blurred_img)
+
+    width, height = blurred_img.size
+    models = modelsClass(height,width)
+    model = models.getDeepBlind()
+    model.load_weights("checkpoints/DeepBlind.hdf5")
+
+    x = np.reshape(blurred_np,[1,height,width,3])
+    prediction = model.predict(x, batch_size=1,verbose=0,steps=None)
+    prediction = prediction[0,:,:,:]
+
+    deblurred_img = array_to_img(prediction)
+    deblurred_img.save(outpath+"/%s"%(fname))
+
+print("DONE!")
diff --git a/deepgyro.py b/deepgyro.py
@@ -0,0 +1,60 @@
+import os, os.path, errno
+import argparse
+from PIL import Image
+import numpy as np
+from keras.models import Model
+from keras.preprocessing.image import array_to_img
+from models import modelsClass
+
+# Parse input arguments
+desc = "DeepGyro - Gyro-aided deblurring method."
+parser = argparse.ArgumentParser(description=desc)
+parser.add_argument('-i', '--input', type=str, metavar='', required=True, help='input folder')
+args = parser.parse_args()
+inpath = args.input
+
+# Deblurred images will be saved to 'output' folder
+outpath = "output"
+try:
+    os.makedirs(outpath)
+except OSError as e:
+    if e.errno != errno.EEXIST:
+        raise
+
+blurred_path = inpath + "/blurred/"
+blurx_path = inpath + "/blurx/"
+blury_path = inpath + "/blury/"
+
+blurred_names = os.listdir(blurred_path)
+blurx_names = os.listdir(blurx_path)
+blury_names = os.listdir(blury_path)
+
+for fname in blurred_names:
+
+    print("Deblurring '%s' with DeepGyro" %(fname))
+
+    blurred_img = Image.open(blurred_path + fname)
+    blurx_img = Image.open(blurx_path + fname)
+    blury_img = Image.open(blury_path + fname)
+
+    blurred_np = (1./255)*np.array(blurred_img)
+    blurx_np = (1./255)*np.array(blurx_img)
+    blury_np = (1./255)*np.array(blury_img)
+
+    width, height = blurred_img.size
+    models = modelsClass(height,width)
+    model = models.getDeepGyro()
+    model.load_weights("checkpoints/DeepGyro.hdf5")
+
+    b = np.reshape(blurred_np,[1,height,width,3])
+    bx = np.reshape(blurx_np,[1,height,width,1])
+    by = np.reshape(blury_np,[1,height,width,1]) 
+    x = [b,bx,by]
+
+    prediction = model.predict(x, batch_size=1,verbose=0,steps=None)
+    prediction = prediction[0,:,:,:]
+
+    deblurred_img = array_to_img(prediction)
+    deblurred_img.save(outpath+"/%s"%(fname))
+
+print("DONE!")
diff --git a/input/paper/blurred/bridge.png b/input/paper/blurred/bridge.png
diff --git a/input/paper/blurred/car.png b/input/paper/blurred/car.png
diff --git a/input/paper/blurred/church.png b/input/paper/blurred/church.png
diff --git a/input/paper/blurred/entrance.png b/input/paper/blurred/entrance.png
diff --git a/input/paper/blurred/flower.png b/input/paper/blurred/flower.png
diff --git a/input/paper/blurred/graffiti.png b/input/paper/blurred/graffiti.png
diff --git a/input/paper/blurred/office.png b/input/paper/blurred/office.png
diff --git a/input/paper/blurred/posters.png b/input/paper/blurred/posters.png
diff --git a/input/paper/blurred/tower.png b/input/paper/blurred/tower.png
diff --git a/input/paper/blurx/bridge.png b/input/paper/blurx/bridge.png
diff --git a/input/paper/blurx/car.png b/input/paper/blurx/car.png
diff --git a/input/paper/blurx/church.png b/input/paper/blurx/church.png
diff --git a/input/paper/blurx/entrance.png b/input/paper/blurx/entrance.png
diff --git a/input/paper/blurx/flower.png b/input/paper/blurx/flower.png
diff --git a/input/paper/blurx/graffiti.png b/input/paper/blurx/graffiti.png
diff --git a/input/paper/blurx/office.png b/input/paper/blurx/office.png
diff --git a/input/paper/blurx/posters.png b/input/paper/blurx/posters.png
diff --git a/input/paper/blurx/tower.png b/input/paper/blurx/tower.png
diff --git a/input/paper/blury/bridge.png b/input/paper/blury/bridge.png
diff --git a/input/paper/blury/car.png b/input/paper/blury/car.png
diff --git a/input/paper/blury/church.png b/input/paper/blury/church.png
diff --git a/input/paper/blury/entrance.png b/input/paper/blury/entrance.png
diff --git a/input/paper/blury/flower.png b/input/paper/blury/flower.png
diff --git a/input/paper/blury/graffiti.png b/input/paper/blury/graffiti.png
diff --git a/input/paper/blury/office.png b/input/paper/blury/office.png
diff --git a/input/paper/blury/posters.png b/input/paper/blury/posters.png
diff --git a/input/paper/blury/tower.png b/input/paper/blury/tower.png
diff --git a/preprocess/IO.py b/preprocess/IO.py
@@ -0,0 +1,76 @@
+import argparse
+import numpy as np
+import os
+import glob
+from PIL import Image
+
+def parseInputs():
+    desc = "Estimate gyro-based blur fields."
+    parser = argparse.ArgumentParser(description=desc)
+    parser.add_argument('-i', '--input', type=str, metavar='', required=True, help='input folder')
+    parser.add_argument('-o', '--output', type=str, metavar='', required=True, help='output folder')
+    args = parser.parse_args()
+    return args.input, args.output
+
+def readGyroscope(inpath):
+    datapath = inpath + '/imu/imu.txt'
+    imu = np.loadtxt(datapath, dtype='float_', delimiter=' ')
+
+    # The first column indicates the sensors type
+    gyr_idx = imu[:,0].astype(np.int_) == 4
+
+    # Extract gyroscope timestamps and readings
+    tgyr = imu[gyr_idx,1]
+    gyr = imu[gyr_idx,2:5]
+
+    return gyr, tgyr
+
+def readImageInfo(inpath):
+    datapath = inpath + '/images/images.txt'
+    info = np.loadtxt(datapath, dtype='float_', delimiter=' ')
+
+    # Extract timestamps and exposure times
+    tf = info[:,0]
+    te = info[:,1]
+
+    return tf, te
+
+def readImage(inpath, scaling, idx):
+
+    fnames = []
+    for ext in ('*.jpg', '*.png'):
+        datapath = inpath + '/images/' + ext
+        fnames.extend(sorted(glob.glob(datapath)))
+
+    if idx < len(fnames):
+        img = Image.open(fnames[idx])
+    else:
+        raise ValueError('Could not read image with index: %d' %idx)
+
+    # Downsample
+    if scaling < 1.0:
+        w = int(scaling*img.size[0])
+        h = int(scaling*img.size[1])
+        img = img.resize((w,h), resample=Image.BICUBIC)
+
+    img = np.array(img)
+
+    return img
+
+
+def writeImage(img, outpath, folder, idx):
+
+    fname = '%04d.png' %(idx)
+    img = Image.fromarray(img.astype(np.uint8))
+    img.save(outpath + '/' + folder + '/' + fname)
+
+def createOutputFolders(outpath):
+
+    try:
+        os.makedirs(outpath + '/blurred')
+        os.makedirs(outpath + '/blurx')
+        os.makedirs(outpath + '/blury')
+        os.makedirs(outpath + '/visualization/')
+    except FileExistsError:
+        # Directory already exists
+        pass
diff --git a/preprocess/calibration.py b/preprocess/calibration.py
@@ -0,0 +1,32 @@
+import numpy as np
+
+'''
+   Specify the calibration information. The following
+   parameters are for the NVIDIA Shield tablet.
+'''
+
+# Downsampling factor. For example, the image
+# resolution is halved when scaling is set to 0.5
+scaling = 0.5
+
+# Camera intrinsics [fx 0 cx; 0 fy cy; 0 0 1] at
+# the original resolution.
+K = np.array([[1558.6899, 0, 939.6533],
+			  [0, 1558.6899, 518.4131],
+			  [0, 0, 1]])
+
+# Rotation from IMU frame to camera frame. 
+# Obtained using the Android documentation.
+Ri = np.array([[0, 1, 0],
+			   [1, 0, 0],
+			   [0, 0, 1]])
+
+# Camera readout time (rolling shutter skew) in seconds. The 
+# time difference between the first and last row exposure. Can 
+# be obtained using the Android camera2 API or via calibration.
+tr = 0.0244944
+
+# IMU-camera temporal offset in seconds. If 'td' is set to zero,
+# it is assumed that the first gyroscope measurement in 'imu.txt' 
+# corresponds to the start of the first image exposure in 'images.txt'.
+td = 0.022
diff --git a/preprocess/generate.py b/preprocess/generate.py
@@ -0,0 +1,71 @@
+import numpy as np
+
+# Load camera and IMU calibration information
+import calibration as calib
+
+import IO    # For reading and writing data
+from utils import *
+from visualize import plotBlurVectors
+
+if __name__ == '__main__':
+
+    inpath, outpath = IO.parseInputs()
+    print("Input folder: %s" %inpath)
+
+    ''' Read input data and calibration parameters '''
+
+    # Read gyroscope measurements and timestamps
+    gyr, tgyr = IO.readGyroscope(inpath)
+
+    # Read image timestamps and exposure times
+    tf, te = IO.readImageInfo(inpath)
+
+    # Load calibration parameters from calibration.py
+    scaling = calib.scaling # Downsample images?
+    K = calib.K             # Camera intrinsics
+    tr = calib.tr           # Camera readout time
+    td = calib.td           # IMU-camera temporal offset
+    Ri = calib.Ri           # IMU-to-camera rotation
+
+    # If we downsample, we also need to scale intrinsics
+    if scaling < 1.0:
+        K = scaling*K
+        K[2,2] = 1
+
+    IO.createOutputFolders(outpath)
+
+    ''' Temporal and spatial alignment of gyroscope and camera '''
+
+    # Read the first image to get image dimensions
+    img = IO.readImage(inpath, scaling, idx=0)
+    height, width = img.shape[:2]
+
+    dt = tr/height # Sampling interval
+    gyr = alignSpatial(gyr, Ri)
+    gyr, t, tf, te = alignTemporal(gyr, tgyr, tf, te, tr, td, dt)
+
+    ''' Generate blur field for each image '''
+
+    num_images = tf.shape[0]
+    for i in range(num_images):
+
+        print("Processing image: %d/%d" %(i+1,num_images))
+        img = IO.readImage(inpath, scaling, idx=i)
+
+        # Start and end of the exposure            
+        t1 = tf[i]           
+        t2 = t1 + te[i] + tr
+
+        # Convert from seconds to samples
+        n1 = int(t1/dt)
+        n2 = int(t2/dt)
+        ne = int(te[i]/dt)
+
+        R = computeRotations(gyr[n1:n2+2,:],t[n1:n2+2])
+        Bx, By = computeBlurfield(R,K,ne,height,width)
+
+        IO.writeImage(img, outpath, 'blurred/', idx=i)
+        IO.writeImage(Bx, outpath, 'blurx/', idx=i)
+        IO.writeImage(By, outpath, 'blury/', idx=i)
+
+        plotBlurVectors(Bx, By, img, outpath, idx=i) # Optional
diff --git a/preprocess/myrawdata/images/0001.jpg b/preprocess/myrawdata/images/0001.jpg
diff --git a/preprocess/myrawdata/images/0002.jpg b/preprocess/myrawdata/images/0002.jpg
diff --git a/preprocess/myrawdata/images/0003.jpg b/preprocess/myrawdata/images/0003.jpg
diff --git a/preprocess/myrawdata/images/0004.jpg b/preprocess/myrawdata/images/0004.jpg
diff --git a/preprocess/myrawdata/images/0005.jpg b/preprocess/myrawdata/images/0005.jpg
diff --git a/preprocess/myrawdata/images/0006.jpg b/preprocess/myrawdata/images/0006.jpg
diff --git a/preprocess/myrawdata/images/0007.jpg b/preprocess/myrawdata/images/0007.jpg
diff --git a/preprocess/myrawdata/images/images.txt b/preprocess/myrawdata/images/images.txt
@@ -0,0 +1,7 @@
+767700989000 20000000
+767734321000 20000000
+767767659000 20000000
+767800985000 20000000
+767834314000 20000000
+767867646000 20000000
+767900977000 20000000
diff --git a/preprocess/myrawdata/imu/imu.txt b/preprocess/myrawdata/imu/imu.txt
@@ -0,0 +1,69 @@
+2 767706256650 -60.6192169 -3.9757228 -7.3631215
+1 767705436983 8.0077066 -0.9913492 5.7260327
+4 767705436983 -3.1111047 -0.1292299 -0.5319997
+2 767716233983 -60.4727745 -3.8075621 -7.3008876
+1 767715432233 7.9023619 -0.9099463 5.8672905
+4 767715432233 -3.1451933 -0.1441436 -0.5277386
+2 767726222816 -60.3438301 -3.8833742 -7.2483864
+1 767725427483 7.9670053 -0.8524854 6.3030353
+4 767725427483 -3.1643679 -0.1516005 -0.5202818
+2 767736234483 -60.0780144 -3.9270744 -7.3297014
+1 767735422733 8.0843210 -0.9362825 6.4849944
+4 767735422733 -3.1707597 -0.1792973 -0.5149555
+2 767746218983 -59.6881142 -3.6943893 -7.4142895
+1 767745417983 8.1729069 -0.8955811 6.5951281
+4 767745417983 -3.1803470 -0.2144510 -0.5106944
+2 767756244733 -59.3532448 -3.3312271 -7.2535682
+1 767755413233 8.2662802 -1.0176854 6.6262527
+4 767755413233 -3.1824775 -0.2357563 -0.5117597
+2 767766274400 -59.1258049 -2.9924865 -6.9035754
+1 767765408483 8.3548660 -1.0847230 6.6023107
+4 767765408483 -3.1835427 -0.2517353 -0.5224124
+2 767776211733 -59.0912018 -2.6433845 -6.5250487
+1 767775403733 8.3381071 -1.2307693 6.4826002
+4 767775403733 -3.1856732 -0.2645185 -0.5405218
+2 767786217733 -59.3485451 -2.3510182 -6.2152510
+1 767785398983 8.2375498 -1.3576620 6.4299278
+4 767785398983 -3.1941953 -0.2613227 -0.5586313
+2 767796252650 -59.7686234 -2.1807425 -5.9847574
+1 767795394233 8.1393881 -1.2762592 6.3844380
+4 767795394233 -3.2027175 -0.2496048 -0.5724798
+2 767806220566 -60.0941772 -2.0653670 -5.8006511
+1 767805389483 8.0316486 -1.2642882 6.3652844
+4 767805389483 -3.2218924 -0.2400174 -0.5831324
+2 767816280483 -60.2210541 -1.9025455 -5.6974144
+1 767815384733 7.9119387 -1.2642882 6.3413424
+4 767815384733 -3.2453282 -0.2357563 -0.5863282
+2 767826204483 -60.2650528 -1.6602150 -5.6383901
+1 767825379983 7.8209591 -1.2235867 6.5041480
+4 767825379983 -3.2804818 -0.2325605 -0.5841976
+2 767836189566 -60.2756004 -1.3833429 -5.5230832
+1 767835375233 7.6988549 -1.1397897 7.0380549
+4 767835375233 -3.3220272 -0.2474742 -0.5746103
+2 767846250566 -60.1344604 -1.1522064 -5.1439009
+1 767845370483 7.7204027 -1.1733085 6.8489132
+4 767845370483 -3.3614419 -0.2986069 -0.5554355
+2 767856219900 -59.8226204 -1.0183632 -4.3648520
+1 767855365733 7.7970171 -1.2355577 7.0141129
+4 767855365733 -3.4008567 -0.3305649 -0.5341302
+2 767866403150 -59.5800705 -0.9203643 -3.4673924
+1 767865360983 7.8353243 -1.1900679 6.9063740
+4 767865360983 -3.4402714 -0.3476091 -0.5149555
+2 767876206816 -59.6561775 -0.7698016 -2.7578530
+1 767875356233 7.8161707 -1.2523171 6.7603278
+4 767875356233 -3.4743600 -0.3540007 -0.4947155
+2 767886221733 -59.8174438 -0.6610575 -2.3601809
+1 767885351483 7.8281417 -1.2044331 6.7052612
+4 767885351483 -3.5191011 -0.3412175 -0.4787365
+2 767896233399 -59.7498970 -0.7348601 -2.1827543
+1 767895346733 7.9071503 -1.2451345 6.8848262
+4 767895346733 -3.5659726 -0.3231080 -0.4606270
+2 767906171316 -59.4379768 -0.8366951 -1.9823223
+1 767905341983 7.8879967 -1.3983635 6.9901710
+4 767905341983 -3.6085832 -0.3188470 -0.4425175
+2 767916336066 -59.0042801 -0.8055584 -1.6398404
+1 767915337233 7.8449011 -1.6306010 6.8393364
+4 767915337233 -3.6639769 -0.3284343 -0.4276038
+2 767926184483 -58.6799431 -0.7170786 -1.1339971
+1 767925332483 7.7060375 -1.7167922 6.7746930
+4 767925332483 -3.7310886 -0.3316301 -0.4094943