-
Notifications
You must be signed in to change notification settings - Fork 17
/
correspondence_database.py
337 lines (284 loc) · 13.7 KB
/
correspondence_database.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
import random
import time
import numpy as np
import os
import cv2
from skimage.io import imread, imsave
from dataset.GL3D_dataset import get_gl3d_dataset
from utils.base_utils import perspective_transform, read_pickle, save_pickle
def worker_init_fn(worker_id):
np.random.seed(worker_id + (int(round(time.time() * 1000) % (2 ** 16))))
def scale_transform_img(img, min_ratio=0.15, max_ratio=0.25, base_ratio=2, flip=True):
h, w = img.shape[0], img.shape[1]
pts0 = np.asarray([[0, 0], [w, 0], [w, h], [0, h]], np.float32)
scale_ratio = base_ratio ** (-np.random.uniform(min_ratio, max_ratio))
if np.random.random() < 0.5 and flip:
scale_ratio = 1.0 / scale_ratio
center = np.mean(pts0, 0, keepdims=True)
pts1 = (pts0 - center) * scale_ratio + center
if scale_ratio > 1:
min_pt = np.min(pts1, 0) # <0
max_pt = np.max(pts1, 0) # >w,h
min_w, min_h = -(max_pt - np.asarray([w, h]))
max_w, max_h = -min_pt
else:
min_pt = np.min(pts1, 0) # >0
max_pt = np.max(pts1, 0) # <w,h
min_w, min_h = -min_pt
max_w, max_h = np.asarray([w, h]) - max_pt
offset_h = np.random.uniform(min_h, max_h)
offset_w = np.random.uniform(min_w, max_w)
pts1 += np.asarray([[offset_w, offset_h]], np.float32)
th, tw = h, w # int(h * scale_ratio), int(w * scale_ratio)
H = cv2.getPerspectiveTransform(pts0.astype(np.float32), pts1.astype(np.float32))
img1 = cv2.warpPerspective(img, H, (tw, th), flags=cv2.INTER_LINEAR)
return img1, H
def perspective_transform_img(img, perspective_type='lr', min_ratio=0.05, max_ratio=0.1):
h, w = img.shape[0], img.shape[1]
pts0 = np.asarray([[0, 0], [w, 0], [w, h], [0, h]], np.float32)
pts1 = np.asarray([[0, 0], [w, 0], [w, h], [0, h]], np.float32)
# left right
if perspective_type == 'lr':
val = h * np.random.uniform(min_ratio, max_ratio)
if np.random.random() < 0.5: val *= -1
pts1[0, 1] -= val
pts1[1, 1] += val
pts1[2, 1] -= val
pts1[3, 1] += val
val = h * np.random.uniform(min_ratio, max_ratio)
pts1[0, 0] += val
pts1[1, 0] -= val
pts1[2, 0] -= val
pts1[3, 0] += val
else: # 'ud'
val = w * np.random.uniform(min_ratio, max_ratio)
if np.random.random() < 0.5: val *= -1
pts1[0, 0] += val
pts1[1, 0] -= val
pts1[2, 0] += val
pts1[3, 0] -= val
val = h * np.random.uniform(min_ratio, max_ratio)
pts1[0, 1] += val
pts1[1, 1] += val
pts1[2, 1] -= val
pts1[3, 1] -= val
pts1 = pts1 - np.min(pts1, 0, keepdims=True)
tw, th = np.max(pts1, 0)
H = cv2.getPerspectiveTransform(pts0.astype(np.float32), pts1.astype(np.float32))
img1 = cv2.warpPerspective(img, H, (tw, th), flags=cv2.INTER_LINEAR)
return img1, H
def rotate_transform_img(img, min_angle=0, max_angle=360, random_flip=False):
h, w = img.shape[0], img.shape[1]
pts0 = np.asarray([[0, 0], [w, 0], [w, h], [0, h]], np.float32)
center = np.mean(pts0, 0, keepdims=True)
theta = np.random.uniform(min_angle / 180 * np.pi, max_angle / 180 * np.pi)
if random_flip and np.random.random() < 0.5: theta = -theta
R = np.asarray([[np.cos(theta), -np.sin(theta)], [np.sin(theta), np.cos(theta)]], np.float32)
pts1 = (pts0 - center) @ R + center
H = cv2.getPerspectiveTransform(pts0.astype(np.float32), pts1.astype(np.float32))
img1 = cv2.warpPerspective(img, H, (w, h))
return img1, H
class CorrespondenceDatabase:
@staticmethod
def get_SUN2012_image_paths():
img_dir = os.path.join('data', 'SUN2012Images', 'JPEGImages')
img_pths = [os.path.join(img_dir, fn) for fn in os.listdir(img_dir)]
return img_pths
@staticmethod
def get_COCO_image_paths():
img_dir = os.path.join('data', 'coco', 'train2014')
img_pths = [os.path.join(img_dir, fn) for fn in os.listdir(img_dir)]
img_dir = os.path.join('data', 'coco', 'val2014')
img_pths += [os.path.join(img_dir, fn) for fn in os.listdir(img_dir)]
return img_pths
@staticmethod
def generate_homography_database(img_list):
return [{'type': 'homography', 'img_pth': img_pth} for img_pth in img_list]
@staticmethod
def get_hpatch_sequence_database(name='resize', max_size=480):
"""
Get hpatches_resize if it exists,
else generate one
"""
def resize_and_save(pth_in, max_size, pth_out):
img = imread(pth_in)
h, w = img.shape[:2]
ratio = max_size / max(h, w)
h, w = int(h * ratio), int(w * ratio)
img = cv2.resize(img, (w, h))
imsave(pth_out, img)
return ratio
root_dir = os.path.join('data', 'hpatches_sequence')
output_dir = os.path.join('data', 'hpatches_{}'.format(name))
pkl_file = os.path.join(output_dir, 'info.pkl')
if os.path.exists(pkl_file):
return read_pickle(pkl_file)
if not os.path.exists(output_dir): os.mkdir(output_dir)
illumination_dataset = []
viewpoint_dataset = []
for dir in os.listdir(root_dir):
if not os.path.exists(os.path.join(output_dir, dir)):
os.mkdir(os.path.join(output_dir, dir))
img_pattern = os.path.join(root_dir, dir, '{}.ppm')
hmg_pattern = os.path.join(root_dir, dir, 'H_1_{}')
omg_pattern = os.path.join(output_dir, dir, '{}.png')
ratio0 = resize_and_save(img_pattern.format(1), max_size, omg_pattern.format(1))
# resize image
for k in range(2, 7):
ratio1 = resize_and_save(img_pattern.format(k), max_size, omg_pattern.format(k))
H = np.loadtxt(hmg_pattern.format(k))
H = np.matmul(np.diag([ratio1, ratio1, 1.0]), np.matmul(H, np.diag([1 / ratio0, 1 / ratio0, 1.0])))
data = {'type': 'hpatch',
'img0_pth': omg_pattern.format(1),
'img1_pth': omg_pattern.format(k),
'H': H}
if dir.startswith('v'):
viewpoint_dataset.append(data)
if dir.startswith('i'):
illumination_dataset.append(data)
save_pickle([illumination_dataset, viewpoint_dataset], pkl_file)
return illumination_dataset, viewpoint_dataset
@staticmethod
def add_homography_background(img, H):
img_dir = os.path.join('data', 'SUN2012Images', 'JPEGImages')
background_pths = [os.path.join(img_dir, fn) for fn in os.listdir(img_dir)]
bpth = background_pths[np.random.randint(0, len(background_pths))]
h, w, _ = img.shape
bimg = cv2.resize(imread(bpth), (w, h))
if len(bimg.shape) == 2: bimg = np.repeat(bimg[:, :, None], 3, axis=2)
if bimg.shape[2] > 3: bimg = bimg[:, :, :3]
msk_tgt = cv2.warpPerspective(np.ones([h, w], np.uint8), H, (w, h), flags=cv2.INTER_NEAREST).astype(np.bool)
img[np.logical_not(msk_tgt)] = bimg[np.logical_not(msk_tgt)]
return img
@staticmethod
def make_hpatch_transform_database_combine(in_dataset, output_name, transform, add_background, identity=False):
hpatch_transform_root_dir = os.path.join('data', 'hpatches_{}'.format(output_name))
if not os.path.exists(hpatch_transform_root_dir): os.mkdir(hpatch_transform_root_dir)
hpatch_transform_pkl = os.path.join(hpatch_transform_root_dir, 'info.pkl')
if os.path.exists(hpatch_transform_pkl):
return read_pickle(hpatch_transform_pkl)
print('begin making {} dataset'.format(output_name))
dataset = []
random.shuffle(in_dataset)
for in_data in in_dataset:
pth0 = in_data['img0_pth']
pth1 = in_data['img1_pth']
in_dir = pth1.split('/')[-2]
in_id = pth1.split('/')[-1].split('.')[-2]
output_dir = os.path.join(hpatch_transform_root_dir, in_dir)
if not os.path.exists(output_dir): os.mkdir(output_dir)
img1 = imread(pth1)
img1, H = transform(img1)
if add_background: img1 = CorrespondenceDatabase.add_homography_background(img1, H)
img1_pth = os.path.join(output_dir, '{}.png'.format(in_id))
imsave(img1_pth, img1)
if identity:
data = {'type': 'hpatch',
'img0_pth': pth1,
'img1_pth': img1_pth,
'H': H}
else:
data = {'type': 'hpatch',
'img0_pth': pth0,
'img1_pth': img1_pth,
'H': H @ in_data['H']}
dataset.append(data)
save_pickle(dataset, hpatch_transform_pkl)
return dataset
@staticmethod
def warp_flow(H, norm_flow):
h, w, _ = norm_flow.shape
xs, ys = np.meshgrid(np.arange(w), np.arange(h))
pts = np.concatenate([xs.reshape([-1, 1]), ys.reshape([-1, 1])], 1)
norm_flow = norm_flow.reshape([h * w, 2])
nan_mask = np.sum(np.isnan(norm_flow), 1) > 0
norm_flow[nan_mask] = (0, 0)
norm_flow *= np.asarray((w, h))
norm_flow += pts
outside_mask = np.logical_or(np.logical_or(norm_flow[:, 0] < 0, norm_flow[:, 0] >= w),
np.logical_or(norm_flow[:, 1] < 0, norm_flow[:, 1] >= h))
norm_flow = perspective_transform(norm_flow, H)
norm_flow -= pts
norm_flow /= np.asarray((w, h))
norm_flow[nan_mask] = np.nan
norm_flow[outside_mask] = np.nan
norm_flow = norm_flow.reshape([h, w, 2])
return norm_flow
@staticmethod
def make_sun3d_all_dataset(img_num=499):
dataset = []
for k in range(img_num):
data = {
'img0_pth': 'data/sun3d_all/img0/{}.png'.format(k),
'img1_pth': 'data/sun3d_all/img1/{}.png'.format(k),
'flow_pth': 'data/sun3d_all/flow_01/{}.npy'.format(k),
'type': 'sun3d_all'
}
dataset.append(data)
return dataset
@staticmethod
def make_transformed_sun3d_dataset(dataset_in, dataset_out_name, warp_fn, add_background=True):
dir_out = os.path.join('data', dataset_out_name)
if not os.path.exists(dir_out): os.mkdir(dir_out)
info_pkl = os.path.join(dir_out, 'info.pkl')
if os.path.exists(info_pkl):
return read_pickle(info_pkl)
if not os.path.exists(os.path.join(dir_out, 'img1')):
os.mkdir(os.path.join(dir_out, 'img1'))
if not os.path.exists(os.path.join(dir_out, 'flow_01')):
os.mkdir(os.path.join(dir_out, 'flow_01'))
print('begin making {} dataset'.format(dataset_out_name))
dataset_out = []
for data_in in dataset_in:
image_id = data_in['img0_pth'].split('/')[-1].split('.')[0]
img1 = imread(data_in['img1_pth']).astype(np.uint8)
flow = np.load(data_in['flow_pth']).astype(np.float32).transpose([1, 2, 0])
img1, H = warp_fn(img1)
if add_background:
img1 = CorrespondenceDatabase.add_homography_background(img1, H)
flow_warp = CorrespondenceDatabase.warp_flow(H, flow)
data_out = {
'img0_pth': data_in['img0_pth'],
'img1_pth': os.path.join(dir_out, 'img1', '{}.png'.format(image_id)),
'flow_pth': os.path.join(dir_out, 'flow_01', '{}.npy'.format(image_id)),
'type': 'sun3d_all'
}
imsave(data_out['img1_pth'], img1)
np.save(data_out['flow_pth'], flow_warp.astype(np.float32))
dataset_out.append(data_out)
save_pickle(dataset_out, info_pkl)
return dataset_out
def __getattr__(self, item):
if item == 'coco_set':
self.coco_set = self.generate_homography_database(self.get_COCO_image_paths())
print('coco_len {}'.format(len(self.coco_set)))
return self.coco_set
elif item== 'hi_set' or item=='hv_set':
self.hi_set, self.hv_set = self.get_hpatch_sequence_database()
if item=='hi_set': return self.hi_set
else: return self.hv_set
elif item.startswith('erotate') or item.startswith('escale'):
scaling = lambda img: scale_transform_img(img, 1.5, 2.0)
rotating = lambda img: rotate_transform_img(img, -90, 90, True)
escale_set = self.make_hpatch_transform_database_combine(
self.hv_set, 'escale', scaling, True)
escale_illm_set = self.make_hpatch_transform_database_combine(
self.hi_set, 'escale_illm', scaling, True)
self.escale_set=escale_set+escale_illm_set
erotate_set = self.make_hpatch_transform_database_combine(
self.hv_set, 'erotate', rotating, True)
erotate_illm_set = self.make_hpatch_transform_database_combine(
self.hi_set, 'erotate_illm', rotating, True)
self.erotate_set=erotate_set+erotate_illm_set
if item=='erotate_set':
return self.erotate_set
else:
return self.escale_set
elif item=='gl3d_set':
self.gl3d_set=get_gl3d_dataset()
print('gl3d len {}'.format(len(self.gl3d_set)))
return self.gl3d_set
else:
super(CorrespondenceDatabase, self).__getattribute__(item)
def __init__(self):
pass