staged.

Chunk
1 parent bbd2f705
Showing 4 changed files with 21 additions and 14 deletions Show diff stats
mdata/ILSVRC.py
res/cropped_pil.jpg
test/test_data.py
test/test_jpeg.py
@@ -279,20 +279,21 @@ class DataILSVRC(DataDumperBase):
                 W, H = size
                 try:
                     im = Image.open(image)
+                    qt = im.quantization
                     w, h = im.size
                     if w < W or h < H:
                         continue
                     left, upper = random.randint(0, w - W), random.randint(0, h - H)
                     im = im.crop((left, upper, left + W, upper + H))
-                    im.save(os.path.join(self.data_dir + '_crop_pil', name))
+                    im.save(os.path.join(self.data_dir + '_crop_pil', name), qtables=qt)
                 except Exception as e:
                     print '[EXCPT]', e
                     pass
  
                     # try:
                     # img = cv2.imread(image, cv2.CV_LOAD_IMAGE_UNCHANGED)
-                    #     h, w = img.shape[:2]
-                    #     if w < 300 or h < 300:
+                    # h, w = img.shape[:2]
+                    # if w < 300 or h < 300:
                     #         continue
                     #     left, upper = random.randint(0, w - 300), random.randint(0, h - 300)
                     #     img_crop = img[upper:upper + 300, left:left + 300]
@@ -453,7 +454,11 @@ class DataILSVRC(DataDumperBase):
                         image = os.path.join(self.img_dir, hash[:3], hash[3:] + '.jpg')
                         if image:
                             im = Jpeg(image, key=sample_key)
-                            dict_dataset[hash] = (tag, im.getCoefBlocks('Y'))
+                            dict_dataset[hash] = (tag, im.getCoefMatrix(channel='Y'))
+
+                for tag, feat in dict_dataset.values():
+                    X.append(feat.tolist())
+                    Y.append(int(tag))
  
             else:
                 with open(self.list_file, 'rb') as tsvfile:
@@ -466,10 +471,10 @@ class DataILSVRC(DataDumperBase):
                             with open(path_feat, 'rb') as featfile:
                                 dict_dataset[hash] = (tag, json.loads(featfile.read()))
  
-            for tag, feat in dict_dataset.values():
-                # X.append([item for sublist in feat for subsublist in sublist for item in subsublist])
-                X.append(np.array(feat).ravel().tolist())
-                Y.append(int(tag))
+                for tag, feat in dict_dataset.values():
+                    # X.append([item for sublist in feat for subsublist in sublist for item in subsublist])
+                    X.append(np.array(feat).ravel().tolist())
+                    Y.append(int(tag))
  
         elif mode == "hbase":  # remote
             if self.table == None:
@@ -115,6 +115,7 @@ def test_ILSVRC_S_SPARK():
     dils._extract_feat(mode='spark', feattype='ibd', readforward=False, writeback=True, withdata=True)
     timer.report()
  
+
 def test_ILSVRC_S():
     # test_ILSVRC_S_LOCAL()
     test_ILSVRC_S_SPARK()
@@ -137,17 +138,18 @@ def test_pipeline():
 def test_crop():
     # crop.crop_Test()
  
-    # dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Test_1')
-    # dil.crop()
+    dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Test_1')
+    dil.crop()
  
     dil2 = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Test_1_crop_pil')
  
-    # dil2.format()
-    # dil2.embed(rate=0.2)
+    dil2.format()
+    dil2.embed(rate=0.2)
  
-    X,Y = dil2.load_data(mode='local',feattype='coef')
+    X, Y = dil2.load_data(mode='local', feattype='coef')
     print X[0]
     print Y
+    print np.array(X).shape, np.array(Y).shape
  
  
 if __name__ == '__main__':
@@ -151,7 +151,7 @@ def test_iter():
  
  
 def test_jpeg():
-    ima = mjpeg.Jpeg(os.path.join(package_dir, "../res/50c488a2b163ca8a1f52da6022f03.jpg"), key=sample_key)
+    ima = mjpeg.Jpeg(os.path.join(package_dir, "../res/cropped_pil.jpg"), key=sample_key)
     print ima.getQuality()
     print ima.getCapacity('All')
     sys.exit(0)
...	...	@@ -279,20 +279,21 @@ class DataILSVRC(DataDumperBase):
279	279	W, H = size
280	280	try:
281	281	im = Image.open(image)
	282	+ qt = im.quantization
282	283	w, h = im.size
283	284	if w < W or h < H:
284	285	continue
285	286	left, upper = random.randint(0, w - W), random.randint(0, h - H)
286	287	im = im.crop((left, upper, left + W, upper + H))
287		- im.save(os.path.join(self.data_dir + '_crop_pil', name))
	288	+ im.save(os.path.join(self.data_dir + '_crop_pil', name), qtables=qt)
288	289	except Exception as e:
289	290	print '[EXCPT]', e
290	291	pass
291	292
292	293	# try:
293	294	# img = cv2.imread(image, cv2.CV_LOAD_IMAGE_UNCHANGED)
294		- # h, w = img.shape[:2]
295		- # if w < 300 or h < 300:
	295	+ # h, w = img.shape[:2]
	296	+ # if w < 300 or h < 300:
296	297	# continue
297	298	# left, upper = random.randint(0, w - 300), random.randint(0, h - 300)
298	299	# img_crop = img[upper:upper + 300, left:left + 300]
...	...	@@ -453,7 +454,11 @@ class DataILSVRC(DataDumperBase):
453	454	image = os.path.join(self.img_dir, hash[:3], hash[3:] + '.jpg')
454	455	if image:
455	456	im = Jpeg(image, key=sample_key)
456		- dict_dataset[hash] = (tag, im.getCoefBlocks('Y'))
	457	+ dict_dataset[hash] = (tag, im.getCoefMatrix(channel='Y'))
	458	+
	459	+ for tag, feat in dict_dataset.values():
	460	+ X.append(feat.tolist())
	461	+ Y.append(int(tag))
457	462
458	463	else:
459	464	with open(self.list_file, 'rb') as tsvfile:
...	...	@@ -466,10 +471,10 @@ class DataILSVRC(DataDumperBase):
466	471	with open(path_feat, 'rb') as featfile:
467	472	dict_dataset[hash] = (tag, json.loads(featfile.read()))
468	473
469		- for tag, feat in dict_dataset.values():
470		- # X.append([item for sublist in feat for subsublist in sublist for item in subsublist])
471		- X.append(np.array(feat).ravel().tolist())
472		- Y.append(int(tag))
	474	+ for tag, feat in dict_dataset.values():
	475	+ # X.append([item for sublist in feat for subsublist in sublist for item in subsublist])
	476	+ X.append(np.array(feat).ravel().tolist())
	477	+ Y.append(int(tag))
473	478
474	479	elif mode == "hbase": # remote
475	480	if self.table == None:
...	...
...	...	@@ -115,6 +115,7 @@ def test_ILSVRC_S_SPARK():
115	115	dils._extract_feat(mode='spark', feattype='ibd', readforward=False, writeback=True, withdata=True)
116	116	timer.report()
117	117
	118	+
118	119	def test_ILSVRC_S():
119	120	# test_ILSVRC_S_LOCAL()
120	121	test_ILSVRC_S_SPARK()
...	...	@@ -137,17 +138,18 @@ def test_pipeline():
137	138	def test_crop():
138	139	# crop.crop_Test()
139	140
140		- # dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Test_1')
141		- # dil.crop()
	141	+ dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Test_1')
	142	+ dil.crop()
142	143
143	144	dil2 = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Test_1_crop_pil')
144	145
145		- # dil2.format()
146		- # dil2.embed(rate=0.2)
	146	+ dil2.format()
	147	+ dil2.embed(rate=0.2)
147	148
148		- X,Y = dil2.load_data(mode='local',feattype='coef')
	149	+ X, Y = dil2.load_data(mode='local', feattype='coef')
149	150	print X[0]
150	151	print Y
	152	+ print np.array(X).shape, np.array(Y).shape
151	153
152	154
153	155	if __name__ == '__main__':
...	...
...	...	@@ -151,7 +151,7 @@ def test_iter():
151	151
152	152
153	153	def test_jpeg():
154		- ima = mjpeg.Jpeg(os.path.join(package_dir, "../res/50c488a2b163ca8a1f52da6022f03.jpg"), key=sample_key)
	154	+ ima = mjpeg.Jpeg(os.path.join(package_dir, "../res/cropped_pil.jpg"), key=sample_key)
155	155	print ima.getQuality()
156	156	print ima.getCapacity('All')
157	157	sys.exit(0)
...	...