Commit b9990e77ddc8aa4dbce2753d13eb0c319a139f1b
1 parent
bbd2f705
Exists in
master
and in
1 other branch
staged.
Showing
4 changed files
with
21 additions
and
14 deletions
Show diff stats
mdata/ILSVRC.py
| @@ -279,20 +279,21 @@ class DataILSVRC(DataDumperBase): | @@ -279,20 +279,21 @@ class DataILSVRC(DataDumperBase): | ||
| 279 | W, H = size | 279 | W, H = size |
| 280 | try: | 280 | try: |
| 281 | im = Image.open(image) | 281 | im = Image.open(image) |
| 282 | + qt = im.quantization | ||
| 282 | w, h = im.size | 283 | w, h = im.size |
| 283 | if w < W or h < H: | 284 | if w < W or h < H: |
| 284 | continue | 285 | continue |
| 285 | left, upper = random.randint(0, w - W), random.randint(0, h - H) | 286 | left, upper = random.randint(0, w - W), random.randint(0, h - H) |
| 286 | im = im.crop((left, upper, left + W, upper + H)) | 287 | im = im.crop((left, upper, left + W, upper + H)) |
| 287 | - im.save(os.path.join(self.data_dir + '_crop_pil', name)) | 288 | + im.save(os.path.join(self.data_dir + '_crop_pil', name), qtables=qt) |
| 288 | except Exception as e: | 289 | except Exception as e: |
| 289 | print '[EXCPT]', e | 290 | print '[EXCPT]', e |
| 290 | pass | 291 | pass |
| 291 | 292 | ||
| 292 | # try: | 293 | # try: |
| 293 | # img = cv2.imread(image, cv2.CV_LOAD_IMAGE_UNCHANGED) | 294 | # img = cv2.imread(image, cv2.CV_LOAD_IMAGE_UNCHANGED) |
| 294 | - # h, w = img.shape[:2] | ||
| 295 | - # if w < 300 or h < 300: | 295 | + # h, w = img.shape[:2] |
| 296 | + # if w < 300 or h < 300: | ||
| 296 | # continue | 297 | # continue |
| 297 | # left, upper = random.randint(0, w - 300), random.randint(0, h - 300) | 298 | # left, upper = random.randint(0, w - 300), random.randint(0, h - 300) |
| 298 | # img_crop = img[upper:upper + 300, left:left + 300] | 299 | # img_crop = img[upper:upper + 300, left:left + 300] |
| @@ -453,7 +454,11 @@ class DataILSVRC(DataDumperBase): | @@ -453,7 +454,11 @@ class DataILSVRC(DataDumperBase): | ||
| 453 | image = os.path.join(self.img_dir, hash[:3], hash[3:] + '.jpg') | 454 | image = os.path.join(self.img_dir, hash[:3], hash[3:] + '.jpg') |
| 454 | if image: | 455 | if image: |
| 455 | im = Jpeg(image, key=sample_key) | 456 | im = Jpeg(image, key=sample_key) |
| 456 | - dict_dataset[hash] = (tag, im.getCoefBlocks('Y')) | 457 | + dict_dataset[hash] = (tag, im.getCoefMatrix(channel='Y')) |
| 458 | + | ||
| 459 | + for tag, feat in dict_dataset.values(): | ||
| 460 | + X.append(feat.tolist()) | ||
| 461 | + Y.append(int(tag)) | ||
| 457 | 462 | ||
| 458 | else: | 463 | else: |
| 459 | with open(self.list_file, 'rb') as tsvfile: | 464 | with open(self.list_file, 'rb') as tsvfile: |
| @@ -466,10 +471,10 @@ class DataILSVRC(DataDumperBase): | @@ -466,10 +471,10 @@ class DataILSVRC(DataDumperBase): | ||
| 466 | with open(path_feat, 'rb') as featfile: | 471 | with open(path_feat, 'rb') as featfile: |
| 467 | dict_dataset[hash] = (tag, json.loads(featfile.read())) | 472 | dict_dataset[hash] = (tag, json.loads(featfile.read())) |
| 468 | 473 | ||
| 469 | - for tag, feat in dict_dataset.values(): | ||
| 470 | - # X.append([item for sublist in feat for subsublist in sublist for item in subsublist]) | ||
| 471 | - X.append(np.array(feat).ravel().tolist()) | ||
| 472 | - Y.append(int(tag)) | 474 | + for tag, feat in dict_dataset.values(): |
| 475 | + # X.append([item for sublist in feat for subsublist in sublist for item in subsublist]) | ||
| 476 | + X.append(np.array(feat).ravel().tolist()) | ||
| 477 | + Y.append(int(tag)) | ||
| 473 | 478 | ||
| 474 | elif mode == "hbase": # remote | 479 | elif mode == "hbase": # remote |
| 475 | if self.table == None: | 480 | if self.table == None: |
13.9 KB
test/test_data.py
| @@ -115,6 +115,7 @@ def test_ILSVRC_S_SPARK(): | @@ -115,6 +115,7 @@ def test_ILSVRC_S_SPARK(): | ||
| 115 | dils._extract_feat(mode='spark', feattype='ibd', readforward=False, writeback=True, withdata=True) | 115 | dils._extract_feat(mode='spark', feattype='ibd', readforward=False, writeback=True, withdata=True) |
| 116 | timer.report() | 116 | timer.report() |
| 117 | 117 | ||
| 118 | + | ||
| 118 | def test_ILSVRC_S(): | 119 | def test_ILSVRC_S(): |
| 119 | # test_ILSVRC_S_LOCAL() | 120 | # test_ILSVRC_S_LOCAL() |
| 120 | test_ILSVRC_S_SPARK() | 121 | test_ILSVRC_S_SPARK() |
| @@ -137,17 +138,18 @@ def test_pipeline(): | @@ -137,17 +138,18 @@ def test_pipeline(): | ||
| 137 | def test_crop(): | 138 | def test_crop(): |
| 138 | # crop.crop_Test() | 139 | # crop.crop_Test() |
| 139 | 140 | ||
| 140 | - # dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Test_1') | ||
| 141 | - # dil.crop() | 141 | + dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Test_1') |
| 142 | + dil.crop() | ||
| 142 | 143 | ||
| 143 | dil2 = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Test_1_crop_pil') | 144 | dil2 = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Test_1_crop_pil') |
| 144 | 145 | ||
| 145 | - # dil2.format() | ||
| 146 | - # dil2.embed(rate=0.2) | 146 | + dil2.format() |
| 147 | + dil2.embed(rate=0.2) | ||
| 147 | 148 | ||
| 148 | - X,Y = dil2.load_data(mode='local',feattype='coef') | 149 | + X, Y = dil2.load_data(mode='local', feattype='coef') |
| 149 | print X[0] | 150 | print X[0] |
| 150 | print Y | 151 | print Y |
| 152 | + print np.array(X).shape, np.array(Y).shape | ||
| 151 | 153 | ||
| 152 | 154 | ||
| 153 | if __name__ == '__main__': | 155 | if __name__ == '__main__': |
test/test_jpeg.py
| @@ -151,7 +151,7 @@ def test_iter(): | @@ -151,7 +151,7 @@ def test_iter(): | ||
| 151 | 151 | ||
| 152 | 152 | ||
| 153 | def test_jpeg(): | 153 | def test_jpeg(): |
| 154 | - ima = mjpeg.Jpeg(os.path.join(package_dir, "../res/50c488a2b163ca8a1f52da6022f03.jpg"), key=sample_key) | 154 | + ima = mjpeg.Jpeg(os.path.join(package_dir, "../res/cropped_pil.jpg"), key=sample_key) |
| 155 | print ima.getQuality() | 155 | print ima.getQuality() |
| 156 | print ima.getCapacity('All') | 156 | print ima.getCapacity('All') |
| 157 | sys.exit(0) | 157 | sys.exit(0) |