Commit 35cf2e3a65bffa59197629a0c54cf244d5773456
1 parent
24768a99
Exists in
master
and in
1 other branch
staged.
Showing
2 changed files
with
14 additions
and
5 deletions
Show diff stats
mdata/ILSVRC_S.py
| @@ -41,7 +41,7 @@ class DataILSVRC_S(DataDumperBase): | @@ -41,7 +41,7 @@ class DataILSVRC_S(DataDumperBase): | ||
| 41 | Each step includes reading from & writing to Hbase (though PC). | 41 | Each step includes reading from & writing to Hbase (though PC). |
| 42 | And each step must have a 'spark' mode option, which means that the operation is performed by spark with reading & wrting through RDDs. | 42 | And each step must have a 'spark' mode option, which means that the operation is performed by spark with reading & wrting through RDDs. |
| 43 | 43 | ||
| 44 | - chunkplus@gmail.com | 44 | + copyright(c) 2015 chunkplus@gmail.com |
| 45 | """ | 45 | """ |
| 46 | 46 | ||
| 47 | def __init__(self, base_dir='/media/chunk/Elements/D/data/ImageNet/img/ILSVRC2013_DET_val', category='Train'): | 47 | def __init__(self, base_dir='/media/chunk/Elements/D/data/ImageNet/img/ILSVRC2013_DET_val', category='Train'): |
| @@ -191,7 +191,8 @@ class DataILSVRC_S(DataDumperBase): | @@ -191,7 +191,8 @@ class DataILSVRC_S(DataDumperBase): | ||
| 191 | tmpf_dst = tempfile.NamedTemporaryFile(suffix='.jpg', mode='w+b') | 191 | tmpf_dst = tempfile.NamedTemporaryFile(suffix='.jpg', mode='w+b') |
| 192 | 192 | ||
| 193 | if rate == None: | 193 | if rate == None: |
| 194 | - embed_rate = f5.embed_raw_data(tmpf_src.name, os.path.join(package_dir, '../res/toembed'), tmpf_dst.name) | 194 | + embed_rate = f5.embed_raw_data(tmpf_src.name, os.path.join(package_dir, '../res/toembed'), |
| 195 | + tmpf_dst.name) | ||
| 195 | else: | 196 | else: |
| 196 | assert (rate >= 0 and rate < 1) | 197 | assert (rate >= 0 and rate < 1) |
| 197 | # print capacity | 198 | # print capacity |
test/test_data.py
| @@ -58,11 +58,19 @@ def test_ILSVRC(): | @@ -58,11 +58,19 @@ def test_ILSVRC(): | ||
| 58 | 58 | ||
| 59 | def test_ILSVRC_S(): | 59 | def test_ILSVRC_S(): |
| 60 | timer = Timer() | 60 | timer = Timer() |
| 61 | + | ||
| 62 | + dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Train_2') | ||
| 63 | + dil.format() | ||
| 64 | + dil.store_img() | ||
| 65 | + | ||
| 66 | + | ||
| 61 | dils = ILSVRC_S.DataILSVRC_S(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Train_2') | 67 | dils = ILSVRC_S.DataILSVRC_S(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Train_2') |
| 62 | 68 | ||
| 63 | - dils._extract_data(mode='hbase', writeback=True) | ||
| 64 | - dils._embed_data(mode='hbase', rate=0.1, readforward=True, writeback=True) | ||
| 65 | - dils._extract_feat( mode='hbase', feattype='ibd', readforward=True, writeback=True) | 69 | + # dils._extract_data(mode='hbase', writeback=True) |
| 70 | + # dils._embed_data(mode='hbase', rate=0.1, readforward=True, writeback=True) | ||
| 71 | + # dils._extract_feat( mode='hbase', feattype='ibd', readforward=True, writeback=True) | ||
| 72 | + | ||
| 73 | + dils.format() | ||
| 66 | 74 | ||
| 67 | if __name__ == '__main__': | 75 | if __name__ == '__main__': |
| 68 | # test_MSR() | 76 | # test_MSR() |