0d9a20ea
Chunk
staged.
|
1
2
|
__author__ = 'chunk'
|
2c2d57c7
Chunk
ILSVRC datapath h...
|
3
|
from ..common import *
|
be12257b
Chunk
data-feat-model f...
|
4
|
|
84648488
Chunk
reverted.
|
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
|
from ..mdata import MSR, CV, ILSVRC, ILSVRC_S, crop
def test_MSR():
dmsr = MSR.DataMSR()
# msrd.format()
# msrd.build_list()
# dmsr.store_image()
# dmsr.store_tag()
# dmsr.extract_feat(feattype='ibd')
dmsr.store_feat(feattype='ibd')
def test_CV():
dcv = CV.DataCV()
# dcv.format()
# dcv.build_list()
# dcv.get_feat()
# dcv.extract_feat()
print dcv.get_feat("/home/hadoop/data/HeadShoulder/dst/Train/Img/132/7c5fe33bd194fc1ae7b0023956ebd.jpg", 'ibd')
X, Y = dcv.load_data()
print len(X), len(Y)
|
be12257b
Chunk
data-feat-model f...
|
31
|
def test_ILSVRC():
|
24b3b616
Chunk
staged.
|
32
|
timer = Timer()
|
1821e0e3
Chunk
benchmarking...
|
33
|
timer.mark()
|
554a7b9a
Chunk
staged.
|
34
|
# dil = ILSVRC.DataILSVRC(base='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Train')
|
1821e0e3
Chunk
benchmarking...
|
35
|
dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Test_1')
|
1dc7c44b
Chunk
crawler-hbase-spa...
|
36
|
# dil = ILSVRC.DataILSVRC(base='/media/chunk/Elements/D/data/ImageNet/img/ILSVRC2013_DET_val', category='Train_1')
|
84648488
Chunk
reverted.
|
37
|
timer.report()
|
1dc7c44b
Chunk
crawler-hbase-spa...
|
38
|
|
1821e0e3
Chunk
benchmarking...
|
39
|
timer.mark()
|
554a7b9a
Chunk
staged.
|
40
|
dil.format()
|
02528074
Chunk
staged.
|
41
|
timer.report()
|
1821e0e3
Chunk
benchmarking...
|
42
|
|
ad70caf6
Chunk
staged.
|
43
|
timer.mark()
|
02528074
Chunk
staged.
|
44
45
46
|
dil.embed(rate=0.2)
timer.report()
|
1821e0e3
Chunk
benchmarking...
|
47
|
timer.mark()
|
02528074
Chunk
staged.
|
48
49
|
dil.extract_feat(feattype='ibd')
timer.report()
|
2c2d57c7
Chunk
ILSVRC datapath h...
|
50
|
|
554a7b9a
Chunk
staged.
|
51
|
# dil.extract_feat(feattype='hog')
|
1821e0e3
Chunk
benchmarking...
|
52
|
|
02528074
Chunk
staged.
|
53
|
# timer.mark()
|
554a7b9a
Chunk
staged.
|
54
|
# dil.store_img()
|
489c5608
Chunk
debugging...
|
55
|
# timer.report()
|
02528074
Chunk
staged.
|
56
57
|
# timer.mark()
|
4f36b116
Chunk
staged.
|
58
59
60
|
# dil.store_tag()
# timer.report()
#
|
ad70caf6
Chunk
staged.
|
61
|
# timer.mark()
|
4f36b116
Chunk
staged.
|
62
63
64
65
66
67
68
69
70
71
72
|
# dil.store_info()
# timer.report()
#
# timer.mark()
# dil.store_feat()
# timer.report()
def test_ILSVRC_S_LOCAL():
timer = Timer()
|
554a7b9a
Chunk
staged.
|
73
|
timer.mark()
|
cb798a7f
Chunk
libs & scripts in...
|
74
|
dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Train_2')
|
1dc7c44b
Chunk
crawler-hbase-spa...
|
75
|
dil.delete_table()
|
24768a99
Chunk
mode 'hbase' fini...
|
76
|
dil.format()
|
35cf2e3a
Chunk
staged.
|
77
|
dil.store_img()
|
1dc7c44b
Chunk
crawler-hbase-spa...
|
78
|
timer.report()
|
84648488
Chunk
reverted.
|
79
|
|
d47ae6ce
Chunk
staged.
|
80
81
82
|
dils = ILSVRC_S.DataILSVRC_S(base='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Train_2')
# dils._extract_data(mode='hbase', writeback=True)
|
1dc7c44b
Chunk
crawler-hbase-spa...
|
83
|
# dils._embed_data(mode='hbase', rate=0.1, readforward=True, writeback=True)
|
35cf2e3a
Chunk
staged.
|
84
|
# dils._extract_feat( mode='hbase', feattype='ibd', readforward=True, writeback=True)
|
84648488
Chunk
reverted.
|
85
|
|
35cf2e3a
Chunk
staged.
|
86
|
dils._extract_data(mode='hbase', writeback=False)
|
35cf2e3a
Chunk
staged.
|
87
|
dils._embed_data(mode='hbase', rate=0.1, readforward=False, writeback=False)
|
1dc7c44b
Chunk
crawler-hbase-spa...
|
88
89
90
91
92
93
94
95
96
97
|
dils._extract_feat(mode='hbase', feattype='ibd', readforward=False, writeback=True)
timer.report()
def test_ILSVRC_S_SPARK():
timer = Timer()
timer.mark()
dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Test_1')
|
26616791
Chunk
RDD-hbase bug fix...
|
98
|
dil.delete_table()
|
1dc7c44b
Chunk
crawler-hbase-spa...
|
99
100
|
dil.format()
dil.store_img()
|
54e2adda
Chunk
staged.
|
101
|
timer.report()
|
84648488
Chunk
reverted.
|
102
|
return
|
54e2adda
Chunk
staged.
|
103
|
|
f4fb4381
Chunk
staged.
|
104
|
dils = ILSVRC_S.DataILSVRC_S(base='ILSVRC2013_DET_val', category='Test_1')
|
54e2adda
Chunk
staged.
|
105
106
|
timer.mark()
|
26616791
Chunk
RDD-hbase bug fix...
|
107
|
dils._extract_data(mode='spark', writeback=False)
|
489c5608
Chunk
debugging...
|
108
|
timer.report()
|
1821e0e3
Chunk
benchmarking...
|
109
|
|
489c5608
Chunk
debugging...
|
110
|
timer.mark()
|
1dc7c44b
Chunk
crawler-hbase-spa...
|
111
|
dils._embed_data(mode='spark', rate=0.2, readforward=False, writeback=False)
|
489c5608
Chunk
debugging...
|
112
|
timer.report()
|
489c5608
Chunk
debugging...
|
113
|
|
84648488
Chunk
reverted.
|
114
|
timer.mark()
|
1dc7c44b
Chunk
crawler-hbase-spa...
|
115
116
|
dils._extract_feat(mode='spark', feattype='ibd', readforward=False, writeback=True, withdata=True)
timer.report()
|
b9990e77
Chunk
staged.
|
117
|
|
4f36b116
Chunk
staged.
|
118
|
def test_ILSVRC_S():
|
e3e7e73a
Chunk
spider standalone...
|
119
120
|
# test_ILSVRC_S_LOCAL()
test_ILSVRC_S_SPARK()
|
e3ec1f74
Chunk
staged.
|
121
122
123
|
def test_pipeline():
|
f4fb4381
Chunk
staged.
|
124
|
timer = Timer()
|
e3ec1f74
Chunk
staged.
|
125
126
127
|
timer.mark()
|
e3e7e73a
Chunk
spider standalone...
|
128
|
dils = ILSVRC_S.DataILSVRC_S(base='MSPIDER', category=None)
|
4f36b116
Chunk
staged.
|
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
|
dils._extract_data(mode='spark', writeback=False)
dils._embed_data(mode='spark', rate=0.1, readforward=False, writeback=False)
dils._extract_feat(mode='spark', feattype='ibd', readforward=False, writeback=True, withdata=True)
timer.report()
def test_crop():
crop.crop_Test()
if __name__ == '__main__':
# test_MSR()
# test_CV()
# test_ILSVRC()
# test_ILSVRC_S()
test_pipeline()
print 'helllo'
|
e3e7e73a
Chunk
spider standalone...
|
|
|
e3e7e73a
Chunk
spider standalone...
|
|
|
f4fb4381
Chunk
staged.
|
|
|
e3e7e73a
Chunk
spider standalone...
|
|
|
1dc7c44b
Chunk
crawler-hbase-spa...
|
|
|
4f36b116
Chunk
staged.
|
|
|
1dc7c44b
Chunk
crawler-hbase-spa...
|
|
|
84648488
Chunk
reverted.
|
|
|
1dc7c44b
Chunk
crawler-hbase-spa...
|
|
|
3b4e250d
Chunk
staged.
|
|
|
2c2d57c7
Chunk
ILSVRC datapath h...
|
|
|
84648488
Chunk
reverted.
|
|
|
51708346
Chunk
final experiments...
|
|
|
84648488
Chunk
reverted.
|
|
|
4f36b116
Chunk
staged.
|
|
|
84648488
Chunk
reverted.
|
|
|
24b3b616
Chunk
staged.
|
|
|
be12257b
Chunk
data-feat-model f...
|
|
|
2c2d57c7
Chunk
ILSVRC datapath h...
|
|
|
0fbc087e
Chunk
staged.
|
|
|
1dc7c44b
Chunk
crawler-hbase-spa...
|
|
|
0fbc087e
Chunk
staged.
|
|
|
1dc7c44b
Chunk
crawler-hbase-spa...
|
|
|
be12257b
Chunk
data-feat-model f...
|
|
|
be12257b
Chunk
data-feat-model f...
|
|
|