Chunk / ImageR

Blame view

test/test_data.py 4.62 KB
Edit Raw Normal View History
0d9a20ea   Chunk   staged. 
1
2
 
__author__ = 'chunk'
2c2d57c7   Chunk   ILSVRC datapath h... 
3
 
from ..common import *
be12257b   Chunk   data-feat-model f... 
4
84648488   Chunk   reverted. 
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
 
from ..mdata import ILSVRC, ILSVRC_S


def test_ILSVRC(category='Train_100'):
    timer = Timer()

    # dil = ILSVRC.DataILSVRC(base='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Train')
    dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category=category)
    # dil = ILSVRC.DataILSVRC(base='/media/chunk/Elements/D/data/ImageNet/img/ILSVRC2013_DET_val', category='Train_1')
    print '[time]category:', category

    timer.mark()
    print '[time]formatting...'
    dil.format()
    timer.report()

    timer.mark()
    print '[time]embedding...'
    dil.embed(rate=0.2)
    timer.report()

    timer.mark()
    print '[time]extracting...'
    dil.extract_feat(feattype='ibd')
    timer.report()
be12257b   Chunk   data-feat-model f... 
31
 
    # dil.extract_feat(feattype='hog')
24b3b616   Chunk   staged. 
32
1821e0e3   Chunk   benchmarking... 
33
 
    timer.mark()
554a7b9a   Chunk   staged. 
34
 
    dil.store_img()
1821e0e3   Chunk   benchmarking... 
35
 
    timer.report()
1dc7c44b   Chunk   crawler-hbase-spa... 
36
84648488   Chunk   reverted. 
37
 
    timer.mark()
1dc7c44b   Chunk   crawler-hbase-spa... 
38
 
    dil.store_tag()
1821e0e3   Chunk   benchmarking... 
39
 
    timer.report()
554a7b9a   Chunk   staged. 
40
02528074   Chunk   staged. 
41
 
    timer.mark()
1821e0e3   Chunk   benchmarking... 
42
 
    dil.store_info()
ad70caf6   Chunk   staged. 
43
 
    timer.report()
02528074   Chunk   staged. 
44
45
46
 

    timer.mark()
    dil.store_feat()
1821e0e3   Chunk   benchmarking... 
47
 
    timer.report()
02528074   Chunk   staged. 
48
49
2c2d57c7   Chunk   ILSVRC datapath h... 
50
 
def test_ILSVRC_S_LOCAL():
554a7b9a   Chunk   staged. 
51
 
    timer = Timer()
1821e0e3   Chunk   benchmarking... 
52
02528074   Chunk   staged. 
53
 
    timer.mark()
554a7b9a   Chunk   staged. 
54
 
    dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Train_2')
489c5608   Chunk   debugging... 
55
 
    dil.delete_table()
02528074   Chunk   staged. 
56
57
 
    dil.format()
    dil.store_img()
4f36b116   Chunk   staged. 
58
59
60
 
    timer.report()

    dils = ILSVRC_S.DataILSVRC_S(base='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category='Train_2')
ad70caf6   Chunk   staged. 
61
4f36b116   Chunk   staged. 
62
63
64
65
66
67
68
69
70
71
72
 
    # dils._extract_data(mode='hbase', writeback=True)
    # dils._embed_data(mode='hbase', rate=0.1, readforward=True, writeback=True)
    # dils._extract_feat( mode='hbase', feattype='ibd', readforward=True, writeback=True)

    dils._extract_data(mode='hbase', writeback=False)
    dils._embed_data(mode='hbase', rate=0.1, readforward=False, writeback=False)
    dils._extract_feat(mode='hbase', feattype='ibd', readforward=False, writeback=True)

    timer.report()
554a7b9a   Chunk   staged. 
73
 
def test_ILSVRC_S_SPARK(category='Train_100'):
cb798a7f   Chunk   libs & scripts in... 
74
 
    timer = Timer()
1dc7c44b   Chunk   crawler-hbase-spa... 
75
24768a99   Chunk   mode 'hbase' fini... 
76
 
    timer.mark()
35cf2e3a   Chunk   staged. 
77
 
    dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category=category)
1dc7c44b   Chunk   crawler-hbase-spa... 
78
 
    dil.delete_table()
84648488   Chunk   reverted. 
79
 
    # dil.format()
d47ae6ce   Chunk   staged. 
80
81
82
 
    dil.store_img()
    timer.report()
    # return
1dc7c44b   Chunk   crawler-hbase-spa... 
83
35cf2e3a   Chunk   staged. 
84
 
    dils = ILSVRC_S.DataILSVRC_S(base='ILSVRC2013_DET_val', category=category)
84648488   Chunk   reverted. 
85
35cf2e3a   Chunk   staged. 
86
 
    timer.mark()
35cf2e3a   Chunk   staged. 
87
 
    dils._extract_data(mode='spark', writeback=False)
1dc7c44b   Chunk   crawler-hbase-spa... 
88
89
90
91
92
93
94
95
96
97
 
    dils._embed_data(mode='spark', rate=0.2, readforward=False, writeback=False)
    dils._extract_feat(mode='spark', feattype='ibd', readforward=False, writeback=True, withdata=True)
    timer.report()


def test_ILSVRC_S_ANALYSIS(category='Train_1000', tablename=None):
    timer = Timer()

    timer.mark()
    dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category=category)
26616791   Chunk   RDD-hbase bug fix... 
98
 
    dil.delete_table()
1dc7c44b   Chunk   crawler-hbase-spa... 
99
100
 
    # dil.format()
    dil.store_img()
54e2adda   Chunk   staged. 
101
 
    timer.report()
84648488   Chunk   reverted. 
102
 
    # return
54e2adda   Chunk   staged. 
103
f4fb4381   Chunk   staged. 
104
 
    dils = ILSVRC_S.DataILSVRC_S(base='ILSVRC2013_DET_val', category=category, tablename=tablename)
54e2adda   Chunk   staged. 
105
106
 

    timer.mark()
26616791   Chunk   RDD-hbase bug fix... 
107
 
    dils._extract_data(mode='analysis', writeback=False)
489c5608   Chunk   debugging... 
108
 
    timer.report()
1821e0e3   Chunk   benchmarking... 
109
489c5608   Chunk   debugging... 
110
 
    timer.mark()
1dc7c44b   Chunk   crawler-hbase-spa... 
111
 
    dils._analysis(mode='analysis', feattype='ibd', readforward=False, writeback=True, withdata=False)
489c5608   Chunk   debugging... 
112
 
    timer.report()
489c5608   Chunk   debugging... 
113
84648488   Chunk   reverted. 
114
1dc7c44b   Chunk   crawler-hbase-spa... 
115
116
 
def test_ILSVRC_S_ANALYSIS2(category='Train_1000', tablename='MSPIDER'):
    timer = Timer()
b9990e77   Chunk   staged. 
117
4f36b116   Chunk   staged. 
118
 
    # timer.mark()
e3e7e73a   Chunk   spider standalone... 
119
120
 
    # dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category=category)
    # dil.delete_table()
e3ec1f74   Chunk   staged. 
121
122
123
 
    # # dil.format()
    # dil.store_img()
    # timer.report()
f4fb4381   Chunk   staged. 
124
 
    # # return
e3ec1f74   Chunk   staged. 
125
126
127
 

    dils = ILSVRC_S.DataILSVRC_S(base='ILSVRC2013_DET_val', category=category, tablename=tablename)
e3e7e73a   Chunk   spider standalone... 
128
 
    timer.mark()
4f36b116   Chunk   staged. 
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
 
    dils._extract_data(mode='analysis', writeback=False)
    timer.report()

    timer.mark()
    dils._analysis(mode='analysis', feattype='ibd', readforward=False, writeback=True, withdata=False)
    timer.report()


def test_ILSVRC_S():
    # test_ILSVRC_S_LOCAL()
    test_ILSVRC_S_SPARK()


def test_ILSVRC_IMPORT(category='Train_1000'):
    dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val', category=category)
    dil.delete_table()
    # dil.format()
    dil.store_img()


def test_ILSVRC_EXPORT(category='Train_1000'):
    pass
e3e7e73a   Chunk   spider standalone... 
152
153
154
155
156
 

def test_pipeline():
    timer = Timer()

    timer.mark()
e3e7e73a   Chunk   spider standalone... 
157
f4fb4381   Chunk   staged. 
158
 
    dils = ILSVRC_S.DataILSVRC_S(base='MSPIDER', category=None)
e3e7e73a   Chunk   spider standalone... 
159
160
161
 

    dils._extract_data(mode='spark', writeback=False)
    dils._embed_data(mode='spark', rate=0.1, readforward=False, writeback=False)
1dc7c44b   Chunk   crawler-hbase-spa... 
162
163
164
165
166
 
    dils._extract_feat(mode='spark', feattype='ibd', readforward=False, writeback=True, withdata=True)

    timer.report()
4f36b116   Chunk   staged. 
167
168
169
170
171
172
173
174
175
 
if __name__ == '__main__':
    # test_MSR()
    # test_CV()
    # test_ILSVRC()
    # test_ILSVRC_S()

    test_pipeline()

    print 'helllo'
1dc7c44b   Chunk   crawler-hbase-spa... 












84648488
 
 Chunk
 
reverted.
















1dc7c44b
 
 Chunk
 
crawler-hbase-spa...
















3b4e250d
 
 Chunk
 
staged.
















2c2d57c7
 
 Chunk
 
ILSVRC datapath h...
















84648488
 
 Chunk
 
reverted.
















51708346
 
 Chunk
 
final experiments...
















84648488
 
 Chunk
 
reverted.
















4f36b116
 
 Chunk
 
staged.
















84648488
 
 Chunk
 
reverted.
















24b3b616
 
 Chunk
 
staged.
















be12257b
 
 Chunk
 
data-feat-model f...
















2c2d57c7
 
 Chunk
 
ILSVRC datapath h...
















0fbc087e
 
 Chunk
 
staged.
















1dc7c44b
 
 Chunk
 
crawler-hbase-spa...
















0fbc087e
 
 Chunk
 
staged.
















1dc7c44b
 
 Chunk
 
crawler-hbase-spa...
















be12257b
 
 Chunk
 
data-feat-model f...
















be12257b
 
 Chunk
 
data-feat-model f...