Chunk / ImageR

Blame view

test/test_model.py 3.83 KB
Edit Raw Normal View History
be12257b   Chunk   data-feat-model f... 
1
2
 
__author__ = 'chunk'
9371f8fa   Chunk   SVM param engenee... 
3
 
from sklearn import cross_validation
f4fb4381   Chunk   staged. 
4
2bf33465   Chunk   staged. 
5
 
from ..common import *
84648488   Chunk   reverted. 
6
 
from ..mdata import ILSVRC, ILSVRC_S
61e78eb3   Chunk   staged. 
7
 
from ..mmodel.svm import SVM
84648488   Chunk   reverted. 
8
2bf33465   Chunk   staged. 
9
 
import gzip
5a469df5   Chunk   staged. 
10
11
12
 
import cPickle

timer = Timer()
84648488   Chunk   reverted. 
13
 
package_dir = os.path.dirname(os.path.abspath(__file__))
5a469df5   Chunk   staged. 
14
15
be12257b   Chunk   data-feat-model f... 
16
 
def test_SVM_ILSVRC():
d2603183   Chunk   staged. 
17
 
    timer.mark()
84648488   Chunk   reverted. 
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
 
    dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val',
                            category='Train_5000_0.05_orig')
    X, Y = dil.load_data(mode='local')  #
    # X, Y = dil.load_data(mode='hbase') #
    # X, Y = dil.load_data(mode='spark') #
    X_train, X_test, Y_train, Y_test = cross_validation.train_test_split(X, Y, test_size=0.4,
                                                                         random_state=0)
    print np.array(Y).shape, np.array(X).shape
    print np.array(X_train).shape, np.array(Y_train).shape
    print np.array(X_test).shape, np.array(Y_test).shape

    timer.report()

    timer.mark()
    msvm = SVM.ModelSVM(toolset='sklearn')  # 4.884247s 0.777853030816
    # msvm = SVM.ModelSVM(toolset='opencv') #
    # msvm = SVM.ModelSVM(toolset='libsvm') #
    # msvm = SVM.ModelSVM(toolset='spark')
    msvm.train(X_train, Y_train)
    timer.report()

    timer.mark()
    print msvm.test(X_test, Y_test)  #
    timer.report()  #

    # timer.mark()
    # print 'or like this:'
    # scores = cross_validation.cross_val_score(msvm.model, X, Y)
    # print scores
    # timer.report()
d0be60e7   Chunk   jpeg update. 
48
49
84648488   Chunk   reverted. 
50
 
def test_SVM_ILSVRC_HBASE():
d0be60e7   Chunk   jpeg update. 
51
 
    timer.mark()
02528074   Chunk   staged. 
52
53
 

    # dil = ILSVRC.DataILSVRC(base_dir='ILSVRC2013_DET_val', category='Train_3')
84648488   Chunk   reverted. 
54
 
    # X, Y = dil.load_data(mode='hbase') # pass
6d6d75b8   Chunk   spider LOG system. 
55
02528074   Chunk   staged. 
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
 
    dils = ILSVRC_S.DataILSVRC_S(base='ILSVRC2013_DET_val', category='Test_1')
    X, Y = dils.load_data(mode='hbase')  # pass

    dil = ILSVRC.DataILSVRC(base_dir='/data/hadoop/ImageNet/ILSVRC/ILSVRC2013_DET_val',
                            category='Test_1')
    X1, Y1 = dil.load_data(mode='local')

    X_train, X_test, Y_train, Y_test = cross_validation.train_test_split(X, Y, test_size=0.4,
                                                                         random_state=0)
    print Y, np.sum(np.array(Y) == 0), np.sum(np.array(Y) == 1)
    print np.array(Y).shape, np.array(X).shape
    print np.array(X_train).shape, np.array(Y_train).shape
    print np.array(X_test).shape, np.array(Y_test).shape

    timer.report()

    timer.mark()
    msvm = SVM.ModelSVM(toolset='sklearn')  # 4.884247s 0.777853030816
    # msvm = SVM.ModelSVM(toolset='opencv') #
    # msvm = SVM.ModelSVM(toolset='libsvm') #
    # msvm = SVM.ModelSVM(toolset='spark')
    msvm.train(X_train, Y_train)
    timer.report()

    timer.mark()
    print msvm.test(X_test, Y_test)  #
    timer.report()  #

    timer.mark()
    print msvm.test(X1, Y1)  #
4f36b116   Chunk   staged. 
86
 
    timer.report()  #
02528074   Chunk   staged. 
87
 
    # timer.mark()
9371f8fa   Chunk   SVM param engenee... 
88
 
    # print 'or like this:'
51708346   Chunk   final experiments... 
89
90
 
    # scores = cross_validation.cross_val_score(msvm.model, X, Y)
    # print scores
2bd3da3e   Chunk   staged. 
91
 
    # timer.report()
84648488   Chunk   reverted. 
92
d2603183   Chunk   staged. 
93
02528074   Chunk   staged. 
94
 
def test_SVM_ILSVRC_SPARK():
9371f8fa   Chunk   SVM param engenee... 
95
96
97
 
    timer.mark()
    dils = ILSVRC_S.DataILSVRC_S(base='ILSVRC2013_DET_val', category='Test_1')
    rdd_dataset = dils.load_data(mode='spark')  # pass
d0be60e7   Chunk   jpeg update. 
98
99
100
 

    timer.report()
4f36b116   Chunk   staged. 
101
 
    timer.mark()
51708346   Chunk   final experiments... 
102
 
    # msvm = SVM.ModelSVM(toolset='sklearn')  #
9371f8fa   Chunk   SVM param engenee... 
103
 
    # msvm = SVM.ModelSVM(toolset='opencv') #
4f36b116   Chunk   staged. 
104
 
    # msvm = SVM.ModelSVM(toolset='libsvm') #
9371f8fa   Chunk   SVM param engenee... 
105
 
    msvm = SVM.ModelSVM(toolset='spark', sc=dils.sparker)
d0be60e7   Chunk   jpeg update. 
106
107
108
 
    msvm.train(rdd_dataset)
    timer.report()
9371f8fa   Chunk   SVM param engenee... 
109
 
    dataset = rdd_dataset.collect()
d0be60e7   Chunk   jpeg update. 
110
 
    length = len(dataset)
6d6d75b8   Chunk   spider LOG system. 
111
51708346   Chunk   final experiments... 
112
113
114
115
 
    X_test, Y_test = [dataset[i].features for i in range(length)], [dataset[i].label for i in
                                                                    range(length)]

    timer.mark()
e3e7e73a   Chunk   spider standalone... 
116
117
118
119
120
121
122
123
124
125
126
127
128
 
    print msvm.test(dils.sparker.sc.parallelize(X_test), Y_test)  #
    timer.report()  #


def test_SVM_ILSVRC_S():
    test_SVM_ILSVRC_HBASE()
    # test_SVM_ILSVRC_SPARK()


if __name__ == '__main__':
    # test_SVM_CV()
    test_SVM_ILSVRC()
    print 'helllo'
2bd3da3e   Chunk   staged. 












02528074
 
 Chunk
 
staged.
















f4fb4381
 
 Chunk
 
staged.
















02528074
 
 Chunk
 
staged.
















9371f8fa
 
 Chunk
 
SVM param engenee...
















02528074
 
 Chunk
 
staged.
















9371f8fa
 
 Chunk
 
SVM param engenee...
















02528074
 
 Chunk
 
staged.
















84648488
 
 Chunk
 
reverted.
















02528074
 
 Chunk
 
staged.
















51708346
 
 Chunk
 
final experiments...
















02528074
 
 Chunk
 
staged.
















9371f8fa
 
 Chunk
 
SVM param engenee...
















84648488
 
 Chunk
 
reverted.
















be12257b
 
 Chunk
 
data-feat-model f...
















d0be60e7
 
 Chunk
 
jpeg update.
















be12257b
 
 Chunk
 
data-feat-model f...
















84648488
 
 Chunk
 
reverted.