Blame view

mmodel/svm/SVM.py 7.79 KB
c7fa1d60   Chunk   refractoration st...
1
2
3
4
5
6
7
'''
SVM Model.

@author: chunk
chunkplus@gmail.com
2014 Dec
'''
e3e7e73a   Chunk   spider standalone...
8
import os, sys
2bf33465   Chunk   staged.
9
10
11
# from ...mfeat import *
from ...mmodel import *
# from ...mmodel.svm.svmutil import *
e3e7e73a   Chunk   spider standalone...
12
from ...mspark import SC
2bf33465   Chunk   staged.
13
from ...common import *
c7fa1d60   Chunk   refractoration st...
14

c7fa1d60   Chunk   refractoration st...
15
import numpy as np
84648488   Chunk   reverted.
16
17
import csv
import json
c7fa1d60   Chunk   refractoration st...
18
import pickle
84648488   Chunk   reverted.
19
# import cv2
c7fa1d60   Chunk   refractoration st...
20
21
from sklearn import svm

e3e7e73a   Chunk   spider standalone...
22
package_dir = os.path.dirname(os.path.abspath(__file__))
84648488   Chunk   reverted.
23

c7fa1d60   Chunk   refractoration st...
24
25
26
27
28
29
30
dict_Train = {}
dict_databuf = {}
dict_tagbuf = {}
dict_featbuf = {}


class ModelSVM(ModelBase):
02528074   Chunk   staged.
31
    def __init__(self, toolset='sklearn', sc=None):
c7fa1d60   Chunk   refractoration st...
32
        ModelBase.__init__(self)
f20e20ce   Chunk   staged.
33
        self.toolset = toolset
02528074   Chunk   staged.
34
        self.sparker = sc
c7fa1d60   Chunk   refractoration st...
35

84648488   Chunk   reverted.
36

10b4f63f   Chunk   staged. Before Pa...
37
    def _train_sklearn(self, X, Y):
5ec38adb   Chunk   spark-local of da...
38
        clf = svm.SVC(C=4, kernel='linear', shrinking=False, verbose=True)
d0be60e7   Chunk   jpeg update.
39
        clf.fit(X, Y)
a9c10957   Chunk   hbase-svm & spark...
40
        with open(os.path.join(package_dir, '../..', 'res/svm_sklearn.model'), 'wb') as modelfile:
10b4f63f   Chunk   staged. Before Pa...
41
42
            model = pickle.dump(clf, modelfile)

a9c10957   Chunk   hbase-svm & spark...
43
        self.model = clf
e3e7e73a   Chunk   spider standalone...
44

a9c10957   Chunk   hbase-svm & spark...
45
46
47
48
49
        return clf


    def _predict_sklearn(self, feat, model=None):
        """N.B. sklearn.svm.base.predict :
f20e20ce   Chunk   staged.
50
            Perform classification on samples in X.
a9c10957   Chunk   hbase-svm & spark...
51
52
53
54
                Parameters
                ----------
                X : {array-like, sparse matrix}, shape = [n_samples, n_features]

1c2a3fa0   Chunk   staged.
55
                Returns
e3e7e73a   Chunk   spider standalone...
56
                -------
a9c10957   Chunk   hbase-svm & spark...
57
                y_pred : array, shape = [n_samples]
5ec38adb   Chunk   spark-local of da...
58
59
60
                    Class labels for samples in X.
        """
        if model is None:
a9c10957   Chunk   hbase-svm & spark...
61
62
            if self.model != None:
                model = self.model
84648488   Chunk   reverted.
63
            else:
f20e20ce   Chunk   staged.
64
                print 'loading model ...'
a9c10957   Chunk   hbase-svm & spark...
65
66
67
68
                with open(os.path.join(package_dir, '../..', 'res/svm_sklearn.model'), 'rb') as modelfile:
                    model = pickle.load(modelfile)

        return model.predict(feat)
1c2a3fa0   Chunk   staged.
69

e3e7e73a   Chunk   spider standalone...
70
    def __test_sklearn(self, X, Y, model=None):
a9c10957   Chunk   hbase-svm & spark...
71
        if model is None:
5ec38adb   Chunk   spark-local of da...
72
            if self.model != None:
a9c10957   Chunk   hbase-svm & spark...
73
74
75
76
                model = self.model
            else:
                print 'loading model ...'
                with open(os.path.join(package_dir, '../..', 'res/svm_sklearn.model'), 'rb') as modelfile:
c7fa1d60   Chunk   refractoration st...
77
                    model = pickle.load(modelfile)
1c2a3fa0   Chunk   staged.
78
79

        result_Y = np.array(self._predict_sklearn(X, model))
9371f8fa   Chunk   SVM param engenee...
80

e3e7e73a   Chunk   spider standalone...
81
        fp = 0
9371f8fa   Chunk   SVM param engenee...
82
        tp = 0
c7fa1d60   Chunk   refractoration st...
83
        sum = np.sum(np.array(Y) == 1)
9371f8fa   Chunk   SVM param engenee...
84
        positive, negative = np.sum(np.array(Y) == 1), np.sum(np.array(Y) == 0)
c7fa1d60   Chunk   refractoration st...
85
        print positive, negative
9371f8fa   Chunk   SVM param engenee...
86
        for i in range(len(Y)):
c7fa1d60   Chunk   refractoration st...
87
            if Y[i] == 0 and result_Y[i] == 1:
84648488   Chunk   reverted.
88
                fp += 1
f20e20ce   Chunk   staged.
89
            elif Y[i] == 1 and result_Y[i] == 1:
be12257b   Chunk   data-feat-model f...
90
91
92
93
94
                tp += 1
        return float(fp) / negative, float(tp) / positive, np.mean(Y == result_Y)

    def _test_sklearn(self, X, Y, model=None):
        if model is None:
c7fa1d60   Chunk   refractoration st...
95
            if self.model != None:
be12257b   Chunk   data-feat-model f...
96
97
98
99
100
101
102
103
104
                model = self.model
            else:
                print 'loading model ...'
                with open(os.path.join(package_dir, '../..', 'res/svm_sklearn.model'), 'rb') as modelfile:
                    model = pickle.load(modelfile)

        return model.score(X, Y)

    # def _train_libsvm(self, X, Y):
1c2a3fa0   Chunk   staged.
105
    #     X, Y = list(X), list(Y)
e3e7e73a   Chunk   spider standalone...
106
    #     # X, Y = [float(i) for i in X], [float(i) for i in Y]
be12257b   Chunk   data-feat-model f...
107
    #     prob = svm_problem(Y, X)
c7fa1d60   Chunk   refractoration st...
108
    #     param = svm_parameter('-t 0 -c 4 -b 1 -h 0')
be12257b   Chunk   data-feat-model f...
109
    #     # param = svm_parameter(kernel_type=LINEAR, C=10)
c7fa1d60   Chunk   refractoration st...
110
    #     m = svm_train(prob, param)
9371f8fa   Chunk   SVM param engenee...
111
    #     svm_save_model(os.path.join(package_dir, '../..', 'res/svm_libsvm.model'), m)
be12257b   Chunk   data-feat-model f...
112
113
114
115
    #
    #     self.model = m
    #
    #     return m
1c2a3fa0   Chunk   staged.
116
    #
e3e7e73a   Chunk   spider standalone...
117
    # def _predict_libsvm(self, feat, model=None):
be12257b   Chunk   data-feat-model f...
118
    #     if model is None:
c7fa1d60   Chunk   refractoration st...
119
    #         if self.model != None:
f20e20ce   Chunk   staged.
120
    #             model = self.model
c7fa1d60   Chunk   refractoration st...
121
    #         else:
13a594f1   Chunk   before 中期答辩~~~
122
123
124
125
126
127
128
129
130
131
132
    #             print 'loading model ...'
    #             model = svm_load_model(os.path.join(package_dir, '../..', 'res/svm_libsvm.model'))
    #
    #     feat = [list(feat)]
    #     # print len(feat),[0] * len(feat)
    #     label, _, _ = svm_predict([0] * len(feat), feat, model)
    #     return label
    #
    #
    # def _test_libsvm(self, X, Y, model=None):
    #     if model is None:
c7fa1d60   Chunk   refractoration st...
133
    #         if self.model != None:
9371f8fa   Chunk   SVM param engenee...
134
    #             model = self.model
1c2a3fa0   Chunk   staged.
135
    #         else:
9371f8fa   Chunk   SVM param engenee...
136
137
138
139
    #             print 'loading model ...'
    #             model = svm_load_model(os.path.join(package_dir, '../..', 'res/svm_libsvm.model'))
    #
    #     X, Y = list(X), list(Y)
e3e7e73a   Chunk   spider standalone...
140
    #     p_labs, p_acc, p_vals = svm_predict(Y, X, model)
9371f8fa   Chunk   SVM param engenee...
141
    #     # ACC, MSE, SCC = evaluations(Y, p_labs)
1c2a3fa0   Chunk   staged.
142
143
144
    #
    #     return p_acc

84648488   Chunk   reverted.
145
146
147
148
    # def _train_opencv(self, X, Y):
    # svm_params = dict(kernel_type=cv2.SVM_LINEAR,
    #                       svm_type=cv2.SVM_C_SVC,
    #                       C=4)
f4fb4381   Chunk   staged.
149
    #
84648488   Chunk   reverted.
150
151
152
153
154
    #     X, Y = np.array(X, dtype=np.float32), np.array(Y, dtype=np.float32)
    #
    #     svm = cv2.SVM()
    #     svm.train(X, Y, params=svm_params)
    #     svm.save(os.path.join(package_dir, '../..', 'res/svm_opencv.model'))
e3e7e73a   Chunk   spider standalone...
155
    #
84648488   Chunk   reverted.
156
157
158
159
160
161
162
163
164
165
166
167
    #     self.model = svm
    #
    #     return svm
    #
    #
    # def _predict_opencv(self, feat, model=None):
    #     if model is None:
    #         if self.model != None:
    #             model = self.model
    #         else:
    #             print 'loading model ...'
    #             with open(os.path.join(package_dir, '../..', 'res/svm_opencv.model'), 'rb') as modelfile:
e3e7e73a   Chunk   spider standalone...
168
    #                 model = pickle.load(modelfile)
84648488   Chunk   reverted.
169
170
171
172
173
174
175
176
177
178
179
180
    #     feat = np.array(feat, dtype=np.float32)
    #
    #     return model.predict(feat)
    #
    #
    # def _test_opencv(self, X, Y, model=None):
    #     if model is None:
    #         if self.model != None:
    #             model = self.model
    #         else:
    #             print 'loading model ...'
    #             with open(os.path.join(package_dir, '../..', 'res/svm_opencv.model'), 'rb') as modelfile:
e3e7e73a   Chunk   spider standalone...
181
    #                 model = pickle.load(modelfile)
84648488   Chunk   reverted.
182
183
184
185
    #
    #     X, Y = np.array(X, dtype=np.float32), np.array(Y, dtype=np.float32)
    #
    #     # result_Y = np.array([self._predict_opencv(x, model) for x in X])
f4fb4381   Chunk   staged.
186
    #     result_Y = np.array(model.predict_all(X)).ravel()
84648488   Chunk   reverted.
187
    #     # print X[0]
f4fb4381   Chunk   staged.
188
189
    #     # print result_Y,Y
    #     return np.mean(Y == result_Y)
84648488   Chunk   reverted.
190
191
192


    def _train_spark(self, X, Y=None):
02528074   Chunk   staged.
193
        if self.sparker == None:
f20e20ce   Chunk   staged.
194
            self.sparker = SC.Sparker(host='HPC-server', appname='ImageCV', master='spark://HPC-server:7077')
e3e7e73a   Chunk   spider standalone...
195

f20e20ce   Chunk   staged.
196
197
        self.model = self.sparker.train_svm(X, Y)

02528074   Chunk   staged.
198
        return svm
f20e20ce   Chunk   staged.
199
200
201
202
203
204
205
206

    def _predict_spark(self, feat, model=None):
        return self.sparker.predict_svm(feat, model)

    def _test_spark(self, X, Y, model=None):
        return self.sparker.test_svm(X, Y, model)


84648488   Chunk   reverted.
207
    def train(self, X, Y=None):
02528074   Chunk   staged.
208

10b4f63f   Chunk   staged. Before Pa...
209
210
211
        if self.toolset == 'sklearn':
            return self._train_sklearn(X, Y)
        # elif self.toolset == 'opencv':
84648488   Chunk   reverted.
212
213
        #     return self._train_opencv(X, Y)
        # elif self.toolset == 'libsvm':
10b4f63f   Chunk   staged. Before Pa...
214
215
216
217
218
219
        #     return self._train_libsvm(X, Y)
        elif self.toolset == 'spark':
            return self._train_spark(X, Y)
        else:
            raise Exception("Unknown toolset!")

c7fa1d60   Chunk   refractoration st...
220
    def predict(self, feat, model=None):
be12257b   Chunk   data-feat-model f...
221

f20e20ce   Chunk   staged.
222
        if self.toolset == 'sklearn':
10b4f63f   Chunk   staged. Before Pa...
223
224
            return self._predict_sklearn(feat, model)
        # elif self.toolset == 'opencv':
84648488   Chunk   reverted.
225
226
        #     return self._predict_opencv(feat, model)
        # elif self.toolset == 'libsvm':
10b4f63f   Chunk   staged. Before Pa...
227
228
        #     return self._predict_libsvm(feat, model)
        elif self.toolset == 'spark':
02528074   Chunk   staged.
229
            return self._predict_spark(feat, model)
10b4f63f   Chunk   staged. Before Pa...
230
231
232
        else:
            raise Exception("Unknown toolset!")

f20e20ce   Chunk   staged.
233

84648488   Chunk   reverted.
234
    def test(self, X, Y=None, model=None):
02528074   Chunk   staged.
235

f20e20ce   Chunk   staged.
236
        if self.toolset == 'sklearn':
10b4f63f   Chunk   staged. Before Pa...
237
            return self.__test_sklearn(X, Y, model)
13a594f1   Chunk   before 中期答辩~~~
238
        # elif self.toolset == 'opencv':
84648488   Chunk   reverted.
239
240
        #     return self._test_opencv(X, Y, model)
        # elif self.toolset == 'libsvm':
10b4f63f   Chunk   staged. Before Pa...
241
242
        #     return self._test_libsvm(X, Y, model)
        elif self.toolset == 'spark':
02528074   Chunk   staged.
243
            return self._test_spark(X, Y, model)
10b4f63f   Chunk   staged. Before Pa...
244
245
        else:
            raise Exception("Unknown toolset!")
84648488   Chunk   reverted.