c7fa1d60
Chunk
refractoration st...
|
1
2
3
4
5
6
7
|
'''
SVM Model.
@author: chunk
chunkplus@gmail.com
2014 Dec
'''
|
e3e7e73a
Chunk
spider standalone...
|
8
|
from ...mfeat import *
|
2bf33465
Chunk
staged.
|
9
10
11
|
from ...mmodel import *
from ...mmodel.svm.svmutil import *
from ...mspark import SC
|
e3e7e73a
Chunk
spider standalone...
|
12
|
from ...common import *
|
2bf33465
Chunk
staged.
|
13
|
|
c7fa1d60
Chunk
refractoration st...
|
14
|
import numpy as np
|
c7fa1d60
Chunk
refractoration st...
|
15
|
import csv
|
84648488
Chunk
reverted.
|
16
17
|
import json
import pickle
|
c7fa1d60
Chunk
refractoration st...
|
18
|
import cv2
|
84648488
Chunk
reverted.
|
19
|
from sklearn import svm
|
c7fa1d60
Chunk
refractoration st...
|
20
21
|
|
e3e7e73a
Chunk
spider standalone...
|
22
|
dict_Train = {}
|
84648488
Chunk
reverted.
|
23
|
dict_databuf = {}
|
c7fa1d60
Chunk
refractoration st...
|
24
25
26
27
28
29
30
|
dict_tagbuf = {}
dict_featbuf = {}
class ModelSVM(ModelBase):
def __init__(self, toolset='sklearn', sc=None):
ModelBase.__init__(self)
|
02528074
Chunk
staged.
|
31
|
self.toolset = toolset
|
c7fa1d60
Chunk
refractoration st...
|
32
|
self.sparker = sc
|
f20e20ce
Chunk
staged.
|
33
|
|
02528074
Chunk
staged.
|
34
|
|
c7fa1d60
Chunk
refractoration st...
|
35
|
def _train_libsvm(self, X, Y):
|
84648488
Chunk
reverted.
|
36
|
X, Y = list(X), list(Y)
|
10b4f63f
Chunk
staged. Before Pa...
|
37
|
# X, Y = [float(i) for i in X], [float(i) for i in Y]
|
5ec38adb
Chunk
spark-local of da...
|
38
|
prob = svm_problem(Y, X)
|
d0be60e7
Chunk
jpeg update.
|
39
|
param = svm_parameter('-t 0 -c 4 -b 1 -h 0')
|
a9c10957
Chunk
hbase-svm & spark...
|
40
|
# param = svm_parameter(kernel_type=LINEAR, C=10)
|
10b4f63f
Chunk
staged. Before Pa...
|
41
42
|
m = svm_train(prob, param)
svm_save_model('res/svm_libsvm.model', m)
|
a9c10957
Chunk
hbase-svm & spark...
|
43
|
|
e3e7e73a
Chunk
spider standalone...
|
44
|
self.model = m
|
a9c10957
Chunk
hbase-svm & spark...
|
45
46
47
48
49
|
return m
def _predict_libsvm(self, feat, model=None):
if model is None:
|
f20e20ce
Chunk
staged.
|
50
|
if self.model != None:
|
a9c10957
Chunk
hbase-svm & spark...
|
51
52
53
54
|
model = self.model
else:
print 'loading model ...'
model = svm_load_model('res/svm_libsvm.model')
|
1c2a3fa0
Chunk
staged.
|
55
|
|
e3e7e73a
Chunk
spider standalone...
|
56
|
feat = [list(feat)]
|
a9c10957
Chunk
hbase-svm & spark...
|
57
|
# print len(feat),[0] * len(feat)
|
5ec38adb
Chunk
spark-local of da...
|
58
59
60
|
label, _, _ = svm_predict([0] * len(feat), feat, model)
return label
|
a9c10957
Chunk
hbase-svm & spark...
|
61
62
|
def _test_libsvm(self, X, Y, model=None):
|
84648488
Chunk
reverted.
|
63
|
if model is None:
|
f20e20ce
Chunk
staged.
|
64
|
if self.model != None:
|
a9c10957
Chunk
hbase-svm & spark...
|
65
66
67
68
|
model = self.model
else:
print 'loading model ...'
model = svm_load_model('res/svm_libsvm.model')
|
1c2a3fa0
Chunk
staged.
|
69
|
|
e3e7e73a
Chunk
spider standalone...
|
70
|
X, Y = list(X), list(Y)
|
a9c10957
Chunk
hbase-svm & spark...
|
71
|
p_labs, p_acc, p_vals = svm_predict(Y, X, model)
|
5ec38adb
Chunk
spark-local of da...
|
72
|
# ACC, MSE, SCC = evaluations(Y, p_labs)
|
a9c10957
Chunk
hbase-svm & spark...
|
73
74
75
76
|
return p_acc
def _train_sklearn(self, X, Y):
|
c7fa1d60
Chunk
refractoration st...
|
77
|
clf = svm.SVC(C=4, kernel='linear', shrinking=False, verbose=True)
|
1c2a3fa0
Chunk
staged.
|
78
79
|
clf.fit(X, Y)
with open('res/svm_sk.model', 'wb') as modelfile:
|
9371f8fa
Chunk
SVM param engenee...
|
80
|
model = pickle.dump(clf, modelfile)
|
e3e7e73a
Chunk
spider standalone...
|
81
|
|
9371f8fa
Chunk
SVM param engenee...
|
82
|
self.model = clf
|
c7fa1d60
Chunk
refractoration st...
|
83
|
|
9371f8fa
Chunk
SVM param engenee...
|
84
|
return clf
|
c7fa1d60
Chunk
refractoration st...
|
85
|
|
9371f8fa
Chunk
SVM param engenee...
|
86
|
|
c7fa1d60
Chunk
refractoration st...
|
87
|
def _predict_sklearn(self, feat, model=None):
|
84648488
Chunk
reverted.
|
88
|
"""N.B. sklearn.svm.base.predict :
|
f20e20ce
Chunk
staged.
|
89
|
Perform classification on samples in X.
|
be12257b
Chunk
data-feat-model f...
|
90
91
92
93
94
|
Parameters
----------
X : {array-like, sparse matrix}, shape = [n_samples, n_features]
Returns
|
c7fa1d60
Chunk
refractoration st...
|
95
|
-------
|
be12257b
Chunk
data-feat-model f...
|
96
97
98
99
100
101
102
103
104
|
y_pred : array, shape = [n_samples]
Class labels for samples in X.
"""
if model is None:
if self.model != None:
model = self.model
else:
print 'loading model ...'
with open('res/svm_sklearn.model', 'rb') as modelfile:
|
1c2a3fa0
Chunk
staged.
|
105
|
model = pickle.load(modelfile)
|
e3e7e73a
Chunk
spider standalone...
|
106
|
|
be12257b
Chunk
data-feat-model f...
|
107
|
return model.predict(feat)
|
c7fa1d60
Chunk
refractoration st...
|
108
|
|
be12257b
Chunk
data-feat-model f...
|
109
|
def __test_sklearn(self, X, Y, model=None):
|
c7fa1d60
Chunk
refractoration st...
|
110
|
if model is None:
|
9371f8fa
Chunk
SVM param engenee...
|
111
|
if self.model != None:
|
be12257b
Chunk
data-feat-model f...
|
112
113
114
115
|
model = self.model
else:
print 'loading model ...'
with open('res/svm_sklearn.model', 'rb') as modelfile:
|
1c2a3fa0
Chunk
staged.
|
116
|
model = pickle.load(modelfile)
|
e3e7e73a
Chunk
spider standalone...
|
117
|
|
be12257b
Chunk
data-feat-model f...
|
118
|
result_Y = np.array(self._predict_sklearn(X, model))
|
c7fa1d60
Chunk
refractoration st...
|
119
|
|
f20e20ce
Chunk
staged.
|
120
|
return np.mean(Y == result_Y)
|
c7fa1d60
Chunk
refractoration st...
|
121
|
|
13a594f1
Chunk
before 中期答辩~~~
|
122
123
124
125
126
127
128
129
130
131
132
|
def _test_sklearn(self, X, Y, model=None):
if model is None:
if self.model != None:
model = self.model
else:
print 'loading model ...'
with open('res/svm_sklearn.model', 'rb') as modelfile:
model = pickle.load(modelfile)
return model.score(X, Y)
|
c7fa1d60
Chunk
refractoration st...
|
133
|
|
9371f8fa
Chunk
SVM param engenee...
|
134
|
def _train_opencv(self, X, Y):
|
1c2a3fa0
Chunk
staged.
|
135
|
svm_params = dict(kernel_type=cv2.SVM_LINEAR,
|
9371f8fa
Chunk
SVM param engenee...
|
136
137
138
139
|
svm_type=cv2.SVM_C_SVC,
C=2.67, gamma=5.383)
X, Y = np.array(X, dtype=np.float32), np.array(Y, dtype=np.float32)
|
e3e7e73a
Chunk
spider standalone...
|
140
|
|
9371f8fa
Chunk
SVM param engenee...
|
141
|
svm = cv2.SVM()
|
1c2a3fa0
Chunk
staged.
|
142
143
144
|
svm.train(X, Y, params=svm_params)
svm.save('res/svm_opencv.model')
|
84648488
Chunk
reverted.
|
145
146
147
148
|
self.model = svm
return svm
|
f4fb4381
Chunk
staged.
|
149
|
|
84648488
Chunk
reverted.
|
150
151
152
153
154
|
def _predict_opencv(self, feat, model=None):
if model is None:
if self.model != None:
model = self.model
else:
|
e3e7e73a
Chunk
spider standalone...
|
155
|
print 'loading model ...'
|
84648488
Chunk
reverted.
|
156
157
158
159
160
161
162
163
164
165
166
167
|
with open('res/svm_opencv.model', 'rb') as modelfile:
model = pickle.load(modelfile)
feat = np.array(feat, dtype=np.float32)
return model.predict(feat)
def _test_opencv(self, X, Y, model=None):
if model is None:
if self.model != None:
model = self.model
else:
|
e3e7e73a
Chunk
spider standalone...
|
168
|
print 'loading model ...'
|
84648488
Chunk
reverted.
|
169
170
171
172
173
174
175
176
177
178
179
180
|
with open('res/svm_opencv.model', 'rb') as modelfile:
model = pickle.load(modelfile)
X, Y = np.array(X, dtype=np.float32), np.array(Y, dtype=np.float32)
# result_Y = np.array([self._predict_cv(x, model) for x in X])
result_Y = np.array(model.predict_all(X)).ravel()
return np.mean(Y == result_Y)
def _train_spark(self, X, Y=None):
|
e3e7e73a
Chunk
spider standalone...
|
181
|
if self.sparker == None:
|
84648488
Chunk
reverted.
|
182
183
184
185
|
self.sparker = SC.Sparker(host='HPC-server', appname='ImageCV', master='spark://HPC-server:7077')
self.model = self.sparker.train_svm(X, Y)
|
f4fb4381
Chunk
staged.
|
186
|
return svm
|
84648488
Chunk
reverted.
|
187
|
|
f4fb4381
Chunk
staged.
|
188
189
|
def _predict_spark(self, feat, model=None):
return self.sparker.predict_svm(feat, model)
|
84648488
Chunk
reverted.
|
190
191
192
|
def _test_spark(self, X, Y, model=None):
return self.sparker.test_svm(X, Y, model)
|
02528074
Chunk
staged.
|
193
|
|
f20e20ce
Chunk
staged.
|
194
|
|
e3e7e73a
Chunk
spider standalone...
|
195
|
def train(self, X, Y=None):
|
f20e20ce
Chunk
staged.
|
196
197
|
if self.toolset == 'sklearn':
|
02528074
Chunk
staged.
|
198
|
return self._train_sklearn(X, Y)
|
f20e20ce
Chunk
staged.
|
199
200
201
202
203
204
205
206
|
elif self.toolset == 'opencv':
return self._train_opencv(X, Y)
elif self.toolset == 'libsvm':
return self._train_libsvm(X, Y)
elif self.toolset == 'spark':
return self._train_spark(X, Y)
else:
raise Exception("Unknown toolset!")
|
84648488
Chunk
reverted.
|
207
|
|
02528074
Chunk
staged.
|
208
|
def predict(self, feat, model=None):
|
10b4f63f
Chunk
staged. Before Pa...
|
209
210
211
|
if self.toolset == 'sklearn':
return self._predict_sklearn(feat, model)
|
84648488
Chunk
reverted.
|
212
213
|
elif self.toolset == 'opencv':
return self._predict_opencv(feat, model)
|
10b4f63f
Chunk
staged. Before Pa...
|
214
215
216
217
218
219
|
elif self.toolset == 'libsvm':
return self._predict_libsvm(feat, model)
elif self.toolset == 'spark':
return self._predict_spark(feat, model)
else:
raise Exception("Unknown toolset!")
|
c7fa1d60
Chunk
refractoration st...
|
220
|
|
be12257b
Chunk
data-feat-model f...
|
221
|
|
f20e20ce
Chunk
staged.
|
222
|
def test(self, X, Y=None, model=None):
|
10b4f63f
Chunk
staged. Before Pa...
|
223
224
|
if self.toolset == 'sklearn':
|
84648488
Chunk
reverted.
|
225
226
|
return self._test_sklearn(X, Y, model)
elif self.toolset == 'opencv':
|
10b4f63f
Chunk
staged. Before Pa...
|
227
228
|
return self._test_opencv(X, Y, model)
elif self.toolset == 'libsvm':
|
02528074
Chunk
staged.
|
229
|
return self._test_libsvm(X, Y, model)
|
10b4f63f
Chunk
staged. Before Pa...
|
230
231
232
|
elif self.toolset == 'spark':
return self._test_spark(X, Y, model)
else:
|
f20e20ce
Chunk
staged.
|
233
|
raise Exception("Unknown toolset!")
|
84648488
Chunk
reverted.
|
|
|
02528074
Chunk
staged.
|
|
|
f20e20ce
Chunk
staged.
|
|
|
10b4f63f
Chunk
staged. Before Pa...
|
|
|
13a594f1
Chunk
before 中期答辩~~~
|
|
|
84648488
Chunk
reverted.
|
|
|
10b4f63f
Chunk
staged. Before Pa...
|
|
|
02528074
Chunk
staged.
|
|
|
10b4f63f
Chunk
staged. Before Pa...
|
|
|
84648488
Chunk
reverted.
|
|
|