staged.

Chunk
1 parent 5ec38adb
Showing 24 changed files with 91 additions and 15 deletions Show diff stats
.idea/ImageR.iml
common.pyc
mdata/CV.pyc
mdata/MSR.pyc
mdata/__init__.pyc
mfeat/HOG.pyc
mfeat/IntraBlockDiff.pyc
mfeat/__init__.pyc
mjpeg/__init__.pyc
mjpeg/base.pyc
mjpeg/dct.pyc
mmodel/SVM.py
mmodel/SVM.pyc
mmodel/__init__.pyc
mmodel/libsvm.pyc
mmodel/svmutil.pyc
mspark/SC.py
mspark/SC.pyc
mspark/__init__.pyc
mspark/dependencies.pyc
@@ -2,7 +2,7 @@
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$" />
-    <orderEntry type="jdk" jdkName="Python 2.7.8 virtualenv at ~/.virtualenvs/env1" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 2.7.6 virtualenv at ~/.virtualenvs/env0" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>
 \ No newline at end of file
@@ -8,6 +8,7 @@ chunkplus@gmail.com
 from mfeat import *
 from mmodel import *
 from svmutil import *
+from mspark import SC
  
 from common import *
 import numpy as np
@@ -25,10 +26,13 @@ dict_featbuf = {}
  
  
 class ModelSVM(ModelBase):
-    def __init__(self):
+    def __init__(self, toolset='sklearn'):
         ModelBase.__init__(self)
+        self.toolset = toolset
+        self.sparker = None
  
-    def _train_lib(self, X, Y):
+
+    def _train_libscm(self, X, Y):
         X, Y = list(X), list(Y)
  
         prob = svm_problem(Y, X)
@@ -41,7 +45,7 @@ class ModelSVM(ModelBase):
  
         return m
  
-    def _predict_lib(self, feat, model=None):
+    def _predict_libsvm(self, feat, model=None):
         if model is None:
             if self.model != None:
                 model = self.model
@@ -54,7 +58,7 @@ class ModelSVM(ModelBase):
         return label
  
  
-    def _test_lib(self, X, Y, model=None):
+    def _test_libsvm(self, X, Y, model=None):
         if model is None:
             if self.model != None:
                 model = self.model
@@ -67,7 +71,7 @@ class ModelSVM(ModelBase):
  
         return p_acc
  
-    def _train_sk(self, X, Y):
+    def _train_sklearn(self, X, Y):
         lin_clf = svm.LinearSVC()
         lin_clf.fit(X, Y)
         with open('res/svm_sk.model', 'wb') as modelfile:
@@ -78,7 +82,7 @@ class ModelSVM(ModelBase):
         return lin_clf
  
  
-    def _predict_sk(self, feat, model=None):
+    def _predict_sklearn(self, feat, model=None):
         """N.B. sklearn.svm.base.predict :
             Perform classification on samples in X.
                 Parameters
@@ -99,7 +103,7 @@ class ModelSVM(ModelBase):
  
         return model.predict(feat)
  
-    def _test_sk(self, X, Y, model=None):
+    def _test_sklearn(self, X, Y, model=None):
         if model is None:
             if self.model != None:
                 model = self.model
@@ -107,12 +111,12 @@ class ModelSVM(ModelBase):
                 with open('res/svm_sk.model', 'rb') as modelfile:
                     model = pickle.load(modelfile)
  
-        result_Y = np.array(self._predict_sk(X, model))
+        result_Y = np.array(self._predict_sklearn(X, model))
  
         return np.mean(Y == result_Y)
  
  
-    def _train_cv(self, X, Y):
+    def _train_opencv(self, X, Y):
         svm_params = dict(kernel_type=cv2.SVM_LINEAR,
                           svm_type=cv2.SVM_C_SVC,
                           C=2.67, gamma=5.383)
@@ -128,7 +132,7 @@ class ModelSVM(ModelBase):
         return svm
  
  
-    def _predict_cv(self, feat, model=None):
+    def _predict_opencv(self, feat, model=None):
         if model is None:
             if self.model != None:
                 model = self.model
@@ -140,7 +144,7 @@ class ModelSVM(ModelBase):
         return model.predict(feat)
  
  
-    def _test_cv(self, X, Y, model=None):
+    def _test_opencv(self, X, Y, model=None):
         if model is None:
             if self.model != None:
                 model = self.model
@@ -156,14 +160,43 @@ class ModelSVM(ModelBase):
         return np.mean(Y == result_Y)
  
  
+    def _train_spark(self, X, Y):
+        if self.sparker == None:
+            self.sparker = SC.Sparker(host='HPC-server', appname='ImageCV', master='spark://HPC-server:7077')
+
+        self.model = self.sparker.train_svm(X, Y)
+        return svm
+
+    def _predict_spark(self, feat, model=None):
+        return self.sparker.predict_svm(feat, model)
+
+    def _test_spark(self, X, Y, model=None):
+        return self.sparker.test_svm(X, Y, model)
+
+
     def train(self, X, Y):
-        return self._train_sk(X, Y)
+        try:
+            return locals()["self._train_" + self.toolset](X, Y)
+        except Exception as expt:
+            print "Exception when training!"
+            raise
  
     def predict(self, feat, model=None):
-        return self._predict_sk(feat, model)
+
+        try:
+            return locals()["self._predict_" + self.toolset](X, Y)
+        except Exception as expt:
+            print "Exception when predicting!"
+            raise
+
  
     def test(self, X, Y, model=None):
-        return self._test_sk(X, Y, model)
+
+        try:
+            return locals()["self._test_" + self.toolset](X, Y)
+        except Exception as expt:
+            print "Exception when predicting!"
+            raise
  
  
  
@@ -132,6 +132,9 @@ class Sparker(object):
  
         return model.predict(x)
  
+    def test_svm(self, X, Y, model=None):
+        pass
+
  
  
  
@@ -0,0 +1,40 @@
+    def train(self, X, Y):
+        try:
+
+        if self.toolset == 'sklearn':
+            return self._train_sklearn(X, Y)
+        elif self.toolset == 'opencv':
+            return self._train_opencv(X, Y)
+        elif self.toolset == 'libsvm':
+            return self._train_libscm(X, Y)
+        elif self.toolset == 'spark':
+            return self._train_spark(X, Y)
+        else:
+            raise Exception("Unknown toolset!")
+
+    def predict(self, feat, model=None):
+
+        if self.toolset == 'sklearn':
+            return self._predict_sklearn(feat, model)
+        elif self.toolset == 'opencv':
+            return self._predict_opencv(feat, model)
+        elif self.toolset == 'libsvm':
+            return self._predict_libsvm(feat, model)
+        elif self.toolset == 'libsvm':
+            return self._predict_spark(feat, model)
+        else:
+            raise Exception("Unknown toolset!")
+
+
+    def test(self, X, Y, model=None):
+
+        if self.toolset == 'sklearn':
+            return self._test_sklearn(X, Y, model)
+        elif self.toolset == 'opencv':
+            return self._test_opencv(X, Y, model)
+        elif self.toolset == 'libsvm':
+            return self._test_libsvm(X, Y, model)
+        elif self.toolset == 'libsvm':
+            return self._test_spark(X, Y, model)
+        else:
+            raise Exception("Unknown toolset!")
 \ No newline at end of file
...	...	@@ -2,7 +2,7 @@
2	2	<module type="PYTHON_MODULE" version="4">
3	3	<component name="NewModuleRootManager">
4	4	<content url="file://$MODULE_DIR$" />
5		- <orderEntry type="jdk" jdkName="Python 2.7.8 virtualenv at ~/.virtualenvs/env1" jdkType="Python SDK" />
	5	+ <orderEntry type="jdk" jdkName="Python 2.7.6 virtualenv at ~/.virtualenvs/env0" jdkType="Python SDK" />
6	6	<orderEntry type="sourceFolder" forTests="false" />
7	7	</component>
8	8	</module>
9	9	\ No newline at end of file
...	...
...	...	@@ -8,6 +8,7 @@ chunkplus@gmail.com
8	8	from mfeat import *
9	9	from mmodel import *
10	10	from svmutil import *
	11	+from mspark import SC
11	12
12	13	from common import *
13	14	import numpy as np
...	...	@@ -25,10 +26,13 @@ dict_featbuf = {}
25	26
26	27
27	28	class ModelSVM(ModelBase):
28		- def __init__(self):
	29	+ def __init__(self, toolset='sklearn'):
29	30	ModelBase.__init__(self)
	31	+ self.toolset = toolset
	32	+ self.sparker = None
30	33
31		- def _train_lib(self, X, Y):
	34	+
	35	+ def _train_libscm(self, X, Y):
32	36	X, Y = list(X), list(Y)
33	37
34	38	prob = svm_problem(Y, X)
...	...	@@ -41,7 +45,7 @@ class ModelSVM(ModelBase):
41	45
42	46	return m
43	47
44		- def _predict_lib(self, feat, model=None):
	48	+ def _predict_libsvm(self, feat, model=None):
45	49	if model is None:
46	50	if self.model != None:
47	51	model = self.model
...	...	@@ -54,7 +58,7 @@ class ModelSVM(ModelBase):
54	58	return label
55	59
56	60
57		- def _test_lib(self, X, Y, model=None):
	61	+ def _test_libsvm(self, X, Y, model=None):
58	62	if model is None:
59	63	if self.model != None:
60	64	model = self.model
...	...	@@ -67,7 +71,7 @@ class ModelSVM(ModelBase):
67	71
68	72	return p_acc
69	73
70		- def _train_sk(self, X, Y):
	74	+ def _train_sklearn(self, X, Y):
71	75	lin_clf = svm.LinearSVC()
72	76	lin_clf.fit(X, Y)
73	77	with open('res/svm_sk.model', 'wb') as modelfile:
...	...	@@ -78,7 +82,7 @@ class ModelSVM(ModelBase):
78	82	return lin_clf
79	83
80	84
81		- def _predict_sk(self, feat, model=None):
	85	+ def _predict_sklearn(self, feat, model=None):
82	86	"""N.B. sklearn.svm.base.predict :
83	87	Perform classification on samples in X.
84	88	Parameters
...	...	@@ -99,7 +103,7 @@ class ModelSVM(ModelBase):
99	103
100	104	return model.predict(feat)
101	105
102		- def _test_sk(self, X, Y, model=None):
	106	+ def _test_sklearn(self, X, Y, model=None):
103	107	if model is None:
104	108	if self.model != None:
105	109	model = self.model
...	...	@@ -107,12 +111,12 @@ class ModelSVM(ModelBase):
107	111	with open('res/svm_sk.model', 'rb') as modelfile:
108	112	model = pickle.load(modelfile)
109	113
110		- result_Y = np.array(self._predict_sk(X, model))
	114	+ result_Y = np.array(self._predict_sklearn(X, model))
111	115
112	116	return np.mean(Y == result_Y)
113	117
114	118
115		- def _train_cv(self, X, Y):
	119	+ def _train_opencv(self, X, Y):
116	120	svm_params = dict(kernel_type=cv2.SVM_LINEAR,
117	121	svm_type=cv2.SVM_C_SVC,
118	122	C=2.67, gamma=5.383)
...	...	@@ -128,7 +132,7 @@ class ModelSVM(ModelBase):
128	132	return svm
129	133
130	134
131		- def _predict_cv(self, feat, model=None):
	135	+ def _predict_opencv(self, feat, model=None):
132	136	if model is None:
133	137	if self.model != None:
134	138	model = self.model
...	...	@@ -140,7 +144,7 @@ class ModelSVM(ModelBase):
140	144	return model.predict(feat)
141	145
142	146
143		- def _test_cv(self, X, Y, model=None):
	147	+ def _test_opencv(self, X, Y, model=None):
144	148	if model is None:
145	149	if self.model != None:
146	150	model = self.model
...	...	@@ -156,14 +160,43 @@ class ModelSVM(ModelBase):
156	160	return np.mean(Y == result_Y)
157	161
158	162
	163	+ def _train_spark(self, X, Y):
	164	+ if self.sparker == None:
	165	+ self.sparker = SC.Sparker(host='HPC-server', appname='ImageCV', master='spark://HPC-server:7077')
	166	+
	167	+ self.model = self.sparker.train_svm(X, Y)
	168	+ return svm
	169	+
	170	+ def _predict_spark(self, feat, model=None):
	171	+ return self.sparker.predict_svm(feat, model)
	172	+
	173	+ def _test_spark(self, X, Y, model=None):
	174	+ return self.sparker.test_svm(X, Y, model)
	175	+
	176	+
159	177	def train(self, X, Y):
160		- return self._train_sk(X, Y)
	178	+ try:
	179	+ return locals()["self._train_" + self.toolset](X, Y)
	180	+ except Exception as expt:
	181	+ print "Exception when training!"
	182	+ raise
161	183
162	184	def predict(self, feat, model=None):
163		- return self._predict_sk(feat, model)
	185	+
	186	+ try:
	187	+ return locals()["self._predict_" + self.toolset](X, Y)
	188	+ except Exception as expt:
	189	+ print "Exception when predicting!"
	190	+ raise
	191	+
164	192
165	193	def test(self, X, Y, model=None):
166		- return self._test_sk(X, Y, model)
	194	+
	195	+ try:
	196	+ return locals()["self._test_" + self.toolset](X, Y)
	197	+ except Exception as expt:
	198	+ print "Exception when predicting!"
	199	+ raise
167	200
168	201
169	202
...	...
...	...	@@ -132,6 +132,9 @@ class Sparker(object):
132	132
133	133	return model.predict(x)
134	134
	135	+ def test_svm(self, X, Y, model=None):
	136	+ pass
	137	+
135	138
136	139
137	140
...	...
...	...	@@ -0,0 +1,40 @@
	1	+ def train(self, X, Y):
	2	+ try:
	3	+
	4	+ if self.toolset == 'sklearn':
	5	+ return self._train_sklearn(X, Y)
	6	+ elif self.toolset == 'opencv':
	7	+ return self._train_opencv(X, Y)
	8	+ elif self.toolset == 'libsvm':
	9	+ return self._train_libscm(X, Y)
	10	+ elif self.toolset == 'spark':
	11	+ return self._train_spark(X, Y)
	12	+ else:
	13	+ raise Exception("Unknown toolset!")
	14	+
	15	+ def predict(self, feat, model=None):
	16	+
	17	+ if self.toolset == 'sklearn':
	18	+ return self._predict_sklearn(feat, model)
	19	+ elif self.toolset == 'opencv':
	20	+ return self._predict_opencv(feat, model)
	21	+ elif self.toolset == 'libsvm':
	22	+ return self._predict_libsvm(feat, model)
	23	+ elif self.toolset == 'libsvm':
	24	+ return self._predict_spark(feat, model)
	25	+ else:
	26	+ raise Exception("Unknown toolset!")
	27	+
	28	+
	29	+ def test(self, X, Y, model=None):
	30	+
	31	+ if self.toolset == 'sklearn':
	32	+ return self._test_sklearn(X, Y, model)
	33	+ elif self.toolset == 'opencv':
	34	+ return self._test_opencv(X, Y, model)
	35	+ elif self.toolset == 'libsvm':
	36	+ return self._test_libsvm(X, Y, model)
	37	+ elif self.toolset == 'libsvm':
	38	+ return self._test_spark(X, Y, model)
	39	+ else:
	40	+ raise Exception("Unknown toolset!")
0	41	\ No newline at end of file
...	...