staged.

Chunk
1 parent 5ec38adb
Showing 24 changed files with 91 additions and 15 deletions Show diff stats
.idea/ImageR.iml
common.pyc
mdata/CV.pyc
mdata/MSR.pyc
mdata/__init__.pyc
mfeat/HOG.pyc
mfeat/IntraBlockDiff.pyc
mfeat/__init__.pyc
mjpeg/__init__.pyc
mjpeg/base.pyc
mjpeg/dct.pyc
mmodel/SVM.py
mmodel/SVM.pyc
mmodel/__init__.pyc
mmodel/libsvm.pyc
mmodel/svmutil.pyc
mspark/SC.py
mspark/SC.pyc
mspark/__init__.pyc
mspark/dependencies.pyc
@@ -2,7 +2,7 @@
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$" />
-    <orderEntry type="jdk" jdkName="Python 2.7.8 virtualenv at ~/.virtualenvs/env1" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 2.7.6 virtualenv at ~/.virtualenvs/env0" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>
 \ No newline at end of file
@@ -8,6 +8,7 @@ chunkplus@gmail.com
 from mfeat import *
 from mmodel import *
 from svmutil import *
+from mspark import SC
 from common import *
 import numpy as np
@@ -25,10 +26,13 @@ dict_featbuf = {}
 class ModelSVM(ModelBase):
-    def __init__(self):
+    def __init__(self, toolset='sklearn'):
         ModelBase.__init__(self)
+        self.toolset = toolset
+        self.sparker = None
-    def _train_lib(self, X, Y):
+
+    def _train_libscm(self, X, Y):
         X, Y = list(X), list(Y)
         prob = svm_problem(Y, X)
@@ -41,7 +45,7 @@ class ModelSVM(ModelBase):
         return m
-    def _predict_lib(self, feat, model=None):
+    def _predict_libsvm(self, feat, model=None):
         if model is None:
             if self.model != None:
                 model = self.model
@@ -54,7 +58,7 @@ class ModelSVM(ModelBase):
         return label
-    def _test_lib(self, X, Y, model=None):
+    def _test_libsvm(self, X, Y, model=None):
         if model is None:
             if self.model != None:
                 model = self.model
@@ -67,7 +71,7 @@ class ModelSVM(ModelBase):
         return p_acc
-    def _train_sk(self, X, Y):
+    def _train_sklearn(self, X, Y):
         lin_clf = svm.LinearSVC()
         lin_clf.fit(X, Y)
         with open('res/svm_sk.model', 'wb') as modelfile:
@@ -78,7 +82,7 @@ class ModelSVM(ModelBase):
         return lin_clf
-    def _predict_sk(self, feat, model=None):
+    def _predict_sklearn(self, feat, model=None):
         """N.B. sklearn.svm.base.predict :
             Perform classification on samples in X.
                 Parameters
@@ -99,7 +103,7 @@ class ModelSVM(ModelBase):
         return model.predict(feat)
-    def _test_sk(self, X, Y, model=None):
+    def _test_sklearn(self, X, Y, model=None):
         if model is None:
             if self.model != None:
                 model = self.model
@@ -107,12 +111,12 @@ class ModelSVM(ModelBase):
                 with open('res/svm_sk.model', 'rb') as modelfile:
                     model = pickle.load(modelfile)
-        result_Y = np.array(self._predict_sk(X, model))
+        result_Y = np.array(self._predict_sklearn(X, model))
         return np.mean(Y == result_Y)
-    def _train_cv(self, X, Y):
+    def _train_opencv(self, X, Y):
         svm_params = dict(kernel_type=cv2.SVM_LINEAR,
                           svm_type=cv2.SVM_C_SVC,
                           C=2.67, gamma=5.383)
@@ -128,7 +132,7 @@ class ModelSVM(ModelBase):
         return svm
-    def _predict_cv(self, feat, model=None):
+    def _predict_opencv(self, feat, model=None):
         if model is None:
             if self.model != None:
                 model = self.model
@@ -140,7 +144,7 @@ class ModelSVM(ModelBase):
         return model.predict(feat)
-    def _test_cv(self, X, Y, model=None):
+    def _test_opencv(self, X, Y, model=None):
         if model is None:
             if self.model != None:
                 model = self.model
@@ -156,14 +160,43 @@ class ModelSVM(ModelBase):
         return np.mean(Y == result_Y)
+    def _train_spark(self, X, Y):
+        if self.sparker == None:
+            self.sparker = SC.Sparker(host='HPC-server', appname='ImageCV', master='spark://HPC-server:7077')
+
+        self.model = self.sparker.train_svm(X, Y)
+        return svm
+
+    def _predict_spark(self, feat, model=None):
+        return self.sparker.predict_svm(feat, model)
+
+    def _test_spark(self, X, Y, model=None):
+        return self.sparker.test_svm(X, Y, model)
+
+
     def train(self, X, Y):
-        return self._train_sk(X, Y)
+        try:
+            return locals()["self._train_" + self.toolset](X, Y)
+        except Exception as expt:
+            print "Exception when training!"
+            raise
     def predict(self, feat, model=None):
-        return self._predict_sk(feat, model)
+
+        try:
+            return locals()["self._predict_" + self.toolset](X, Y)
+        except Exception as expt:
+            print "Exception when predicting!"
+            raise
+
     def test(self, X, Y, model=None):
-        return self._test_sk(X, Y, model)
+
+        try:
+            return locals()["self._test_" + self.toolset](X, Y)
+        except Exception as expt:
+            print "Exception when predicting!"
+            raise
@@ -132,6 +132,9 @@ class Sparker(object):
         return model.predict(x)
+    def test_svm(self, X, Y, model=None):
+        pass
+
@@ -0,0 +1,40 @@
+    def train(self, X, Y):
+        try:
+
+        if self.toolset == 'sklearn':
+            return self._train_sklearn(X, Y)
+        elif self.toolset == 'opencv':
+            return self._train_opencv(X, Y)
+        elif self.toolset == 'libsvm':
+            return self._train_libscm(X, Y)
+        elif self.toolset == 'spark':
+            return self._train_spark(X, Y)
+        else:
+            raise Exception("Unknown toolset!")
+
+    def predict(self, feat, model=None):
+
+        if self.toolset == 'sklearn':
+            return self._predict_sklearn(feat, model)
+        elif self.toolset == 'opencv':
+            return self._predict_opencv(feat, model)
+        elif self.toolset == 'libsvm':
+            return self._predict_libsvm(feat, model)
+        elif self.toolset == 'libsvm':
+            return self._predict_spark(feat, model)
+        else:
+            raise Exception("Unknown toolset!")
+
+
+    def test(self, X, Y, model=None):
+
+        if self.toolset == 'sklearn':
+            return self._test_sklearn(X, Y, model)
+        elif self.toolset == 'opencv':
+            return self._test_opencv(X, Y, model)
+        elif self.toolset == 'libsvm':
+            return self._test_libsvm(X, Y, model)
+        elif self.toolset == 'libsvm':
+            return self._test_spark(X, Y, model)
+        else:
+            raise Exception("Unknown toolset!")
 \ No newline at end of file
	@@ -2,7 +2,7 @@		@@ -2,7 +2,7 @@
2	<module type="PYTHON_MODULE" version="4">	2	<module type="PYTHON_MODULE" version="4">
3	<component name="NewModuleRootManager">	3	<component name="NewModuleRootManager">
4	<content url="file://$MODULE_DIR$" />	4	<content url="file://$MODULE_DIR$" />
5	- <orderEntry type="jdk" jdkName="Python 2.7.8 virtualenv at ~/.virtualenvs/env1" jdkType="Python SDK" />	5	+ <orderEntry type="jdk" jdkName="Python 2.7.6 virtualenv at ~/.virtualenvs/env0" jdkType="Python SDK" />
6	<orderEntry type="sourceFolder" forTests="false" />	6	<orderEntry type="sourceFolder" forTests="false" />
7	</component>	7	</component>
8	</module>	8	</module>
9	\ No newline at end of file	9	\ No newline at end of file
	@@ -132,6 +132,9 @@ class Sparker(object):		@@ -132,6 +132,9 @@ class Sparker(object):
132		132
133	return model.predict(x)	133	return model.predict(x)
134		134
		135	+ def test_svm(self, X, Y, model=None):
		136	+ pass
		137	+
135		138
136		139
137		140
	@@ -0,0 +1,40 @@		@@ -0,0 +1,40 @@
		1	+ def train(self, X, Y):
		2	+ try:
		3	+
		4	+ if self.toolset == 'sklearn':
		5	+ return self._train_sklearn(X, Y)
		6	+ elif self.toolset == 'opencv':
		7	+ return self._train_opencv(X, Y)
		8	+ elif self.toolset == 'libsvm':
		9	+ return self._train_libscm(X, Y)
		10	+ elif self.toolset == 'spark':
		11	+ return self._train_spark(X, Y)
		12	+ else:
		13	+ raise Exception("Unknown toolset!")
		14	+
		15	+ def predict(self, feat, model=None):
		16	+
		17	+ if self.toolset == 'sklearn':
		18	+ return self._predict_sklearn(feat, model)
		19	+ elif self.toolset == 'opencv':
		20	+ return self._predict_opencv(feat, model)
		21	+ elif self.toolset == 'libsvm':
		22	+ return self._predict_libsvm(feat, model)
		23	+ elif self.toolset == 'libsvm':
		24	+ return self._predict_spark(feat, model)
		25	+ else:
		26	+ raise Exception("Unknown toolset!")
		27	+
		28	+
		29	+ def test(self, X, Y, model=None):
		30	+
		31	+ if self.toolset == 'sklearn':
		32	+ return self._test_sklearn(X, Y, model)
		33	+ elif self.toolset == 'opencv':
		34	+ return self._test_opencv(X, Y, model)
		35	+ elif self.toolset == 'libsvm':
		36	+ return self._test_libsvm(X, Y, model)
		37	+ elif self.toolset == 'libsvm':
		38	+ return self._test_spark(X, Y, model)
		39	+ else:
		40	+ raise Exception("Unknown toolset!")
0	\ No newline at end of file	41	\ No newline at end of file