run.sh
1.89 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
#!/bin/zsh
# chunk @ 2014
########################################################################################
##
## F**k World!
##
##
## export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:`hbase classpath`
## export SPARK_CLASSPATH=$SPARK_CLASSPATH:`hbase classpath`
##
## spark-submit \
## --driver-memory 1g \
## --executor-memory 1g \
## --executor-cores 2 \
## --deploy-mode client \
## --master yarn \
## --class "FuckWorld" \
## $APP_JAR $ARGS
##
## spark-class org.apache.spark.deploy.yarn.Client \
## --num-executors 2 \
## --executor-cores 2 \
## --driver-memory 1g \
## --executor-memory 1g \
## --name "F**k World" \
## --jar $APP_JAR \
## --class "FuckWorld" \
## --args $ARGS
##
##spark-submit \
## --driver-memory 1g \
## --executor-memory 1g \
## --executor-cores 2 \
## --master spark://HPC-server:7077 \
## --jars $SPARK_HOME/lib/spark-examples-1.2.0-hadoop2.5.1.jar \
## $APP_JAR $ARGS
########################################################################################
source /home/hadoop/.zshrc
v env1
export PYSPARK_PYTHON=/home/hadoop/.virtualenvs/env1/bin/python
export SPARK_CLASSPATH=`hbase classpath`
export SPARK_JAR=hdfs://HPC-server:9000/user/spark/share/lib/spark-assembly-1.2.0-hadoop2.5.1.jar
#COMPRESSED=/home/hadoop/workspace/pycharm/test/ImageR.zip
# --py-files $COMPRESSED \
COMPRESSED=/home/hadoop/workspace/pycharm/tmp/ImageR/mdata.zip,/home/hadoop/workspace/pycharm/tmp/ImageR/mfeat.zip,/home/hadoop/workspace/pycharm/tmp/ImageR/mjpeg.zip,/home/hadoop/workspace/pycharm/tmp/ImageR/msteg.zip,/home/hadoop/workspace/pycharm/tmp/ImageR/mmodel.zip,/home/hadoop/workspace/pycharm/tmp/ImageR/mspark.zip
APP=test_spark.py
#APP=test_model.py
ARGS=
spark-submit \
--driver-memory 1g \
--executor-memory 2g \
--executor-cores 2 \
--master spark://HPC-server:7077 \
--jars $SPARK_HOME/lib/spark-examples-1.2.0-hadoop2.5.1.jar \
--py-files $COMPRESSED \
$APP $ARGS