run.sh
1.78 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
#!/bin/zsh
# chunk @ 2014
########################################################################################
##
## F**k World!
##
##
## export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:`hbase classpath`
## export SPARK_CLASSPATH=$SPARK_CLASSPATH:`hbase classpath`
##
## spark-submit \
## --driver-memory 1g \
## --executor-memory 1g \
## --executor-cores 2 \
## --deploy-mode client \
## --master yarn \
## --class "FuckWorld" \
## $APP_JAR $ARGS
##
## spark-class org.apache.spark.deploy.yarn.Client \
## --num-executors 2 \
## --executor-cores 2 \
## --driver-memory 1g \
## --executor-memory 1g \
## --name "F**k World" \
## --jar $APP_JAR \
## --class "FuckWorld" \
## --args $ARGS
##
########################################################################################
source /home/hadoop/.zshrc
export _JAVA_OPTIONS="-Djava.library.path=/usr/local/lib "
export SPARK_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath`
export SPARK_JAR=hdfs://HPC-server:9000/user/spark/share/lib/spark-assembly-1.2.0-hadoop2.5.1.jar
export SPARK_LIBRARY_PATH=$SPARK_LIBRARY_PATH:/usr/local/lib
APP_JAR=/home/hadoop/workspace/idea/test/ImageS/target/scala-2.11/images_2.11-1.0.jar
ARGS="ImgTest"
#spark-submit \
# --driver-memory 1g \
# --executor-memory 1g \
# --executor-cores 2 \
# --master spark://HPC-server:7077 \
# --conf "spark.executor.extraJavaOptions=-Djava.library.path=/usr/local/lib" \
# --jars /home/hadoop/workspace/idea/test/ImageS/lib/opencv-2410.jar \
# --class "FeatHOGExtractor" \
# $APP_JAR $ARGS
spark-submit \
--verbose \
--driver-memory 512m \
--executor-memory 512m \
--executor-cores 2 \
--master spark://HPC-server:7077 \
--driver-library-path /usr/local/lib \
--jars /usr/local/lib/opencv-2410.jar \
--class "cn.hpclab.sc.FeatHOGExtractor" \
$APP_JAR $ARGS