#!/bin/zsh # chunk @ 2014 ######################################################################################## ## ## F**k World! ## ## ## export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:`hbase classpath` ## export SPARK_CLASSPATH=$SPARK_CLASSPATH:`hbase classpath` ## ## spark-submit \ ## --driver-memory 1g \ ## --executor-memory 1g \ ## --executor-cores 2 \ ## --deploy-mode client \ ## --master yarn \ ## --class "FuckWorld" \ ## $APP_JAR $ARGS ## ## spark-class org.apache.spark.deploy.yarn.Client \ ## --num-executors 2 \ ## --executor-cores 2 \ ## --driver-memory 1g \ ## --executor-memory 1g \ ## --name "F**k World" \ ## --jar $APP_JAR \ ## --class "FuckWorld" \ ## --args $ARGS ## ######################################################################################## source /home/hadoop/.zshrc export _JAVA_OPTIONS="-Djava.library.path=/usr/local/lib " export SPARK_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` export SPARK_JAR=hdfs://HPC-server:9000/user/spark/share/lib/spark-assembly-1.2.0-hadoop2.5.1.jar export SPARK_LIBRARY_PATH=$SPARK_LIBRARY_PATH:/usr/local/lib APP_JAR=/home/hadoop/workspace/idea/test/ImageS/target/scala-2.11/images_2.11-1.0.jar ARGS="ImgTest" #spark-submit \ # --driver-memory 1g \ # --executor-memory 1g \ # --executor-cores 2 \ # --master spark://HPC-server:7077 \ # --conf "spark.executor.extraJavaOptions=-Djava.library.path=/usr/local/lib" \ # --jars /home/hadoop/workspace/idea/test/ImageS/lib/opencv-2410.jar \ # --class "FeatHOGExtractor" \ # $APP_JAR $ARGS spark-submit \ --verbose \ --driver-memory 512m \ --executor-memory 512m \ --executor-cores 2 \ --master spark://HPC-server:7077 \ --driver-library-path /usr/local/lib \ --jars /usr/local/lib/opencv-2410.jar \ --class "cn.hpclab.sc.FeatHOGExtractor" \ $APP_JAR $ARGS