8.7
HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/ pyspark --master yarn --deploy-mode client
textfile=sc.textFile("hdfs://master:9000/user/hduser/wordcount/input/LICENSE.txt")
textfile.count()
exit
cp /usr/local/spark/conf/spark-env.sh.template /usr/local/spark/conf/spark-env.sh
sudo gedit /usr/local/spark/conf/spark-env.sh
export SPARK_MASTER_IP=master
export SPARK_WORKER_CORES=1
export SPARK_WORKER_MEMORY=500m (最少480m,有设定)
export SPARK_WORKER_INSTANCES=3
sssh data1
sudo mkdir /usr/local/spark
sudo chown hduser:hduser /usr/local/spark/
exit
sudo scp -r /usr/local/spark hduser@data1:/usr/local
/usr/local/spark/sbin/start-all.sh
pyspark --master spark://master:7077 --num-executors 1 --total-executor-cores 3 --executor-memory 500m
bash Anaconda2-2.5.0-Linux-x86_64.sh -b
export PATH=/home/hduser/anaconda2/bin:$PATH
export ANACONDA_PATH=/home/hduser/anaconda2
export PYSPARK_DRIVER_PYTHON=$ANACONDA_PATH/bin/ipython
export PYSPARK_PYTHON=$ANACONDA_PATH/bin/python
sudo gedit ~/.bashrc
source ~/.bashrc
python --version
mkdir -p ~/pythonwork/ipynotebook
cd ~/pythonwork/ipynotebook/
PYSPARK_DRIVER_PYTHON=ipython PYSPARK_DRIVER_PYTHON_OPTS="notebook" pyspark （打开jupyter notebook）