spark

export SPARK_HOME='/home/hadoop/spark/'
export PYSPARK_DRIVER_PYTHON=jupyter
export PYSPARK_DRIVER_PYTHON_OPTS='notebook --ip=0.0.0.0'

vi ~/.bashrc
export PYSPARK_DRIVER_PYTHON=jupyter
export PYSPARK_DRIVER_PYTHON_OPTS='notebook --ip=0.0.0.0'
두줄 추가
source ~/.bashrc
초기화
mkdir sparkwork
cd sparkwork
pyspark --master yarn --num-executors 2
주피터로 파이스파크 띄우기