2. 하둡 설정

단계1. hdfs-site.xml

• hdfs에서 사용할 환경정보를 설정하는 파일이다.

vim $HADOOP_HOME/etc/hadoop/hdfs-site.xml
# 아래 내용으로 수정 후 저장
<configuration>
        <!-- configuration hadoop -->
        <property>
                <name>dfs.replication</name>
                <value>2</value>
        </property>
        <property>
                <name>dfs.namenode.name.dir</name>
                <value>/usr/local/hadoop/data/nameNode</value>
        </property>
        <property>
                <name>dfs.datanode.data.dir</name>
                <value>/usr/local/hadoop/data/dataNode</value>
        </property>
        <property>
                <name>dfs.journalnode.edits.dir</name>
                <value>/usr/local/hadoop/data/dfs/journalnode</value>
        </property>
        <property>
                <name>dfs.nameservices</name>
                <value>my-hadoop-cluster</value>
        </property>
        <property>
                <name>dfs.ha.namenodes.my-hadoop-cluster</name>
                <value>namenode1,namenode2</value>
        </property>
        <property>
                <name>dfs.namenode.rpc-address.my-hadoop-cluster.namenode1</name>
                <value>master:8020</value>
        </property>
        <property>
                <name>dfs.namenode.rpc-address.my-hadoop-cluster.namenode2</name>
                <value>worker1:8020</value>
        </property>
        <property>
                <name>dfs.namenode.http-address.my-hadoop-cluster.namenode1</name>
                <value>master:50070</value>
        </property>
        <property>
                <name>dfs.namenode.http-address.my-hadoop-cluster.namenode2</name>
                <value>worker1:50070</value>
        </property>
        <property>
                <name>dfs.namenode.shared.edits.dir</name>
                <value>qjournal://master:8485;worker1:8485;worker2:8485/my-hadoop-cluster</value>
        </property>
        <property>
                <name>dfs.client.failover.proxy.provider.my-hadoop-cluster</name>
                <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
        </property>
        <property>
                <name>dfs.ha.fencing.methods</name>
                <value>shell(/bin/true)</value>
        </property>
        <property>
                <name>dfs.ha.fencing.ssh.private-key-files</name>
                <value>/home/ubuntu/.ssh/id_rsa</value>
        </property>
        <property> 
                <name>dfs.ha.automatic-failover.enabled</name>
                <value>true</value>
        </property>
        <property>
                <name>dfs.name.dir</name>
                <value>/usr/local/hadoop/data/name</value>
        </property>
        <property>
                <name>dfs.data.dir</name>
                <value>/usr/local/hadoop/data/data</value>
        </property>
</configuration>

단계2. core-site.xml

• hadoop 시스템 설정 파일

vim $HADOOP_HOME/etc/hadoop/core-site.xml
# 아래 내용으로 수정 후 저장
<configuration>
        <property>
                <name>fs.default.name</name>
                <value>hdfs://master:9000</value>
        </property>
        <property>
                <name>fs.defaultFS</name>
                <value>hdfs://my-hadoop-cluster</value>
        </property>
        <property>
                <name>ha.zookeeper.quorum</name>
                <value>master:2181,worker1:2181,worker2:2181</value>
        </property>
</configuration>

단계3. yarn-site.xml

vim $HADOOP_HOME/etc/hadoop/yarn-site.xml

# 아래 내용으로 수정 후 저장
<configuration>
        <!-- Site specific YARN configuration properties -->
        <property>
                <name>yarn.nodemanager.aux-services</name>
                <value>mapreduce_shuffle</value>
        </property>
        <property>
                <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
                <value>org.apache.hadoop.mapred.ShuffleHandler</value>
        </property>
        <property>
                <name>yarn.resourcemanager.hostname</name>
                <value>master</value>
        </property>
        <property>
                <name>yarn.nodemanager.vmem-check-enabled</name>
                <value>false</value>
        </property>
</configuration>

단계4. mapred-site.xml

vim $HADOOP_HOME/etc/hadoop/mapred-site.xml

# 아래 내용으로 수정 후 저장
<configuration>
        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>
        <property>
                <name>yarn.app.mapreduce.am.env</name>
                <value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
        </property>
        <property>
                <name>mapreduce.map.env</name>
                <value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
        </property>
        <property>
                <name>mapreduce.reduce.env</name>
                <value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
        </property>
</configuration>

단계5. hadoop-env.sh

vim $HADOOP_HOME/etc/hadoop/hadoop-env.sh

# 아래 내용 수정 후 저장
# Java
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64

# Hadoop
export HADOOP_HOME=/usr/local/hadoop

단계6. hadoop workers

• hadoop의 worker로 동작할 서버 호스트 이름을 설정

vim $HADOOP_HOME/etc/hadoop/workers

# 아래 내용 수정 후 저장
# localhost << 주석 처리 또는 제거
worker1
worker2
worker3

단계7. hadoop masters

• hadoop의 master로 동작할 서버 호스트 이름을 설정

# Hadoop masters 편집
vim $HADOOP_HOME/etc/hadoop/masters

# 아래 내용 수정 후 저장
master
worker1