Flume 은 로그 수집기
UserGuide : https://flume.apache.org/releases/content/1.9.0/FlumeUserGuide.html
mediawiki : https://www.jopenbusiness.com/mediawiki/Flume
RIDI BOOK의 Flume 사용 예제 : https://ridicorp.com/story/weblog-flume-cdh/
해당 사이트에서 apache-flume-1.9.0-bin.tar.gz
다운로드해서 직접 넣거나
https://www.apache.org/dyn/closer.lua/flume/1.9.0/apache-flume-1.9.0-bin.tar.gz
아래 wget
으로 다운로드
$wget https://downloads.apache.org/flume/1.9.0/apache-flume-1.9.0-bin.tar.gz
$tar -xvzf apache-flume-1.9.0-bin.tar.gz
$cp -r apache-flume-1.9.0-bin /home/hadoop/flume-1.9.0
$vi .bashrc
export FLUME_HOME=/home/hadoop/flume-1.9.0
export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin:$HIVE_HOME/bin:$SQOOP_HOME/bin:$FLUME_HOME/bin
$source .bashrc
$flume-ng version
$flume-ng help
$hdfs namenode -format
$start-dfs.sh
$start-yarn.sh
$jps
10529 NodeManager
9747 NameNode
10087 SecondaryNameNode
10312 ResourceManager
9897 DataNode
10781 Jps
$cd /home/hadoop/flume-1.9.0/conf
# flume-env.sh.templat 복사
$cp flume-env.sh.template flume-env.sh
$vi flume-env.sh
export JAVA_HOME=$JAVA_HOME
# log 쌓이는 위치 확인
$cd /var/log
$ll sec*
$tail secure
$chmod o+r secure
# java 위치 : /usr/lib/jvm
$cd /etc/
# backup 만들기
$cp profile profile~
# 설정
$vi profile
$export JAVA_HOME=/usr/lib/jvm/jre-1.8.0/
$source profile
$flume-ng 에이전트 -n $agent_name -c conf -f conf/flume-conf.properties.template
$flume-ng agent -n 이름 -c 위치 -f 파일명
$flume-ng agent -n a00 -c conf -f conf/flume-conf-00.properties
$cp flume-conf.properties.template flume-conf-00.properties
$vi flume-conf-00.properties
a00.sources = s00
a00.channels = c00
a00.sinks = k00 k01 k02
a00.sources.s00.type = exec
a00.sources.s00.command = tail -F /var/log/secure
a00.sources.s00.channels = c00
a00.sources.s00.interceptors=i00
a00.sources.s00.interceptors.i00.type=timestamp
a00.sinks.k00.type=hdfs
a00.sinks.k00.channel=c00
a00.sinks.k00.hdfs.path=hdfs://127.0.0.1:9000/user/flume/events/%y-%m-%d/%H%M/
a00.sinks.k00.writeFormat=Text
a00.channels.c00.type = memory
a00.channels.c00.capacity = 1000
a00.channels.c00.transactionCapacity = 100
$flume-ng agent --conf $FLUME_HOME/conf/ --conf-file $FLUME_HOME/conf/flume-conf-00.properties --name a00 -Dflume.root.logger=INFO,console
$hdfs dfs -ls /
$hdfs dfs -ls /user
# ssh 접근 시도로 log 쌓이는 중
# 1st
$while [ 1 -le 2 ] ; do ssh hadoop00 hostname; done
# 2nd
$x=0; while [ $x –le 100 ]; do ssh hadoop00 hostname; x=$((x+1)) ; done
# 해당 경로에 log 쌓이는 중
$hdfs dfs -ls /user/flume/events/22-02-08
Found 2 items
drwxr-xr-x - hadoop supergroup 0 2022-02-08 03:40 /user/flume/events/22-02-08/0339
drwxr-xr-x - hadoop supergroup 0 2022-02-08 03:41 /user/flume/events/22-02-08/0340
# guava 위치 확인
$cd /home/hadoop/flume-1.9.0/lib
$ll gua*
$-rw-rw-r-- 1 hadoop hadoop 1648200 Feb 8 01:24 guava-11.0.2.jar
# hive에 있던 guava 옮김
$cd /home/hadoop/hive-3.1.2/lib
$ll guava*
-rw-r--r-- 1 hadoop hadoop 2747878 Jan 28 03:30 guava-27.0-jre.jar
# 1. hive 서버 위치 -> flume 서버 복사
$cp /home/hadoop/hive-3.1.2/lib/guava-27.0-jre.jar /home/hadoop/flume-1.9.0/lib/
# 2. hadoop 서버 위치 -> flume 서버 복사
$cp /home/hadoop/hadoop-3.2.2/share/hadoop/hdfs/lib/guava-27.0-jre.jar /home/hadoop/flume-1.9.0/lib/
# woodstox 복사
$cp HADOOP_HOME/share/hadoop/common/lib/woodstox-core-5.0.3.jar $FLUME_HOME/lib
딱 필요한 자료였어요! 감사합니다 :)