CSV 살펴보기
sample.csv 파일 준비
- IDX
- EQPID
- GROUPID
- TIMESTAMP
- 11661
- 11662
- 11665
- 11666
- ....
인덱스 생성
프라이머리 샤드 개수 설정하여 인덱스 생성하기
curl -XPUT "http://localhost:9200/sample_index?pretty" -H 'Content-Type: application/json' -d'
{
"settings": {
"number_of_shards": 5,
"number_of_replicas": 1
}
}'인덱스 생성 확인
curl -X GET "localhost:9200/_cat/indices?pretty"
샤드 개수 설정 확인하기
curl -X GET "localhost:9200/sample_index/_settings?pretty"
정적 매핑을 통해 생성된 매핑 정보 확인
curl -X GET "localhost:9200/sample_index/_mapping?pretty"logstash-sample.conf 작성
input {
stdin {}
file {
path => "/home/dev/elastic/sample.csv"
start_position => "beginning"
}
}
filter {
csv {
separator => ","
columns => ["IDX","EQPID","GROUPID","TIMESTAMP","27","29","32","33","34","124","125","126","127","128","129","11527","11528","11529","11530","11547","11548","11549","11550","11567","11568","11569","11570","901","921","922","923","924","925","926","927","928","929","930","931","932","933","934","935","936","937","938","939","940","941","942","943","944","945","946","947","948","949","950","951","952","953","954","955","956","957","958","959","960","961","962","963","964","965","966","967","968","969","970","971","972","973","974","975","976","977","978","979","980","981","982","983","984","985","986","987","988","989","990","991","992","993","1531","1532","1533","1534","1535","1692","1693","1694","1695","1696","11661","11662","11665","11666","11667","11668","11669","11670","11687","11688","11689","11690","11763","11764","11783","11784","11801","11802","11803","11804","11805","11806","11807","11808","11809","11810","11812","11813","11827","11901","11902","11903","11904","11905","11906","11907","11908","11909","11910","11967","11968","11969","11970","12087","12088","12089","12090","12107","12108","12109","12110","12121","12122","12123","12124","12125","12126","12127","12128","12129","12130","12147","12148","12149","12150","12167","12168","12169","12170","12181","12182","12183","12184","12185","12186","12187","12188","12189","12190","12227","12228","12229","12230","12241","12242","12243","12244","12245","12246","12247","12248","12249","12250","12281","12282","12283","12284","12285","12286","12287","12288","12289","12290","12323","12324","12401","12402","12403","12404","12405","12406","12407","12408","12409","12410","12421","12422","12423","12424","12425","12426","12427","12428","12429","12430","12441","12442","12443","12444","12445","12446","12447","12448","12449","12450","12452","12453","12463","12464","12465","12466","12467","12468","12469","12470","12472","12473","12483","12484","12487","12488","12489","12490","12503","12504","12507","12508","12509","12510","12523","12524","12527","12528","12529","12530","12543","12544","12547","12548","12549","12550","12607","12608","12609","12610","12627","12628","12629","12630","12647","12648","12649","12650","12670","12690","12707","12710","12724","12727","12728","12729","12730","14427","14428","14429","14510","14567","14587","14588","14589","14590","14807","14808","14809","14810","14847","14848","14849","14850","14867","14868","14869","14870","14901","14902","14903","14904","14905","14906","14907","14908","14909","14910","14911","17101","17102","17151","17152","17301","17302","17351","17352","17401","17402","17451","17452","19067","19068","19069","19070","19081","19082","19084","19085","19086","19087","19088","19089","19090","20145","20146","20147","20148","20149","20150","20152","20153","20165","20166","20167","20168","20169","20170","20172","20173","20183","20184","20187","20188","20189","20190","20203","20207","20223","20227","20247","20248","20249","20250","20267","20268","20269","20270","20347","20348","20349","20350","20367","20368","20369","20370","20387","20388","20389","20390","20403","20404","20407","20408","20409","20410","20427","20428","20429","20430","20447","20448","20449","20467","20487","20488","20489","20490","23587","23588","23589","23590","23647","23648","23649","23650","23667","23668","23669","23670","25987","25988","25989","25990","26981","26982","26983","26984","26985","26986","26987","26988","26989","26990","29587","29588","29589","29590","29727","29747","29767","29785","29786","29787","29788","29789","29790","29807","29808","29809","29810","29827","29828","29829","29847","29848","29849","29867","29868","29869","29870","29887","29967","29968","29969","29970","29987","29988","29989","29990","32075","32956","2300500100","2300700101","2300700102","2300700201","2300700202","2300730100","2300730200","2300730300","2300730400","2300740001","2300740002","2300740003","2300740004","2300740005","2300740006","2300740007","2300740008","2300750000","2300760000","2300770000","2300780000","2300790000","2300800000","2300810000","2300820000","2300830000","2300840000","2300850000"]
skip_empty_columns => true
skip_header => true
}
date {
match => [ "TIMESTAMP", "yyyy/MM/dd-HH:mm:ss.SSS"]
}
mutate {
convert => {
"EQPID" => "string"
"11661" => "integer"
"11662" => "integer"
"11665" => "integer"
"11666" => "integer"
"11667" => "integer"
"11668" => "integer"
"11669" => "integer"
"11670" => "integer"
"20145" => "integer"
"20146" => "integer"
"20147" => "integer"
"20148" => "integer"
"20149" => "integer"
"20150" => "integer"
"20152" => "integer"
"20153" => "integer"
"20165" => "integer"
"20166" => "integer"
"20167" => "integer"
"20168" => "integer"
"20169" => "integer"
"20170" => "integer"
"20172" => "integer"
"20173" => "integer"
"127" => "integer"
"128" => "integer"
"129" => "integer"
"11801" => "integer"
"11802" => "integer"
"11803" => "integer"
"11804" => "integer"
"11805" => "integer"
"11806" => "integer"
"11807" => "integer"
"11808" => "integer"
"11809" => "integer"
"11810" => "integer"
"11812" => "integer"
"11813" => "integer"
"11827" => "integer"
"11901" => "integer"
"11902" => "integer"
"11903" => "integer"
"11904" => "integer"
"11905" => "integer"
"11906" => "integer"
"11907" => "integer"
"11908" => "integer"
"11909" => "integer"
"11910" => "integer"
"12441" => "integer"
"12442" => "integer"
"12443" => "integer"
"12444" => "integer"
"12445" => "integer"
"12446" => "integer"
"12447" => "integer"
"12448" => "integer"
"12449" => "integer"
"12450" => "integer"
"12452" => "integer"
"12453" => "integer"
"12463" => "integer"
"12464" => "integer"
"12465" => "integer"
"12466" => "integer"
"12467" => "integer"
"12468" => "integer"
"12469" => "integer"
"12470" => "integer"
"12472" => "integer"
"12473" => "integer"
"12707" => "integer"
"12710" => "integer"
"11547" => "integer"
"11548" => "integer"
"11549" => "integer"
"11550" => "integer"
"11567" => "integer"
"11568" => "integer"
"11569" => "integer"
"11570" => "integer"
"12483" => "integer"
"12484" => "integer"
"12487" => "integer"
"12488" => "integer"
"12489" => "integer"
"12490" => "integer"
"12503" => "integer"
"12504" => "integer"
"12507" => "integer"
"12508" => "integer"
"12509" => "integer"
"12510" => "integer"
"12523" => "integer"
"12524" => "integer"
"12527" => "integer"
"12528" => "integer"
"12529" => "integer"
"12530" => "integer"
"12543" => "integer"
"12544" => "integer"
"12547" => "integer"
"12548" => "integer"
"12549" => "integer"
"12550" => "integer"
"12607" => "integer"
"12608" => "integer"
"12609" => "integer"
"12610" => "integer"
"12627" => "integer"
"12628" => "integer"
"12629" => "integer"
"12630" => "integer"
"12647" => "integer"
"12648" => "integer"
"12649" => "integer"
"12650" => "integer"
"12670" => "integer"
"12690" => "integer"
"14510" => "integer"
"14567" => "integer"
"14587" => "integer"
"14588" => "integer"
"14589" => "integer"
"14590" => "integer"
"20347" => "integer"
"20348" => "integer"
"20349" => "integer"
"20350" => "integer"
"20367" => "integer"
"20368" => "integer"
"20369" => "integer"
"20370" => "integer"
"20387" => "integer"
"20388" => "integer"
"20389" => "integer"
"20390" => "integer"
"20403" => "integer"
"20404" => "integer"
"20407" => "integer"
"20408" => "integer"
"20409" => "integer"
"20410" => "integer"
"20427" => "integer"
"20428" => "integer"
"20429" => "integer"
"20430" => "integer"
"20447" => "integer"
"20448" => "integer"
"20449" => "integer"
"29827" => "integer"
"29828" => "integer"
"29829" => "integer"
"29847" => "integer"
"29848" => "integer"
"29849" => "integer"
"29867" => "integer"
"29868" => "integer"
"29869" => "integer"
"29870" => "integer"
"17101" => "integer"
"17102" => "integer"
"17151" => "integer"
"17152" => "integer"
"17301" => "integer"
"17302" => "integer"
"17351" => "integer"
"17352" => "integer"
"17401" => "integer"
"17402" => "integer"
"17451" => "integer"
"17452" => "integer"
"12401" => "string"
"12402" => "string"
"12403" => "string"
"12404" => "string"
"12405" => "string"
"12406" => "string"
"12407" => "string"
"12408" => "string"
"12409" => "string"
"12410" => "string"
"12121" => "string"
"12122" => "string"
"12123" => "string"
"12124" => "string"
"12125" => "string"
"12126" => "string"
"12127" => "string"
"12128" => "string"
"12129" => "string"
"12130" => "string"
"11967" => "integer"
"11968" => "integer"
"11969" => "integer"
"11970" => "integer"
"12724" => "integer"
"12727" => "integer"
"12728" => "integer"
"12729" => "integer"
"12730" => "integer"
"23647" => "integer"
"23648" => "integer"
"23649" => "integer"
"23650" => "integer"
"23667" => "integer"
"23668" => "integer"
"23669" => "integer"
"23670" => "integer"
"29967" => "integer"
"29968" => "integer"
"29969" => "integer"
"29970" => "integer"
"29987" => "integer"
"29988" => "integer"
"29989" => "integer"
"29990" => "integer"
"12181" => "integer"
"12182" => "integer"
"12183" => "integer"
"12184" => "integer"
"12185" => "integer"
"12186" => "integer"
"12187" => "integer"
"12188" => "integer"
"12189" => "integer"
"12190" => "integer"
}
remove_field => ["message","27","29","32","33","34","124","125","126","901","921","922","923","924","925","926","927","928","929","930","931","932","933","934","935","936","937","938","939","940","941","942","943","944","945","946","947","948","949","950","951","952","953","954","955","956","957","958","959","960","961","962","963","964","965","966","967","968","969","970","971","972","973","974","975","976","977","978","979","980","981","982","983","984","985","986","987","988","989","990","991","992","993","1531","1532","1533","1534","1535","1692","1693","1694","1695","1696","11527","11528","11529","11530","11662","11687","11688","11689","11690","11763","11764","11783","11784","12087","12088","12089","12090","12107","12108","12109","12110","12147","12148","12149","12150","12167","12168","12169","12170","12227","12228","12229","12230","12241","12242","12243","12244","12245","12246","12247","12248","12249","12250","12281","12282","12283","12284","12285","12286","12287","12288","12289","12290","12323","12324","12421","12422","12423","12424","12425","12426","12427","12428","12429","12430","14427","14428","14429","14807","14808","14809","14810","14847","14848","14849","14850","14867","14868","14869","14870","14901","14902","14903","14904","14905","14906","14907","14908","14909","14910","14911","19067","19068","19069","19070","19081","19082","19084","19085","19086","19087","19088","19089","19090","20183","20184","20187","20188","20189","20190","20203","20207","20223","20227","20247","20248","20249","20250","20267","20268","20269","20270","20467","20487","20488","20489","20490","23587","23588","23589","23590","25987","25988","25989","25990","26981","26982","26983","26984","26985","26986","26987","26988","26989","26990","29587","29588","29589","29590","29727","29747","29767","29785","29786","29787","29788","29789","29790","29807","29808","29809","29810","29887","32075","32956","2300500100","2300700101","2300700102","2300700201","2300700202","2300730100","2300730200","2300730300","2300730400","2300740001","2300740002","2300740003","2300740004","2300740005","2300740006","2300740007","2300740008","2300750000","2300760000","2300770000","2300780000","2300790000","2300800000","2300810000","2300820000","2300830000","2300840000","2300850000"]
}
}
output {
elasticsearch {
hosts => "127.0.0.1"
index => "sample_index"
}
file {
path => "/home/dev/elastic/tmp/my_logstash.log"
}
}input.file.path는 절대경로여야 한다.
Pipeline error {:pipeline_id=>"main", :exception=>#<ArgumentError: File paths must be absolute, relative path specified: sample.csv>
참고 URL
- filter-date 형식 참고 URL
https://www.elastic.co/guide/en/logstash/current/plugins-filters-date.html
Logstash 실행
Logstash 실행하기
./logstash-7.15.1/bin/logstash -f logstash-sample.conf Head를 이용해서 모니터링
git 명령을 이용한 Head 소스 코드 다운로드
git clone https://github.com/mobz/elasticsearch-head패키지 설치
cd elasticsearch-head
npm installHead 실행하기
npm startHead에 접속하기
http://localhost:9100/
로그 조회하기
curl -X GET "localhost:9200/sample_index/_search?pretty"{
"took" : 1,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 10000,
"relation" : "gte"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : "sample_index",
"_type" : "_doc",
"_id" : "Kjvz-XwBfQKpo882v2o-",
"_score" : 1.0,
"_source" : {}
}]
}
}참고 자료
- 박상헌, 강진우, 기초부터 다지는 ElasticSearch 운영 노하우: 기본 개념부터 클러스터 구축, 실무 활용 시나리오까지(인사이트, 2021)
작심삼일 무한 루프 ♾️