ELK ์Šคํƒ

์žฅ์—ฌ์ง„ยท2022๋…„ 4์›” 24์ผ
0

Elastic Search-Logstash-Kibana


  • Docker๊ด€๋ จ ์„ค์ • docker-compose.yamlํŒŒ์ผ
version: '3.7'

services:
  my-backend:
    build:
      context: .
      dockerfile: Dockerfile
    volumes:
      - ./src:/myfolder/src
    ports:
      - 3000:3000

  my-database:
    image: mysql:latest
    environment:
      MYSQL_DATABASE: 'mainproject'
      MYSQL_ROOT_PASSWORD: 'root'
    ports:
      - 3306:3306

  my-redis:
    image: redis:latest
    ports:
      - 6379:6379

  # elasticsearch-logstash-kibana๋Š” ํ•œ์Œ! 
  # ๊ฐ๊ฐ์˜ ์„ค์ •ํŒŒ์ผ์—์„œ ์„œ๋กœ์˜ ๊ธฐ๋ณธ ์ด๋ฆ„์œผ๋กœ ์—ฐ๊ฒฐ๋˜์–ด์žˆ๊ธฐ ๋•Œ๋ฌธ์— ์ด๋ฆ„์„ ๊ทธ๋Œ€๋กœ ์‚ฌ์šฉํ•˜๋Š” ๊ฒƒ์ด ์ข‹์Œ
  elasticsearch:
    image: elasticsearch:7.17.0
    environment:
      discovery.type: single-node # elasticsearch db์ด๊ธฐ ๋•Œ๋ฌธ์— ์—ฌ๋Ÿฌ๊ฐœ๋กœ ๋ถ„์‚ฐ๊ฐ€๋Šฅ! ์ง€๊ธˆ์€ test๋กœ ํ•œ ๊ฐœ๋กœ ์‚ฌ์šฉํ•  ๊ฒƒ์ด๊ธฐ๋•Œ๋ฌธ์— single-node์‚ฌ์šฉ
    ports:
      - 9200:9200

  logstash:
    image: logstash:7.17.0
    # docker์•ˆ์— logstash๊ด€๋ จ ํŒŒ์ผ ๋„ฃ๊ธฐ
    volumes:
      - ./elk/logstash/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
      - ./elk/logstash/mysql-connector-java-8.0.28.jar:/usr/share/logstash/mysql-connector-java-8.0.28.jar
      - ./elk/logstash/templates.json:/usr/share/logstash/templates.json

  # kibana:
  #   image: kibana:7.17.0  # ์šฉ๋Ÿ‰์ด ๊ต‰์žฅํžˆ ํผ
  • ํ…์ŠคํŠธ, ์ˆซ์ž, ์ •ํ˜• ๋ฐ ๋น„์ •ํ˜• ๋ฐ์ดํ„ฐ ๋“ฑ ๋ชจ๋“  ์œ ํ˜•์˜ ๋ฐ์ดํ„ฐ๋ฅผ ์œ„ํ•œย ๋ฌด๋ฃŒ ์˜คํ”ˆ ์†Œ์Šค ๊ฒ€์ƒ‰ ๋ฐ ๋ถ„์‚ฐ ์—”์ง„

  • ๋‹จ์ˆœํžˆ ๊ฒ€์ƒ‰ ์—”์ง„์„ ๋„˜์–ด ๋ณด์•ˆ, ๋กœ๊ทธ ๋ถ„์„, ์ „๋ฌธ๋ถ„์„ ๋“ฑ ๋‹ค์–‘ํ•œ ์˜์—ญ์—์„œ ์ค‘์š”ํ•œ ์—ญํ• 

  • Elastic Search-์ฐธ๊ณ  ์‚ฌ์ดํŠธ https://esbook.kimjmin.net

  • ์ธ๋ฑ์Šค(๋น ๋ฅธ๊ฒ€์ƒ‰)
    mySQL
    ex) "์ฒ ์ˆ˜"

  • ์—ญ์ธ๋ฑ์Šค(๋น ๋ฅธ ์ „๋ฌธ(full-text)๊ฒ€์ƒ‰)๋ฌธ์žฅ
    Elasticsearch
    ex) ๋ฌธ์žฅ์— ๋“ค์–ด๊ฐ„ "์ ์‹ฌ"

  • yarn add @nestjs/elasticsearch ์„ค์น˜
    [product.module.tsํŒŒ์ผ์— elasticsearch ์‚ฌ์šฉ์„ ์œ„ํ•œ ์˜์กด์„ฑ ์ฃผ์ž…!]

import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { Product } from './entities/product.entity';
import { ProductResolver } from './product.resolver';
import { ProductService } from './product.service';
import { ElasticsearchModule } from '@nestjs/elasticsearch';

@Module({
  imports: [
    TypeOrmModule.forFeature([
      Product, //
    ]),
    // elasticsearch ์‚ฌ์šฉ์„ ์œ„ํ•œ ์˜์กด์„ฑ ์ฃผ์ž…!
    ElasticsearchModule.register({
      node: 'http://elasticsearch:9200',
    }),
  ], //
  providers: [
    ProductResolver, //
    ProductService,
    ProductSubscriber,
  ],
}) //์˜์กด์„ฑ ์ฃผ์ž… ํ•„์š”
export class ProductModule {}
  • ElasticSearch๋ฅผ ์ƒ์„ฑ์ž ์ฃผ์ž…
  • QueryDSL์„ ์‚ฌ์šฉํ•ด ์ƒ์„ฑ, ์กฐํšŒ
    [product.resolver.ts์— elasticsearch์— ๋“ฑ๋กํ•˜๋Š” ์ฝ”๋“œ ์ถ”๊ฐ€]
@Mutation(() => Product)
  createProduct(
    @Args('createProductInput') createProductInput: CreateProductInput, //
  ) {
    // ์—˜๋ผ์Šคํ‹ฑ์„œ์น˜์—์„œ ๋“ฑ๋ก ์—ฐ์Šตํ•˜๊ธฐ!! => ์—ฐ์Šต! ์‹ค์ œ๋กœ๋Š” mysql์— ์ €์žฅํ•  ์˜ˆ์ •!
    this.elasticsearchService.create({
    	id: 'myid',
        index: 'myproduct', //table ์ด๋ฆ„
      	document: {
         name: '์ฒ ์ˆ˜',
         age: 14,
        school: '๋‹ค๋žŒ์ฅ์ดˆ๋“ฑํ•™๊ต',
       },
     });
  }

[product.resolver.ts์— elasticsearch์— ์กฐํšŒํ•˜๋Š” ์ฝ”๋“œ ์ถ”๊ฐ€]

@Query(() => [Product])
  async fetchProducts() {
    // ๋‹จ์ผ ์ปฌ๋Ÿผ์€ mysql , ๋ฌธ์žฅ์—์„œ ์ฐพ์œผ๋ฉด elasticsearsh์—์„œ ์กฐํšŒ!
    // ์—˜๋ผ์Šคํ‹ฑ์„œ์น˜์—์„œ ์กฐํšŒ ์—ฐ์Šตํ•˜๊ธฐ
    const result = await this.elasticsearchService.search({
      index: 'myproduct',
      query: {
        match_all: {},
      },
    });
    // ๊ฐ์ฒดํ˜•ํƒœ๋กœ ๋“ค์–ด์˜ค๊ธฐ๋•Œ๋ฌธ์— string์œผ๋กœ ๋ณ€ํ™˜ ํ•„์š”
    console.log(JSON.stringify(result, null, ' '));

2.logstash

  • mySQL์—์„œ ๋ฌธ์žฅ์šฉ ๋ฐ์ดํ„ฐ๋“ค์„ Elastic Search์— ์ €์žฅํ•ด์ฃผ๋Š” ์—ญํ• 
  • mySQL์— ์ ‘์†ํ•ด์„œ ๋ฐ์ดํ„ฐ๋ฅผ pollingํ•ด์„œ Elastic Search์— ์ €์žฅ
  • ์ด ๊ณผ์ •์„ ์œ„ํ•œ ์„ค์ • ํŒŒ์ผ ํ•„์š” logstash.confํŒŒ์ผ
    ์ž…๋ ฅ(Inputs) -> ํ•„ํ„ฐ(Filters) -> ์ถœ๋ ฅ(Outputs) ์œผ๋กœ ์ด๋ฃจ์–ด์ง
    ์ž…๋ ฅ - Beats, RDBMS ๋“ฑ ๋‹ค์–‘ํ•œ ๋ฐ์ดํ„ฐ ์ €์žฅ์†Œ๋กœ๋ถ€ํ„ฐ ๋ฐ์ดํ„ฐ๋ฅผ ์ž…๋ ฅ
		# ๋งค๋ฒˆ mysql์—์„œ pullling
        # ์ค‘๋ณต ์ €์žฅ์„ ํ”ผํ•  ์ˆ˜ ์žˆ๋„๋ก updatedat์ด ๊ฐ€์žฅ ๋งˆ์ง€๋ง‰์ธ ์‹œ๊ฐ„์„ ์ €์žฅ
input{
    jdbc{
        # ๋ฐ์ดํ„ฐ ๋ฒ ์ด์Šค์— ์ ‘์†ํ•˜๊ธฐ ์œ„ํ•œ ์ ‘์† ์ •๋ณด MySQL-connector-java
        jdbc_driver_library => "/usr/share/logstash/mysql-connector-java-8.0.28.jar"
        jdbc_driver_class => "com.mysql.cj.jdbc.Driver"
        jdbc_connection_string => "jdbc:mysql://my-database:3306/myprojectDocker" #๋‚˜์˜ db์ •๋ณด(mysql์‚ฌ์šฉ, docker์—์„œ db์ด๋ฆ„)
        jdbc_user => "root"
        jdbc_password => "root"
        schedule => "* * * * *" # ์‹คํ–‰ ์ฃผ๊ธฐ ๋ถ„/ ์‹œ๊ฐ„/ ์ผ/ ์›”/ ์š”์ผ  10 * * * * ๋งค 10๋ถ„๋งˆ๋‹ค(1์‹œ 10๋ถ„,,1์‹œ 20๋ถ„,,) * 2 * * * ๋งค์ผ 2์‹œ ๋งˆ๋‹ค  => ํฌ๋ก ํƒญ
        use_column_value => true # ์ปฌ๋Ÿผ ์‚ฌ์šฉํ• ๊ฒŒ 
        tracking_column => "updatedat" # ์–ด๋–ค ์ปฌ๋Ÿผ ์ถ”์ ํ• ๊ฑด๋ฐ?
        tracking_column_type => "numeric"  # ์ปฌ๋Ÿผ์˜ ํƒ€์ž…์ด ๋ญ”๋ฐ? # numeric = ์ˆซ์ž
        last_run_metadata_path => "./aaa.txt"  # ํŒŒ์ผ ์ƒ์„ฑ ํ›„ ํŒŒ์ผ ์•ˆ์— ๋งˆ์ง€๋ง‰ ๋ฐ์ดํ„ฐ ์ €์žฅ (updatedat์ปฌ๋Ÿผ์ด aaa.txt์— ์ €์žฅ ๋จ) # cat aaa.txt๋กœ ํ™•์ธ ๊ฐ€๋Šฅ
        statement => "select id, name, price, unix_timestamp(updatedat) as updatedat from product where unix_timestamp(updatedat) > :sql_last_value order by updatedat asc"  # ๋ช…๋ น์–ด(๋ชจ๋‘ ์†Œ๋ฌธ์ž๋กœ ์ž‘์„ฑ!)
        # unix_timestamp =>  ๋‚ ์งœ๋ฅผ ์ˆซ์ž๋กœ ๋ณ€๊ฒฝ ํ•ด์คŒ 
        # updatedat์€ aaa.txt์—๋„ ์ €์žฅ (๋‹ค์Œ ์ €์žฅ ๋•Œ ์ €์žฅ์‹œ๊ฐ„์„ ์‚ฌ์šฉํ•˜๊ธฐ์œ„ํ•ด)
        # updatedat์„ ๊ธฐ์ค€์œผ๋กœ ์ •๋ ฌ ํ•„์š”
    } 

}

output{
	# elasticsearch์— ์ €์žฅ!
    elasticsearch{
        hosts => "elasticsearch:9200"
        index => "myproduct" # index = ํ…Œ์ด๋ธ”์ด๋ฆ„
        template => 
    }
}

๐Ÿ’ก Elasticsearch๋ฅผ ์ด์šฉํ•˜์—ฌ ๊ตฌํ˜„ํ•  ๊ธฐ๋Šฅ์€ ๋‹จ์ˆœ๊ฒ€์ƒ‰๊ธฐ๋Šฅ์œผ๋กœ ๋ณต์žกํ•œ ๊ธฐ๋Šฅ์€ ์ฒ˜๋ฆฌํ•˜์ง€ ์•Š๊ธฐ ๋•Œ๋ฌธ์— Logstash๋ฅผ ํ™œ์šฉํ•ด ์ด๋ฏธ ์กด์žฌํ•˜๋Š” RDBMS์˜ Data๋ฅผ ์•Œ๋งž๊ฒŒ ๋ณ€ํ™˜ํ•˜์—ฌE ES์— ์ œ๊ณตํ•˜๋Š” ๊ฒƒ์ด ํ•ต์‹ฌ!

3.Kibana

API์š”์ฒญ์„ ํ•ด์•ผ ๋ฐ์ดํ„ฐ๋ฅผ ๋ณผ ์ˆ˜ ์žˆ์Œ ex) match_all:{}
Elastic Search์ „์šฉ dbeaver์—ญํ• !

  • ๊ฒ€์ƒ‰๊ณผ Aggregation์˜ ์ง‘๊ณ„ ๊ธฐ๋Šฅ์„ ์ด์šฉํ•ด Elasticsearch๋กœ ๋ถ€ํ„ฐ ๋ฌธ์„œ, ์ง‘๊ณ„ ๊ฒฐ๊ณผ ๋“ฑ์„ ๋ถˆ๋Ÿฌ์™€ ์›น ๋„๊ตฌ๋กœ ์‹œ๊ฐํ™” ๋ฐ ๋ชจ๋‹ˆํ„ฐ๋งย 
  • Discover, Visualize, Dashboardย 3๊ฐœ์˜ ๊ธฐ๋ณธ ๋ฉ”๋‰ด์™€ ๋‹ค์–‘ํ•œ App ๋“ค๋กœ ๊ตฌ์„ฑ๋˜์–ด ์žˆ๊ณ , ํ”Œ๋Ÿฌ๊ทธ์ธ์„ ํ†ตํ•ด App์˜ ์„ค์น˜๊ฐ€ ๊ฐ€๋Šฅ

0๊ฐœ์˜ ๋Œ“๊ธ€