Elasticsearch 분산환경 설정들

Elasticsearch 분산환경을 만들기 위한 설정파일 예시이다.

보안관련은 Elasticsearch 보안 시작하기를 참고

Elasticsearch

상세 옵션 설명은
https://esbook.kimjmin.net/02-install/2.3-elasticsearch/2.3.2-elasticsearch.yml 참조

Master node

# ======================== Elasticsearch Configuration =========================
#
cluster.name: "test_cluster"
node.name: "test_node-master"
network.host: "0.0.0.0"
http.port: 9200
transport.tcp.port: 9300
discovery.seed_hosts: ["123.456.789.0"]
cluster.initial_master_nodes: ["test_node-master"]
node.master: true
node.data: false
node.ingest: false
http.cors.enabled: false

# ----------------------------------- Paths ------------------------------------
# Path to directory where to store the data (separate multiple locations by comma):
#
path.data: /data/elasticsearch
#
# Path to log files:
#
path.logs: /log/elasticsearch

indices.breaker.total.limit: "90%"

#xpack.security.enabled: true
#xpack.security.transport.ssl.enabled: true
#xpack.security.transport.ssl.verification_mode: certificate
#xpack.security.transport.ssl.keystore.path: elastic-certificates.p12
#xpack.security.transport.ssl.truststore.path: elastic-certificates.p12

Kibana node

# ======================== Elasticsearch Configuration =========================
#
cluster.name: "test_cluster"
node.name: "test_node-kibana"
network.host: "0.0.0.0"
http.port: 9200
transport.tcp.port: 9300
discovery.seed_hosts: ["123.456.789.0"]
cluster.initial_master_nodes: ["test_node-master"]
node.master: false
node.voting_only: false
node.data: false
node.ingest: false
node.ml: false
xpack.ml.enabled: false
# ----------------------------------- Paths ------------------------------------
# Path to directory where to store the data (separate multiple locations by comma):
#
path.data: /data/elasticsearch
#
# Path to log files:
#
path.logs: /log/elasticsearch

indices.breaker.total.limit: "90%"

xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: elastic-certificates.p12

#PASSWORD apm_system = SWYAQusXf101XX17sELG
#PASSWORD kibana = zEUiw190Xby97T14ylx2
#PASSWORD logstash_system = o50MjuMgwKHi8ZtFnqSW
#PASSWORD beats_system = EpOva5z2Kar5AjvfNjtV
#PASSWORD remote_monitoring_user = kQVmIfSCz1K4pdigtgtX
#PASSWORD elastic = elastic1!

Data node 1,2,3

# ======================== Elasticsearch Configuration =========================
#
cluster.name: "test_cluster"
node.name: "test_node-data1 or test_node-data2 or test_node-data3"
network.host: "0.0.0.0"
http.port: 9200
transport.tcp.port: 9300
discovery.seed_hosts: ["123.456.789.0"]
cluster.initial_master_nodes: ["test_node-master"]
node.master: false
node.data: true
# ----------------------------------- Paths ------------------------------------
# Path to directory where to store the data (separate multiple locations by comma):
#
path.data: /data/elasticsearch
#
# Path to log files:
#
path.logs: /log/elasticsearch

indices.breaker.total.limit: "90%"

xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: elastic-certificates.p12

#PASSWORD apm_system = SWYAQusXf101XX17sELG
#PASSWORD kibana = zEUiw190Xby97T14ylx2
#PASSWORD logstash_system = o50MjuMgwKHi8ZtFnqSW
#PASSWORD beats_system = EpOva5z2Kar5AjvfNjtV
#PASSWORD remote_monitoring_user = kQVmIfSCz1K4pdigtgtX
#PASSWORD elastic = elastic1!

Coordinate node

# ======================== Elasticsearch Configuration =========================
#
cluster.name: "test_cluster"
node.name: "test_node_${HOSTNAME}-coordinating"
network.host: "0.0.0.0"
http.port: 9200
transport.tcp.port: 9300
discovery.seed_hosts: ["123.456.789.0"]
cluster.initial_master_nodes: ["test_node-master"]
node.master: false
node.voting_only: false
node.data: false
node.ingest: false
node.ml: false
xpack.ml.enabled: false
# ----------------------------------- Paths ------------------------------------
# Path to directory where to store the data (separate multiple locations by comma):
#
path.data: /data/elasticsearch
#
# Path to log files:
#
path.logs: /log/elasticsearch

xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: elastic-certificates.p12

Kibana

Kibana

# Kibana is served by a back end server. This setting specifies the port to use.
server.host: "0.0.0.0"
 
elasticsearch.username: "kibana"
elasticsearch.password: ""
 
# Enables you specify a file where Kibana stores log output.
logging.dest: /log/kibana/kibana.log
 
# Set the value of this setting to true to suppress all logging output.
logging.silent: false
 
# Set the value of this setting to true to suppress all logging output other than error messages.
logging.quiet: false
 
# Set the value of this setting to true to log all events, including system usage information
# and all requests.
logging.verbose: false

Logstash

Logstash

input {
  kafka {
    bootstrap_servers => "123.456.789.0"
    topics => ["api_log"]
    group_id => "logstash"
    type => "api"
    consumer_threads => 1
  }
  kafka {
    bootstrap_servers => "123.456.789.0"
    topics => ["admin_log"]
    group_id => "logstash"
    type => "admin"
    consumer_threads => 1
  }
  kafka {
    bootstrap_servers => "123.456.789.0"
    topics => ["web_log"]
    group_id => "logstash"
    type => "web"
    consumer_threads => 1
  }
}
filter {
  json {
    source => "message"
  }
  mutate {
    add_field => {
       "ls_timestamp" => "%{@timestamp}"
       "kst_timestamp" => ""
    }
    remove_field => ["message"]
  }
  ruby {
    code => "event.set('kst_timestamp', event.get('@timestamp').time.localtime('+09:00').strftime('%Y-%m-%d %H:%M:%S'))"
  }
  grok {
    match => {
      "kst_timestamp" => "%{YEAR:yyyy}-%{MONTHNUM:mm}-%{MONTHDAY:dd}%{GREEDYDATA}"
    }
    add_field => {
      "[@metadata][yymmdd]" => "%{yyyy}.%{mm}.%{dd}"
    }
    remove_field => [ "yyyy", "mm", "dd", "ls_timestamp" ]
  }
}
 
output {
 if [type] == "api" {
    elasticsearch {
    hosts => ["localhost:9200"]
    ilm_rollover_alias => "api-log"
    ilm_pattern => "{now/d{yyyy.MM.dd|+09:00}}-000001"
    ilm_policy => "api-log-ilm-policy"
    user => "elastic"
    password => "elastic1!"
   }
 }
 if [type] == "admin" {
    elasticsearch {
    hosts => ["localhost:9200"]
    ilm_rollover_alias => "admin-log"
    ilm_pattern => "{now/d{yyyy.MM.dd|+09:00}}-000001"
    ilm_policy => "admin-log-ilm-policy"
    user => "elastic"
    password => "elastic1!"
   }
 }
 if [type] == "web" {
    elasticsearch {
    hosts => ["localhost:9200"]
    ilm_rollover_alias => "web-log"
    ilm_pattern => "{now/d{yyyy.MM.dd|+09:00}}-000001"
    ilm_policy => "web-log-ilm-policy"
    user => "elastic"
    password => "elastic1!"
   }
 }
  stdout { codec => rubydebug }
}