Opensearch - logstath config

logstash access.log ingest

• 아래 포멧의 access.log를 ingest 하는 fiter

  118.235.12.100 - - [10/Mar/2025:17:23:48 +0900] "POST /login.do HTTP/1.1" 200 52195 76024

input {
  beats {
    port => 5044
  }
}

filter {
  if [source_from] == "access_log_tomcat_ssl" {
    grok {
      match => {
        "message" => "%{IPORHOST:[my_clientip]} - - \[%{HTTPDATE:[my_timestamp]}\] \"%{WORD:[my_method]} %{DATA:[my_request_url]} HTTP/%{NUMBER:[my_http_version]}\" %{NUMBER:[my_http_code]} (?:%{NUMBER:[my_recv_bytes]}|-) %{NUMBER:[my_duration]}"
      }
    }
    mutate {
      remove_field => ["@version", "agent", "input", "log", "ecs", "tags", "host", "event", "message" ]
      convert => {
        "my_recv_bytes" => "integer"
        "my_duration" => "integer"
      }
    }
    date {
      match => [ "my_timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
      target => "@timestamp"
      timezone => "Asia/Seoul"
    }
    ruby {
      code => "
        tmp_timestamp = event.get('my_timestamp')[0,11]
        tmp_parts = tmp_timestamp.split('/')
        tmp_month = case tmp_parts[1]
             when 'Jan' then '01' when 'Feb' then '02' when 'Mar' then '03'
             when 'Apr' then '04' when 'May' then '05' when 'Jun' then '06'
             when 'Jul' then '07' when 'Aug' then '08' when 'Sep' then '09'
             when 'Oct' then '10' when 'Nov' then '11' when 'Dec' then '12'
             else '00'
             end
        tmp_year_month = tmp_parts[2] + tmp_month
        event.set('my_month', tmp_year_month)
      "
    }
  }
}

output {
  stdout { codec => rubydebug }

  if [source_from] == "access_log_tomcat_ssl" {
    opensearch {
      ssl_certificate_verification => false
      ssl      => false
      hosts    => ["https://opensearch-node1:9200"]
      user     => "admin"
      password => "admin"
      index    => "access-log-tomcat-%{[my_month]}"
    }
  }
}

filebeat.yml

- type: log

  # Unique ID among all inputs, an ID is required.
  id: my-tomcat-ssl-accesslog

  # Change to true to enable this input configuration.
  enabled: true

  # Paths that should be crawled and fetched. Glob based paths.
  paths:
    - /etc/httpd/logs/ssl_access_log*
    - /etc/httpd/logs/access_log*

  fields:
    source_from: access_log_tomcat_ssl
  fields_under_root: true


output.logstash:
  # The Logstash hosts
  hosts: ["localhost:35044"]