logstash 相关配置

input {
 kafka {
  bootstrap_servers => "192.168.10.10:9092,192.168.10.20:9092,192.168.10.30:9992"
  topics => ["jar1_logs"]
  codec => json {
     charset => "UTF-8"
  }
  add_field => { "[@metadata][mytopic]" => "jar1_logs"}
 }
 kafka {
 bootstrap_servers => "192.168.10.10:9092,192.168.10.20:9092,192.168.10.30:9992"
  topics => ["jar2_logs"]
  codec => json {
     charset => "UTF-8"
  }
  add_field => { "[@metadata][mytopic]" => "jar2_logs"}
 }
 }
filter {
  mutate {
    gsub => ["message", "\\x", "\\\x"]
  }
  if [@metadata][mytopic] == "jar1_logs" or [@metadata][mytopic] == "jar2_logs" {
    grok{
      match => ["message","\[%{TIMESTAMP_ISO8601:timestamp}\.[0-9]*\]%{SPACE}\[%{GREEDYDATA:thread}\]%{SPACE}\[%{LOGLEVEL:error_level}.*\]%{SPACE}%{JAVACLASS:method}(\.\[\.\[localhost\]\.\[\/\]\.\[dispatcherServlet\])*%{SPACE}\-%{SPACE}%{JAVALOGMESSAGE:mesg}"]
    }
  }else if [@metadata][mytopic] == "jar3_logs" or [@metadata][mytopic] == "jar4_logs" {
    grok{
      match => ["message","%{TIMESTAMP_ISO8601:timestamp}\.[0-9]*\-*\[%{GREEDYDATA:thread}\]%{SPACE}\[%{LOGLEVEL:error_level}\]%{SPACE}%{JAVAMETHOD:class_name}%{SPACE}%{JAVALOGMESSAGE:mesg}"]
    }
  }else if [@metadata][mylogtype] == "rpc_logs"{
    grok{
      match => ["message","%{TIMESTAMP_ISO8601:timestamp}\.[0-9]*(\[%{GREEDYDATA:thread}\])*\-\[%{LOGLEVEL:error_level}\]%{SPACE}(trace\:%{NUMBER:trade_id}\|)*%{JAVALOGMESSAGE:mesg}"]
    }
  }else {
    grok{
      match => ["message","%{TIMESTAMP_ISO8601:timestamp}\.[0-9]*\[%{GREEDYDATA:thread}\]\-\[%{LOGLEVEL:error_level}\]%{SPACE}%{JAVACLASS:method}%{SPACE}%{JAVALOGMESSAGE:mesg}"]
    }
  }
 date {
   match => ["timestamp", "dd/MM/yyyy:HH:mm:ss Z"]
   target => "@timestamp"
 }
 mutate {
       remove_field => ["message"]
       remove_field  =>  ["tags"]
       remove_field  =>  ["fields"]
       remove_field => ["thread"]
 }
}
output {
 #stdout {
 #  codec=>rubydebug
 #}
 if [@metadata][mytopic] == "jar1_logs" {
   elasticsearch {
    hosts => ["192.168.10.2:9200","192.168.10.3:9200","192.168.10.4:9200","192.168.10.5:9200","192.168.10.6:9200"]
    index => "jar1_logs-%{+YYYY-MM}"
   }
 } else if [@metadata][mytopic] == "jar2_logs" {
   elasticsearch {
   hosts => ["192.168.10.2:9200","192.168.10.3:9200","192.168.10.4:9200","192.168.10.5:9200","192.168.10.6:9200"]
   index => "jar1_logs-%{+YYYY-MM}"
  }
 }
}
最后修改日期: 2021年4月19日

作者

留言

撰写回覆或留言

发布留言必须填写的电子邮件地址不会公开。