官方helm 

https://helm.elastic.co

 

通过kafka消息队列获取日志

 

---
replicas: 2
extraEnvs:
  - name: TZ 
    value: Asia/Shanghai
logstashConfig:
  patterns.conf: |
    JAVA_TIME 20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:?%{MINUTE}(?::?%{SECOND})
    COMMONAPACHELOGS %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:local_time}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
    COMBINEDAPACHELOGS %{COMMONAPACHELOGS} %{QS:referrer} %{QS:agent}
  logstash.yml: |
    http.host: 0.0.0.0
    xpack.monitoring.enabled: true
    xpack.monitoring.elasticsearch.username: logstash_system
    xpack.monitoring.elasticsearch.password: ****
    xpack.monitoring.elasticsearch.hosts: ["elasticsearch-master.elasticsearch.svc.cluster.local:9200"]
logstashPipeline:
  logstash.conf: |
    input{
        kafka{
            bootstrap_servers => ["***.***.***.***:9092,***.***.***.***:9092,***.***.***.***:9092"]
            topics_pattern => "nginx_.*"
            client_id => "${HOSTNAME}-nginx"
            type => "nginx"
        }
    }
     
    input{
        kafka{
            bootstrap_servers => ["***.***.***.***:9092,***.***.***.***:9092,***.***.***.***:9092"]
            topics_pattern => "java_.*"
            client_id => "${HOSTNAME}-java"
            type => "java"
        }
    }
     
    filter {
        if [type] == "java" {
            json {
                source => "message"
                remove_field => [ "beat","stream","offset","source","prospector" ]
            }
            grok {
                patterns_dir => "/usr/share/logstash/config/patterns.conf"
                match => { "message" => "(%{JAVA_TIME:local_time} \[%{JAVALOGMESSAGE:thread}\] %{LOGLEVEL:level} %{JAVALOGMESSAGE:java_class} - %{JAVALOGMESSAGE}|%{LOGLEVEL:level} %{JAVA_TIME:local_time} %{JAVALOGMESSAGE:thread} %{JAVALOGMESSAGE:java_class} : %{JAVALOGMESSAGE})" }
            }
        }
 
        if [type] == "nginx" {
            json {
                source => "message"
                remove_field => [ "beat","stream","offset","source","prospector" ]
            }
            grok {
                patterns_dir => "/usr/share/logstash/config/patterns.conf"
                match => { "message" => "%{COMBINEDAPACHELOGS}" }
            }
        }
         
        date {
            locale => "cn"
            match => [ "local_time", "ISO8601", "dd/MMM/yyyy:HH:mm:ss Z" ]
            remove_field => [ "local_time" ]
        }
    }
     
    output {
        elasticsearch {
            hosts => ["elasticsearch-master.elasticsearch.svc.cluster.local:9200"]
            index => "%{index}-%{topic}-%{+YYYY-MM-dd}"
            user => elastic
            password => ***
        }
    }
 
volumeClaimTemplate:
  accessModes: [ "ReadWriteOnce" ]
  #storageClassName: "cloud-ssd"
  resources:
    requests:
      storage: 20Gi
       
logstashJavaOpts: "-Xmx2g -Xms2g"
 
resources:
  requests:
    cpu: "1000m"
    memory: "1536Mi"
  limits:
    cpu: "8000m"
    memory: "2536Mi"

 

---