logstash catalina - xiuyanduan/xiuyanduan.github.io GitHub Wiki

title: logstash demo conf
date: 2016-01-06
tags:
- logstash
- nginx
---

tested in Ubuntu 14.04 LTS,logstash 2.1.1,filebeat 1.0.1

catalina.out

encoder of catalina.out

<pattern>[%d{yyyy-MM-dd HH:mm:ss.SSS}] [%-5level] [%logger{36}] [%X{x-jjk-rqid:-notFound}] [%X{jjkUserId:-null}] - %msg%n</pattern>

conf of logstash

input {
    beats {
        port => 5044
        codec => multiline {
                pattern => "^\[%{TIMESTAMP_ISO8601}"
                negate => true
                what => "previous"
        }
    }
}
filter {
     if [type] == "catalina"{
                 grok {
                        match => { "message" => "\[%{TIMESTAMP_ISO8601:logtime}\] \[(?<level>.+?)\] \[(?<logger>.+?)\] \[(?<rqid>.+?)\] \[(?<jjkUserId>\w+)\] - (?<msg>.*)" }
                        #match => { "message" => "\[%{TIMESTAMP_ISO8601:logtime}\] \[(?<level>.+?)\] \[(?<logger>.+?)\] \[%{IP:clientip}-(?<rqid>\d+)\] \[(?<jjkUserId>\w+)\] - (?<javalogmessage>.*)" }
        }       
                 date {
                        match => ["logtime", "yyyy-MM-dd HH:mm:ss.SSS"]
        }
}
}
output {
    elasticsearch {
        hosts => "10.10.10.37:9200" 
    }

}

nginx access_log

log_format in nginx.conf

If the nginx could deal with POST require , such as gunicorn, tomcat . We can add $request_body to the log_format in nginx.conf

log_format newaccess '$remote_addr - $remote_user [$time_local] '
                    '"$request" $status $body_bytes_sent '
                    '"$http_referer" "$http_user_agent"  "$request_body"';

conf of logstash

input {
    file {
        path => "/somepath/access".log"
    }
}
filter {
    grok {
        match => { "message" => "%{COMBINEDAPACHELOG} %{QS:request_body}"}
    }

}
output {
    stdout {codec=>rubydebug}
}

apache access_log

input {
    file {
        path => "/tmp/apache.log"
        start_position => beginning 
    }
}
filter {
    grok {
       #匹配默认的apache日志格式
        match => { "message" => "%{COMBINEDAPACHELOG}"}
    }
    grok {   
	match => ["message", "%{HTTPDATE:logdate}"]
    }
    date {
	#date插件作用将日志内的时间设为timestamp
        match => ["logdate", "dd/MMM/yyyy:HH:mm:ss Z"]
    }
    ruby{
        #此插件作用将logdate转换为整数型UNIX时间戳格式
	code => "event['logdate']=event.sprintf('%{+%s}')"
    }
    mutate {
	#将logdate转换为整数型
	convert => ["logdate","integer"]

    }
    kv {
        #处理GET请求参数
	source => "request"
	field_split => "&?"
    }
    geoip {
        #geoip插件便于在elasticsearch中分析各地区访问情况
        source => "clientip"
    }


}
output {
    stdout {codec => rubydebug}
}
⚠️ **GitHub.com Fallback** ⚠️