Elasticsearch

List Indices of Elasticsearch

http://192.168.10.51:9200/_cat/indices

Delete Index

curl -XDELETE http://192.168.10.51:9200/nginx-access-2020.03.23

解决 Kibana Visulaize: Geo Coordinates => geo_point 问题

准备 template 檔 (nginx.json)

{
    "template": "nginx-access-*",
    "mappings": {
        "_default_": {
            "properties": {
                "geoip": {
                    "properties": {
                        "location": {
                            "type": "geo_point"
                        }
                    }
                }
            }
        }
    }
}

PUT Template to Elasticsearch

curl -XPUT -H'Content-Type:application/json' -d @nginx.json 'http://192.168.10.51:9200/_template/nginx-access?pretty=true'

Filebeat

log tracking folder (delete for retrieve again)

C:\ProgramData\filebeat

filebeat.yml sample

filebeat.prospectors:
- type: log
  enabled: true
  paths:
    - D:\Nginx\Logs\access.log
  fields:
    service: nginx
    host: DevApp
    category: nginx-access
  fields_under_root: true
- type: log
  enabled: true
  paths:
    - D:\Nginx\Logs\error.log
  fields:
    service: nginx
    host: DevApp
    category: nginx-error
  fields_under_root: true
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  path: ${path.config}/modules.d/*.yml
setup.template.settings:
  index.number_of_shards: 3
output.logstash:
  hosts: ["192.168.10.51:5044"]

Logstach

main.conf

input {
  beats {
    # The port to listen on for filebeat connections.
    port => 5044
  }
}

filter {
  if [category] == "nginx-access" {
    grok {
      match => ["message", "%{IP:remoteip} - %{DATA:remoteuser} \[%{HTTPDATE:timelocal}\] \"%{DATA:request}\" %{NUMBER:serverport} %{NUMBER:httpstatus} %{NUMBER:bytesent} \[%{DATA:contenttype}\] \[%{DATA:requestbody}\] \"%{DATA:httpreferer}\" \"%{DATA:useragent}\" \"%{DATA:xforwardedfor}\""]
      remove_field => "message"
    }

    date {
      match => [ "timelocal", "dd/MMM/YYYY:H:m:s Z" ]
      timezone => "Etc/UTC"
      remove_field => "timelocal"
    }

    geoip {
      source => "remoteip"
    }

    mutate {
      remove_field => [ "build", "beat", "input_type", "offset", "source", "type" ]
      remove_tag => "beats_input_codec_plain_applied"
    }
  } else if [category] == "nginx-error" {
    elasticsearch {
      hosts => ["192.168.10.51:9200"]
      index => "nginx-error-%{+yyyy.MM.dd}"
    }
  }
}
 
output {
  if [category] == "nginx-access" {
    elasticsearch {
      hosts => ["192.168.10.51:9200"]
      index => "nginx-access-%{+yyyy.MM.dd}"
    }
  } else if [category] == "nginx-error" {
    elasticsearch {
      hosts => ["192.168.10.51:9200"]
      index => "nginx-error-%{+yyyy.MM.dd}"
    }
  }
}