Fluentd - yusukew62/docs GitHub Wiki

Fluentd

1.環境構築

# curl -L https://toolbelt.treasuredata.com/sh/install-redhat-td-agent2.sh | sh

1-1.エージェント側

# mkdir -p /etc/td-agent/conf.d/

設定差異

# cd /etc/td-agent/
# diff -wu td-agent.conf.org td-agent.conf
--- td-agent.conf.org   2017-05-29 20:28:21.877053487 +0900
+++ td-agent.conf       2017-05-29 21:32:35.650050897 +0900
@@ -8,24 +8,24 @@
 # @see http://docs.fluentd.org/articles/http-to-td
 #
 # This section matches events whose tag is td.DATABASE.TABLE
-<match td.*.*>
-  @type tdlog
-  apikey YOUR_API_KEY
-
-  auto_create_table
-  buffer_type file
-  buffer_path /var/log/td-agent/buffer/td
-
-  <secondary>
-    @type file
-    path /var/log/td-agent/failed_records
-  </secondary>
-</match>
+#<match td.*.*>
+#  @type tdlog
+#  apikey YOUR_API_KEY
+#
+#  auto_create_table
+#  buffer_type file
+#  buffer_path /var/log/td-agent/buffer/td
+#
+#  <secondary>
+#    @type file
+#    path /var/log/td-agent/failed_records
+#  </secondary>
+#</match>

 ## match tag=debug.** and dump to console
-<match debug.**>
-  @type stdout
-</match>
+#<match debug.**>
+#  @type stdout
+#</match>

 ####
 ## Source descriptions:
@@ -33,9 +33,9 @@

 ## built-in TCP input
 ## @see http://docs.fluentd.org/articles/in_forward
-<source>
-  @type forward
-</source>
+#<source>
+#  @type forward
+#</source>

 ## built-in UNIX socket input
 #<source>
@@ -46,17 +46,17 @@
 # POST http://localhost:8888/<tag>?json=<json>
 # POST http://localhost:8888/td.myapp.login?json={"user"%3A"me"}
 # @see http://docs.fluentd.org/articles/in_http
-<source>
-  @type http
-  port 8888
-</source>
+#<source>
+#  @type http
+#  port 8888
+#</source>

 ## live debugging agent
-<source>
-  @type debug_agent
-  bind 127.0.0.1
-  port 24230
-</source>
+#<source>
+#  @type debug_agent
+#  bind 127.0.0.1
+#  port 24230
+#</source>

 ####
 ## Examples:
@@ -105,3 +105,4 @@
 #    path /var/log/td-agent/td-%Y-%m-%d/%H.log
 #  </store>
 #</match>
+@include conf.d/*.conf

例としてSquidのaccess.logの転送で試す
簡略のためformat noneとして動作確認
他のフォーマットは本ページの「フォーマット」を参照

# cat proxy.conf
<source>
  @type tail
  format none
  path /var/log/squid/access.log
  pos_file /var/log/td-agent/squid-access.log.pos
  tag squid.access
</source>
<match squid.access>
  @type forward
  <server>
    host 192.168.1.28
    port 24224
  </server>
</match>

パーミッション設定
ログローテートされるとパーミッションが戻るので注意

# chmod 755 /var/log/squid/
# chmod 644 /var/log/squid/access.log

サービス起動

# service td-agent start

posファイルができていることを確認

# ls -l /var/log/td-agent/squid-access.log.pos

1-2.サーバ側

# mkdir -p /etc/td-agent/conf.d/

設定差異

--- td-agent.conf.org   2017-05-29 21:32:56.207058323 +0900
+++ td-agent.conf       2017-05-29 21:34:12.844058230 +0900
@@ -8,24 +8,24 @@
 # @see http://docs.fluentd.org/articles/http-to-td
 #
 # This section matches events whose tag is td.DATABASE.TABLE
-<match td.*.*>
-  @type tdlog
-  apikey YOUR_API_KEY
-
-  auto_create_table
-  buffer_type file
-  buffer_path /var/log/td-agent/buffer/td
-
-  <secondary>
-    @type file
-    path /var/log/td-agent/failed_records
-  </secondary>
-</match>
+#<match td.*.*>
+#  @type tdlog
+#  apikey YOUR_API_KEY
+#
+#  auto_create_table
+#  buffer_type file
+#  buffer_path /var/log/td-agent/buffer/td
+#
+#  <secondary>
+#    @type file
+#    path /var/log/td-agent/failed_records
+#  </secondary>
+#</match>

 ## match tag=debug.** and dump to console
-<match debug.**>
-  @type stdout
-</match>
+#<match debug.**>
+#  @type stdout
+#</match>


 ####
 ## Source descriptions:
@@ -33,9 +33,9 @@

 ## built-in TCP input
 ## @see http://docs.fluentd.org/articles/in_forward
-<source>
-  @type forward
-</source>
+#<source>
+#  @type forward
+#</source>

 ## built-in UNIX socket input
 #<source>
@@ -46,17 +46,17 @@
 # POST http://localhost:8888/<tag>?json=<json>
 # POST http://localhost:8888/td.myapp.login?json={"user"%3A"me"}
 # @see http://docs.fluentd.org/articles/in_http
-<source>
-  @type http
-  port 8888
-</source>
+#<source>
+#  @type http
+#  port 8888
+#</source>

 ## live debugging agent
-<source>
-  @type debug_agent
-  bind 127.0.0.1
-  port 24230
-</source>
+#<source>
+#  @type debug_agent
+#  bind 127.0.0.1
+#  port 24230
+#</source>

 ####
 ## Examples:
@@ -105,3 +105,4 @@
 #    path /var/log/td-agent/td-%Y-%m-%d/%H.log
 #  </store>
 #</match>
+@include conf.d/*.conf

例としてSquidのaccess.logの転送で試す

# cat efk.conf
<source>
  @type forward
  port 24224
</source>
<match squid.access>
  @type file
  path /var/log/fluent/squid/access.log
</match>

パーミッション設定

# mkdir -p /var/log/fluent/squid
# chmod -R 755 /var/log/fluent/
# chown -R td-agent:td-agent /var/log/fluent/

サービス起動

# service td-agent start

ログ確認

# ls /var/log/fluent/squid/access.log.*

2.プラグイン

2-1.fluentd-plugin

インストール

# td-agent-gem install fluent-plugin-elasticsearch

インストール確認

# td-agent-gem list elasticsearch

*** LOCAL GEMS ***

elasticsearch (5.0.4)
elasticsearch-api (5.0.4)
elasticsearch-transport (5.0.4)
fluent-plugin-elasticsearch (1.9.5)

ログ転送の確認のため、ホスト上のファイル出力とelasticsearchへの出力設定を記載

<source>
  @type forward
  port 24224
</source>
<match squid.access>
  @type copy
  <store>
    @type file
    path /var/log/fluent/squid/access.log
  </store>
  <store>
    @type elasticsearch
    host localhost
    port 9200
    index_name squid_index-
    type_name squid_type
  </store>
</match>

td-agentを起動させようとしたらgemの見つからないとのエラーでサービス起動しなかった
gemsのパスを追加後、サービス起動を試したら起動した

# find / -name gems
/opt/td-agent/embedded/lib/ruby/gems
/opt/td-agent/embedded/lib/ruby/gems/2.1.0/gems
# GEM_PATH="/opt/td-agent/embedded/lib/ruby/gems/2.1.0/"

3.フォーマット

3-1.Squid

3-1-1.ログフォーマット

squid.confのlogformat

logformat combined %>a %ui %un [%tl] "%rm %ru HTTP/%rv" %Hs %<st "%{Referer}>h" "%{User-Agent}>h" "%Ss:%Sh" "%{X-Forwarded-For}>h"

3-1-2.ログ

192.168.1.4 - - [30/May/2017:03:47:01 +0900] "CONNECT shavar.services.mozilla.com:443 HTTP/1.1" 200 3396 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:53.0) Gecko/20100101 Firefox/53.0" "TCP_TUNNEL:HIER_DIRECT" "-"

3-1-3.正規表現

^([\d.]+) (-) (-) \[(\w.+)\] (?:")(\w.+) (\w.+) (\w.+)(?:") (\d+) (\d+) (?:")(-)(?:") (?:")(.+)(?:") (?:")(.+)(?::)(.+)(?:") (?:")(-)(?:")

マッチ例

1.	192.168.1.4
2.	-
3.	-
4.	30/May/2017:03:47:01 +0900
5.	CONNECT
6.	shavar.services.mozilla.com:443
7.	HTTP/1.1
8.	200
9.	3396
10.	-
11.	Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:53.0) Gecko/20100101 Firefox/53.0
12.	TCP_TUNNEL
13.	HIER_DIRECT
14.	-

3-1-4.fluentd format

^(?<src_ip>[\d.]+) (?<user_name_ident>-) (?<user_name>-) \[(?<local_time>\w.+)\] (?:")(?<request_method>\w.+) (?<request_url>\w.+) (?<request_protocol_version>\w.+)(?:") (?<status_code>\d+) (?<total_size_of_request>\d+) (?:")(?<referer>-)(?:") (?:")(?<user_agent>.+)(?:") (?:")(?<squid_request_status>.+)(?::)(?<squid_hierarchy_status>.+)(?:") (?:")(?<x_forwarded_for>-)(?:")

fluentdの設定ファイル

<source>
  @type tail
  format /^(?<src_ip>[\d.]+) (?<user_name_ident>-) (?<user_name>-) (?<local_time>\[\w.+\]) (?:")(?<request_method>\w.+) (?<request_url>\w.+) (?<request_protocol_version>\w.+)(?:") (?<status_code>\d+) (?<total_size_of_request>\d+) (?:")(?<referer>-)(?:") (?:")(?<user_agent>.+)(?:") (?:")(?<squid_request_status>.+)(?::)(?<squid_hierarchy_status>.+)(?:") (?:")(?<x_forwarded_for>-)(?:")/
  path /var/log/squid/access.log
  pos_file /var/log/td-agent/squid-access.log.pos
  tag squid.access
</source>
<match squid.access>
  @type forward
  <server>
    host 192.168.1.28
    port 24224
  </server>
</match>
Key Value
src_ip 192.168.1.4
user_name_ident -
user_name -
local_time 30/May/2017:03:47:01 +0900
request_method CONNECT
request_url shavar.services.mozilla.com:443
request_protocol_version HTTP/1.1
status_code 200
total_size_of_request 3396
referer -
user_agent Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:53.0) Gecko/20100101 Firefox/53.0
squid_request_status TCP_TUNNEL
squid_hierarchy_status HIER_DIRECT
x_forwarded_for -

3-1-5.動作確認

2017-05-30T05:45:21+09:00       squid.access    {"src_ip":"192.168.1.4","user_name_ident":"-","user_name":"-","local_time":"[30/May/2017:05:45:21 +0900]","request_method":"CONNECT","request_url":"www.nifty.com:443","request_protocol_version":"HTTP/1.1","status_code":"200","total_size_of_request":"560396","referer":"-","user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:53.0) Gecko/20100101 Firefox/53.0","squid_request_status":"TCP_TUNNEL","squid_hierarchy_status":"HIER_DIRECT","x_forwarded_for":"-"}

3-2.ifilter

3-2-1.ログフォーマット

ifilterのlogformat

3-2-2.ログ

Apr 1 15:09:09 18051 900/01/ 192.168.1.91:8000 192.168.1.24 - username [1/Apr/2014:15:09:08 +0900] 503 0 385 block 9 51 51 -1 -1 default - 0 - "GET http://www.abcd.local HTTP/1.0" http://www.abcd.local/search Mozilla%2F5.0 text%2Fhtml%3B+charset%3Dutf-8 63EBCF4D

3-2-3.正規表現

^([^ ]* [^ ]* [^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) \[([^ ]* [^ ]*)\] ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) "([^ ]*) ([^ ]*) ([^ "]*)" ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*)

マッチ例

1.	Apr 1 15:09:09
2.	18051
3.	900/01/
4.	192.168.1.91:8000
5.	192.168.1.24
6.	-
7.	username
8.	1/Apr/2014:15:09:08 +0900
9.	503
10.	0
11.	385
12.	block
13.	9
14.	51
15.	51
16.	-1
17.	-1
18.	default
19.	-
20.	0
21.	-
22.	GET
23.	http://www.abcd.local
24.	HTTP/1.0
25.	http://www.abcd.local/search
26.	Mozilla%2F5.0
27.	text%2Fhtml%3B+charset%3Dutf-8
28.	63EBCF4D

3-2-4.fluentd format

^(?<timestamp>[^ ]* [^ ]* [^ ]*) (?<process_number>[^ ]*) (?<ifilter_version>[^ ]*) (?<top_proxy_ip>[^ ]*) (?<client_addr>[^ ]*) (?<computer_name>[^ ]*) (?<username>[^ ]*) \[(?<request_time>[^ ]* [^ ]*)\] (?<http_response_code>[^ ]*) (?<http_response_size>[^ ]*) (?<http_request_size>[^ ]*) (?<filter_action>[^ ]*) (?<object_id>[^ ]*) (?<filter_reason_number>[^ ]*) (?<url_category_list>[^ ]*) (?<web_service_id>[^ ]*) (?<web_service_sub_id>[^ ]*) (?<group_id>[^ ]*) (?<virus_name>[^ ]*) (?<ssl_parameter>[^ ]*) (?<file_info_by_post>[^ ]*) "(?<http_method>[^ ]*) (?<url>[^ ]*) (?<http_version>[^ "]*)" (?<http_referer>[^ ]*) (?<http_user_agent>[^ ]*) (?<http_content_type>[^ ]*) (?<check_sum>[^ ]*)

fluentdの設定ファイル

<source>
  @type tail
  format /^(?<timestamp>[^ ]* [^ ]* [^ ]*) (?<process_number>[^ ]*) (?<ifilter_version>[^ ]*) (?<top_proxy_ip>[^ ]*) (?<client_addr>[^ ]*) (?<computer_name>[^ ]*) (?<username>[^ ]*) \[(?<request_time>[^ ]* [^ ]*)\] (?<http_response_code>[^ ]*) (?<http_response_size>[^ ]*) (?<http_request_size>[^ ]*) (?<filter_action>[^ ]*) (?<object_id>[^ ]*) (?<filter_reason_number>[^ ]*) (?<url_category_list>[^ ]*) (?<web_service_id>[^ ]*) (?<web_service_sub_id>[^ ]*) (?<group_id>[^ ]*) (?<virus_name>[^ ]*) (?<ssl_parameter>[^ ]*) (?<file_info_by_post>[^ ]*) "(?<http_method>[^ ]*) (?<url>[^ ]*) (?<http_version>[^ "]*)" (?<http_referer>[^ ]*) (?<http_user_agent>[^ ]*) (?<http_content_type>[^ ]*) (?<check_sum>[^ ]*)/ 
  path /var/log/squid/access.log
  pos_file /var/log/td-agent/squid-access.log.pos
  tag squid.access
</source>
<match squid.access>
  @type forward
  <server>
    host 192.168.1.28
    port 24224
  </server>
</match>
Key Value
timestamp Apr 1 15:09:09
process_number 18051
ifilter_version 900/01/
top_proxy_ip 192.168.1.91:8000
client_addr 192.168.1.24
computer_name -
username username
request_time 1/Apr/2014:15:09:08 +0900
http_response_code 503
http_response_size 0
http_request_size 385
filter_action block
object_id 9
filter_reason_number 51
url_category_list 51
web_service_id -1
web_service_sub_id -1
group_id default
virus_name -
ssl_parameter 0
file_info_by_post -
http_method GET
url http://www.abcd.local
http_version HTTP/1.0
http_referer http://www.abcd.local/search
http_user_agent Mozilla%2F5.0
http_content_type text%2Fhtml%3B+charset%3Dutf-8
check_sum 63EBCF4D

3-2-5.動作確認

3.その他

  • 正規表現の作成補助ツールはrubularで行う
  • 構文チェックはfluentularで行う
⚠️ **GitHub.com Fallback** ⚠️