Fluentd - yusukew62/docs GitHub Wiki
# curl -L https://toolbelt.treasuredata.com/sh/install-redhat-td-agent2.sh | sh# mkdir -p /etc/td-agent/conf.d/設定差異
# cd /etc/td-agent/
# diff -wu td-agent.conf.org td-agent.conf
--- td-agent.conf.org 2017-05-29 20:28:21.877053487 +0900
+++ td-agent.conf 2017-05-29 21:32:35.650050897 +0900
@@ -8,24 +8,24 @@
# @see http://docs.fluentd.org/articles/http-to-td
#
# This section matches events whose tag is td.DATABASE.TABLE
-<match td.*.*>
- @type tdlog
- apikey YOUR_API_KEY
-
- auto_create_table
- buffer_type file
- buffer_path /var/log/td-agent/buffer/td
-
- <secondary>
- @type file
- path /var/log/td-agent/failed_records
- </secondary>
-</match>
+#<match td.*.*>
+# @type tdlog
+# apikey YOUR_API_KEY
+#
+# auto_create_table
+# buffer_type file
+# buffer_path /var/log/td-agent/buffer/td
+#
+# <secondary>
+# @type file
+# path /var/log/td-agent/failed_records
+# </secondary>
+#</match>
## match tag=debug.** and dump to console
-<match debug.**>
- @type stdout
-</match>
+#<match debug.**>
+# @type stdout
+#</match>
####
## Source descriptions:
@@ -33,9 +33,9 @@
## built-in TCP input
## @see http://docs.fluentd.org/articles/in_forward
-<source>
- @type forward
-</source>
+#<source>
+# @type forward
+#</source>
## built-in UNIX socket input
#<source>
@@ -46,17 +46,17 @@
# POST http://localhost:8888/<tag>?json=<json>
# POST http://localhost:8888/td.myapp.login?json={"user"%3A"me"}
# @see http://docs.fluentd.org/articles/in_http
-<source>
- @type http
- port 8888
-</source>
+#<source>
+# @type http
+# port 8888
+#</source>
## live debugging agent
-<source>
- @type debug_agent
- bind 127.0.0.1
- port 24230
-</source>
+#<source>
+# @type debug_agent
+# bind 127.0.0.1
+# port 24230
+#</source>
####
## Examples:
@@ -105,3 +105,4 @@
# path /var/log/td-agent/td-%Y-%m-%d/%H.log
# </store>
#</match>
+@include conf.d/*.conf例としてSquidのaccess.logの転送で試す
簡略のためformat noneとして動作確認
他のフォーマットは本ページの「フォーマット」を参照
# cat proxy.conf
<source>
@type tail
format none
path /var/log/squid/access.log
pos_file /var/log/td-agent/squid-access.log.pos
tag squid.access
</source>
<match squid.access>
@type forward
<server>
host 192.168.1.28
port 24224
</server>
</match>パーミッション設定
ログローテートされるとパーミッションが戻るので注意
# chmod 755 /var/log/squid/
# chmod 644 /var/log/squid/access.logサービス起動
# service td-agent startposファイルができていることを確認
# ls -l /var/log/td-agent/squid-access.log.pos# mkdir -p /etc/td-agent/conf.d/設定差異
--- td-agent.conf.org 2017-05-29 21:32:56.207058323 +0900
+++ td-agent.conf 2017-05-29 21:34:12.844058230 +0900
@@ -8,24 +8,24 @@
# @see http://docs.fluentd.org/articles/http-to-td
#
# This section matches events whose tag is td.DATABASE.TABLE
-<match td.*.*>
- @type tdlog
- apikey YOUR_API_KEY
-
- auto_create_table
- buffer_type file
- buffer_path /var/log/td-agent/buffer/td
-
- <secondary>
- @type file
- path /var/log/td-agent/failed_records
- </secondary>
-</match>
+#<match td.*.*>
+# @type tdlog
+# apikey YOUR_API_KEY
+#
+# auto_create_table
+# buffer_type file
+# buffer_path /var/log/td-agent/buffer/td
+#
+# <secondary>
+# @type file
+# path /var/log/td-agent/failed_records
+# </secondary>
+#</match>
## match tag=debug.** and dump to console
-<match debug.**>
- @type stdout
-</match>
+#<match debug.**>
+# @type stdout
+#</match>
####
## Source descriptions:
@@ -33,9 +33,9 @@
## built-in TCP input
## @see http://docs.fluentd.org/articles/in_forward
-<source>
- @type forward
-</source>
+#<source>
+# @type forward
+#</source>
## built-in UNIX socket input
#<source>
@@ -46,17 +46,17 @@
# POST http://localhost:8888/<tag>?json=<json>
# POST http://localhost:8888/td.myapp.login?json={"user"%3A"me"}
# @see http://docs.fluentd.org/articles/in_http
-<source>
- @type http
- port 8888
-</source>
+#<source>
+# @type http
+# port 8888
+#</source>
## live debugging agent
-<source>
- @type debug_agent
- bind 127.0.0.1
- port 24230
-</source>
+#<source>
+# @type debug_agent
+# bind 127.0.0.1
+# port 24230
+#</source>
####
## Examples:
@@ -105,3 +105,4 @@
# path /var/log/td-agent/td-%Y-%m-%d/%H.log
# </store>
#</match>
+@include conf.d/*.conf例としてSquidのaccess.logの転送で試す
# cat efk.conf
<source>
@type forward
port 24224
</source>
<match squid.access>
@type file
path /var/log/fluent/squid/access.log
</match>パーミッション設定
# mkdir -p /var/log/fluent/squid
# chmod -R 755 /var/log/fluent/
# chown -R td-agent:td-agent /var/log/fluent/サービス起動
# service td-agent startログ確認
# ls /var/log/fluent/squid/access.log.*インストール
# td-agent-gem install fluent-plugin-elasticsearchインストール確認
# td-agent-gem list elasticsearch
*** LOCAL GEMS ***
elasticsearch (5.0.4)
elasticsearch-api (5.0.4)
elasticsearch-transport (5.0.4)
fluent-plugin-elasticsearch (1.9.5)ログ転送の確認のため、ホスト上のファイル出力とelasticsearchへの出力設定を記載
<source>
@type forward
port 24224
</source>
<match squid.access>
@type copy
<store>
@type file
path /var/log/fluent/squid/access.log
</store>
<store>
@type elasticsearch
host localhost
port 9200
index_name squid_index-
type_name squid_type
</store>
</match>td-agentを起動させようとしたらgemの見つからないとのエラーでサービス起動しなかった
gemsのパスを追加後、サービス起動を試したら起動した
# find / -name gems
/opt/td-agent/embedded/lib/ruby/gems
/opt/td-agent/embedded/lib/ruby/gems/2.1.0/gems
# GEM_PATH="/opt/td-agent/embedded/lib/ruby/gems/2.1.0/"squid.confのlogformat
logformat combined %>a %ui %un [%tl] "%rm %ru HTTP/%rv" %Hs %<st "%{Referer}>h" "%{User-Agent}>h" "%Ss:%Sh" "%{X-Forwarded-For}>h"
192.168.1.4 - - [30/May/2017:03:47:01 +0900] "CONNECT shavar.services.mozilla.com:443 HTTP/1.1" 200 3396 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:53.0) Gecko/20100101 Firefox/53.0" "TCP_TUNNEL:HIER_DIRECT" "-"
^([\d.]+) (-) (-) \[(\w.+)\] (?:")(\w.+) (\w.+) (\w.+)(?:") (\d+) (\d+) (?:")(-)(?:") (?:")(.+)(?:") (?:")(.+)(?::)(.+)(?:") (?:")(-)(?:")
マッチ例
1. 192.168.1.4
2. -
3. -
4. 30/May/2017:03:47:01 +0900
5. CONNECT
6. shavar.services.mozilla.com:443
7. HTTP/1.1
8. 200
9. 3396
10. -
11. Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:53.0) Gecko/20100101 Firefox/53.0
12. TCP_TUNNEL
13. HIER_DIRECT
14. -
^(?<src_ip>[\d.]+) (?<user_name_ident>-) (?<user_name>-) \[(?<local_time>\w.+)\] (?:")(?<request_method>\w.+) (?<request_url>\w.+) (?<request_protocol_version>\w.+)(?:") (?<status_code>\d+) (?<total_size_of_request>\d+) (?:")(?<referer>-)(?:") (?:")(?<user_agent>.+)(?:") (?:")(?<squid_request_status>.+)(?::)(?<squid_hierarchy_status>.+)(?:") (?:")(?<x_forwarded_for>-)(?:")
fluentdの設定ファイル
<source>
@type tail
format /^(?<src_ip>[\d.]+) (?<user_name_ident>-) (?<user_name>-) (?<local_time>\[\w.+\]) (?:")(?<request_method>\w.+) (?<request_url>\w.+) (?<request_protocol_version>\w.+)(?:") (?<status_code>\d+) (?<total_size_of_request>\d+) (?:")(?<referer>-)(?:") (?:")(?<user_agent>.+)(?:") (?:")(?<squid_request_status>.+)(?::)(?<squid_hierarchy_status>.+)(?:") (?:")(?<x_forwarded_for>-)(?:")/
path /var/log/squid/access.log
pos_file /var/log/td-agent/squid-access.log.pos
tag squid.access
</source>
<match squid.access>
@type forward
<server>
host 192.168.1.28
port 24224
</server>
</match>
| Key | Value |
|---|---|
| src_ip | 192.168.1.4 |
| user_name_ident | - |
| user_name | - |
| local_time | 30/May/2017:03:47:01 +0900 |
| request_method | CONNECT |
| request_url | shavar.services.mozilla.com:443 |
| request_protocol_version | HTTP/1.1 |
| status_code | 200 |
| total_size_of_request | 3396 |
| referer | - |
| user_agent | Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:53.0) Gecko/20100101 Firefox/53.0 |
| squid_request_status | TCP_TUNNEL |
| squid_hierarchy_status | HIER_DIRECT |
| x_forwarded_for | - |
2017-05-30T05:45:21+09:00 squid.access {"src_ip":"192.168.1.4","user_name_ident":"-","user_name":"-","local_time":"[30/May/2017:05:45:21 +0900]","request_method":"CONNECT","request_url":"www.nifty.com:443","request_protocol_version":"HTTP/1.1","status_code":"200","total_size_of_request":"560396","referer":"-","user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:53.0) Gecko/20100101 Firefox/53.0","squid_request_status":"TCP_TUNNEL","squid_hierarchy_status":"HIER_DIRECT","x_forwarded_for":"-"}
ifilterのlogformat
Apr 1 15:09:09 18051 900/01/ 192.168.1.91:8000 192.168.1.24 - username [1/Apr/2014:15:09:08 +0900] 503 0 385 block 9 51 51 -1 -1 default - 0 - "GET http://www.abcd.local HTTP/1.0" http://www.abcd.local/search Mozilla%2F5.0 text%2Fhtml%3B+charset%3Dutf-8 63EBCF4D
^([^ ]* [^ ]* [^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) \[([^ ]* [^ ]*)\] ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) "([^ ]*) ([^ ]*) ([^ "]*)" ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*)
マッチ例
1. Apr 1 15:09:09
2. 18051
3. 900/01/
4. 192.168.1.91:8000
5. 192.168.1.24
6. -
7. username
8. 1/Apr/2014:15:09:08 +0900
9. 503
10. 0
11. 385
12. block
13. 9
14. 51
15. 51
16. -1
17. -1
18. default
19. -
20. 0
21. -
22. GET
23. http://www.abcd.local
24. HTTP/1.0
25. http://www.abcd.local/search
26. Mozilla%2F5.0
27. text%2Fhtml%3B+charset%3Dutf-8
28. 63EBCF4D
^(?<timestamp>[^ ]* [^ ]* [^ ]*) (?<process_number>[^ ]*) (?<ifilter_version>[^ ]*) (?<top_proxy_ip>[^ ]*) (?<client_addr>[^ ]*) (?<computer_name>[^ ]*) (?<username>[^ ]*) \[(?<request_time>[^ ]* [^ ]*)\] (?<http_response_code>[^ ]*) (?<http_response_size>[^ ]*) (?<http_request_size>[^ ]*) (?<filter_action>[^ ]*) (?<object_id>[^ ]*) (?<filter_reason_number>[^ ]*) (?<url_category_list>[^ ]*) (?<web_service_id>[^ ]*) (?<web_service_sub_id>[^ ]*) (?<group_id>[^ ]*) (?<virus_name>[^ ]*) (?<ssl_parameter>[^ ]*) (?<file_info_by_post>[^ ]*) "(?<http_method>[^ ]*) (?<url>[^ ]*) (?<http_version>[^ "]*)" (?<http_referer>[^ ]*) (?<http_user_agent>[^ ]*) (?<http_content_type>[^ ]*) (?<check_sum>[^ ]*)
fluentdの設定ファイル
<source>
@type tail
format /^(?<timestamp>[^ ]* [^ ]* [^ ]*) (?<process_number>[^ ]*) (?<ifilter_version>[^ ]*) (?<top_proxy_ip>[^ ]*) (?<client_addr>[^ ]*) (?<computer_name>[^ ]*) (?<username>[^ ]*) \[(?<request_time>[^ ]* [^ ]*)\] (?<http_response_code>[^ ]*) (?<http_response_size>[^ ]*) (?<http_request_size>[^ ]*) (?<filter_action>[^ ]*) (?<object_id>[^ ]*) (?<filter_reason_number>[^ ]*) (?<url_category_list>[^ ]*) (?<web_service_id>[^ ]*) (?<web_service_sub_id>[^ ]*) (?<group_id>[^ ]*) (?<virus_name>[^ ]*) (?<ssl_parameter>[^ ]*) (?<file_info_by_post>[^ ]*) "(?<http_method>[^ ]*) (?<url>[^ ]*) (?<http_version>[^ "]*)" (?<http_referer>[^ ]*) (?<http_user_agent>[^ ]*) (?<http_content_type>[^ ]*) (?<check_sum>[^ ]*)/
path /var/log/squid/access.log
pos_file /var/log/td-agent/squid-access.log.pos
tag squid.access
</source>
<match squid.access>
@type forward
<server>
host 192.168.1.28
port 24224
</server>
</match>
| Key | Value |
|---|---|
| timestamp | Apr 1 15:09:09 |
| process_number | 18051 |
| ifilter_version | 900/01/ |
| top_proxy_ip | 192.168.1.91:8000 |
| client_addr | 192.168.1.24 |
| computer_name | - |
| username | username |
| request_time | 1/Apr/2014:15:09:08 +0900 |
| http_response_code | 503 |
| http_response_size | 0 |
| http_request_size | 385 |
| filter_action | block |
| object_id | 9 |
| filter_reason_number | 51 |
| url_category_list | 51 |
| web_service_id | -1 |
| web_service_sub_id | -1 |
| group_id | default |
| virus_name | - |
| ssl_parameter | 0 |
| file_info_by_post | - |
| http_method | GET |
| url | http://www.abcd.local |
| http_version | HTTP/1.0 |
| http_referer | http://www.abcd.local/search |
| http_user_agent | Mozilla%2F5.0 |
| http_content_type | text%2Fhtml%3B+charset%3Dutf-8 |
| check_sum | 63EBCF4D |
- 正規表現の作成補助ツールはrubularで行う
- 構文チェックはfluentularで行う