![[PukiWiki]](image/owl.png "[PukiWiki]")
# CentOS 6 for pid in $(pgrep -u td-agent); do sudo cat /proc/$pid/limits | grep -P "open|proc"; done # 1024は小さすぎる Max processes 1024 57841 processes Max open files 1024 65536 files Max processes 1024 57841 processes Max open files 1024 65536 file
grep -i limit /etc/systemd/system/multi-user.target.wants/td-agent.service LimitNOFILE=65536
grep -i limit /etc/rc.d/init.d/td-agent ulimit -n 65536 1>/dev/null 2>&1 || true
td-agentログにエラーが出て、bufferファイルが残る場合がある。
bufferファイルの中身を調べたい場合。
yum eraseしただけでは色々ゴミが残っていた。
sudo service td-agent stop sudo yum erase td-agent sudo rm -rf /opt/td-agent/ /etc/td-agent/ /var/tmp/td-agent/
echo "SELECT name, password FROM user WHERE id='12823';" | pt-fingerprint --match-embedded-numbers select name, password from user where id=?
sudo td-agent-gem list sudo td-agent-gem cleanup sudo service td-agent restart
tag_parts[0] = aws_s3_logs tag_parts[1] = syslog tag_parts[2] = messages
sudo gpasswd -a td-agent wheel sudo chown :wheel /var/log/messages /var/log/cron /var/log/maillog /var/log/secure /var/log/yum.log sudo chmod 640 /var/log/messages /var/log/cron /var/log/maillog /var/log/secure /var/log/yum.log sudo service td-agent-agent restart
@include conf.d/*.conf <source> type monitor_agent bind 0.0.0.0 port 24220 </source>
<source>
type tail
format none
path /var/log/cron
pos_file /var/tmp/td-agent/aws_s3_logs.syslog.cron.pos
tag aws_s3_logs.syslog.cron
@label @raw
</source>
<source>
type tail
format none
path /var/log/maillog
pos_file /var/tmp/td-agent/aws_s3_logs.syslog.maillog.pos
tag aws_s3_logs.syslog.maillog
@label @raw
</source>
<source>
type tail
format none
path /var/log/messages
pos_file /var/tmp/td-agent/aws_s3_logs.syslog.messages.pos
tag aws_s3_logs.syslog.messages
@label @raw
</source>
<source>
type tail
format none
path /var/log/secure
pos_file /var/tmp/td-agent/aws_s3_logs.syslog.secure.pos
tag aws_s3_logs.syslog.secure
@label @raw
</source>
<source>
type tail
format none
path /var/log/yum.log
pos_file /var/tmp/td-agent/aws_s3_logs.yum.yum.pos
tag aws_s3_logs.yum.yum
@label @raw
</source>
<label @raw>
<match aws_s3_logs.**>
type forest
subtype s3
<template>
format single_value
s3_bucket "my-bucket"
s3_region "#{`curl -s -m 1 http://169.254.169.254/latest/meta-data/placement/availability-zone`.chop}"
s3_object_key_format "%{path}.%{time_slice}.%{index}.%{file_extension}"
path "${tag_parts[1]}/%Y/%m/%d/${tag_parts[1]}.${tag_parts[2]}.#{`hostname -s`.chomp}.#{`curl -s -m 1 http://169.254.169.254/latest/meta-data/instance-id`.chomp}"
time_slice_format "%Y%m%dT%H00Z"
time_slice_wait 10m
utc
buffer_type file
buffer_path "/var/tmp/td-agent/s3.${tag}"
buffer_chunk_limit 128m
disable_retry_limit true
flush_at_shutdown true
</template>
</match>
</label>sudo td-agent-gem install fluent-plugin-forest
path "#{ENV['FOO']}"path "#{`hostname -s`.chomp}.#{`curl -s -m 1 http://169.254.169.254/latest/meta-data/instance-id`.chomp}.%Y%m%dT%H00Z"sudo chmod o+t /tmp ls -la / | grep tmp drwxrwxrwt. 8 root root 40960 5月 15 18:57 2017 tmp
cat /etc/td-agent/td-agent.conf @include conf.d/*.conf <source> type monitor_agent bind 0.0.0.0 port 24220 </source>
curl -s -m 1 http://127.0.0.1:24220/api/plugins
curl -s -m 1 http://127.0.0.1:24220/api/plugins.json
curl -s -m 1 http://127.0.0.1:24220/api/plugins | perl -ne 'BEGIN{$sum;} if(/type:forward/ && /buffer_queue_length:(\d+)/){$sum+=$1;} END{print $sum;}' 5
curl -s -m 1 127.0.0.1:24220/api/plugins.json | jq '.plugins[] | select(.buffer_queue_length > 0)'
fluentd v0.12以降かつ、forwardプラグイン使用時に「require_ack_response」を付ける事で、ログの欠損を抑える
expire_dns_cache 300
sudo yum erase td-agent
sudo rpm -e --justdb td-agent
2015-05-22現在、デフォルトでインストールされるtd-agentが2.x系に変わっている。
互換性が無いためリポジトリのパスが変わっている。
1.x系を使いたい場合、以下のように手動でbaseurlを変更するとインストールできる
sudo vi /etc/yum.repos.d/td.repo ---- baseurl=http://packages.treasure-data.com/redhat/$basearch ---- sudo yum clean all sudo yum install td-agent-1.1.19-0.x86_64
echo '{"key":"message"}' | /usr/lib64/fluent/ruby/bin/fluent-cat debug.tag -h revice-td-agent-host
tail -f /var/log/td-agent/td-agent.log
