# CentOS 6 for pid in $(pgrep -u td-agent); do sudo cat /proc/$pid/limits | grep -P "open|proc"; done # 1024は小さすぎる Max processes 1024 57841 processes Max open files 1024 65536 files Max processes 1024 57841 processes Max open files 1024 65536 file
grep -i limit /etc/systemd/system/multi-user.target.wants/td-agent.service LimitNOFILE=65536
grep -i limit /etc/rc.d/init.d/td-agent ulimit -n 65536 1>/dev/null 2>&1 || true
td-agentログにエラーが出て、bufferファイルが残る場合がある。
bufferファイルの中身を調べたい場合。
yum eraseしただけでは色々ゴミが残っていた。
sudo service td-agent stop sudo yum erase td-agent sudo rm -rf /opt/td-agent/ /etc/td-agent/ /var/tmp/td-agent/
echo "SELECT name, password FROM user WHERE id='12823';" | pt-fingerprint --match-embedded-numbers select name, password from user where id=?
sudo td-agent-gem list sudo td-agent-gem cleanup sudo service td-agent restart
tag_parts[0] = aws_s3_logs tag_parts[1] = syslog tag_parts[2] = messages
sudo gpasswd -a td-agent wheel sudo chown :wheel /var/log/messages /var/log/cron /var/log/maillog /var/log/secure /var/log/yum.log sudo chmod 640 /var/log/messages /var/log/cron /var/log/maillog /var/log/secure /var/log/yum.log sudo service td-agent-agent restart
@include conf.d/*.conf <source> type monitor_agent bind 0.0.0.0 port 24220 </source>
<source> type tail format none path /var/log/cron pos_file /var/tmp/td-agent/aws_s3_logs.syslog.cron.pos tag aws_s3_logs.syslog.cron @label @raw </source> <source> type tail format none path /var/log/maillog pos_file /var/tmp/td-agent/aws_s3_logs.syslog.maillog.pos tag aws_s3_logs.syslog.maillog @label @raw </source> <source> type tail format none path /var/log/messages pos_file /var/tmp/td-agent/aws_s3_logs.syslog.messages.pos tag aws_s3_logs.syslog.messages @label @raw </source> <source> type tail format none path /var/log/secure pos_file /var/tmp/td-agent/aws_s3_logs.syslog.secure.pos tag aws_s3_logs.syslog.secure @label @raw </source> <source> type tail format none path /var/log/yum.log pos_file /var/tmp/td-agent/aws_s3_logs.yum.yum.pos tag aws_s3_logs.yum.yum @label @raw </source> <label @raw> <match aws_s3_logs.**> type forest subtype s3 <template> format single_value s3_bucket "my-bucket" s3_region "#{`curl -s -m 1 http://169.254.169.254/latest/meta-data/placement/availability-zone`.chop}" s3_object_key_format "%{path}.%{time_slice}.%{index}.%{file_extension}" path "${tag_parts[1]}/%Y/%m/%d/${tag_parts[1]}.${tag_parts[2]}.#{`hostname -s`.chomp}.#{`curl -s -m 1 http://169.254.169.254/latest/meta-data/instance-id`.chomp}" time_slice_format "%Y%m%dT%H00Z" time_slice_wait 10m utc buffer_type file buffer_path "/var/tmp/td-agent/s3.${tag}" buffer_chunk_limit 128m disable_retry_limit true flush_at_shutdown true </template> </match> </label>
sudo td-agent-gem install fluent-plugin-forest
path "#{ENV['FOO']}"
path "#{`hostname -s`.chomp}.#{`curl -s -m 1 http://169.254.169.254/latest/meta-data/instance-id`.chomp}.%Y%m%dT%H00Z"
sudo chmod o+t /tmp ls -la / | grep tmp drwxrwxrwt. 8 root root 40960 5月 15 18:57 2017 tmp
cat /etc/td-agent/td-agent.conf @include conf.d/*.conf <source> type monitor_agent bind 0.0.0.0 port 24220 </source>
curl -s -m 1 http://127.0.0.1:24220/api/plugins
curl -s -m 1 http://127.0.0.1:24220/api/plugins.json
curl -s -m 1 http://127.0.0.1:24220/api/plugins | perl -ne 'BEGIN{$sum;} if(/type:forward/ && /buffer_queue_length:(\d+)/){$sum+=$1;} END{print $sum;}' 5
curl -s -m 1 127.0.0.1:24220/api/plugins.json | jq '.plugins[] | select(.buffer_queue_length > 0)'
fluentd v0.12以降かつ、forwardプラグイン使用時に「require_ack_response」を付ける事で、ログの欠損を抑える
expire_dns_cache 300
sudo yum erase td-agent
sudo rpm -e --justdb td-agent
2015-05-22現在、デフォルトでインストールされるtd-agentが2.x系に変わっている。
互換性が無いためリポジトリのパスが変わっている。
1.x系を使いたい場合、以下のように手動でbaseurlを変更するとインストールできる
sudo vi /etc/yum.repos.d/td.repo ---- baseurl=http://packages.treasure-data.com/redhat/$basearch ---- sudo yum clean all sudo yum install td-agent-1.1.19-0.x86_64
echo '{"key":"message"}' | /usr/lib64/fluent/ruby/bin/fluent-cat debug.tag -h revice-td-agent-host
tail -f /var/log/td-agent/td-agent.log