Data Firehorseでs3へ集めたログをAthenaやGlue等で検索する時、パーティション(s3://bucket/prefix/dt=<YYYY-mm-dd-HH>/file.gz)があったほうが速度、金額的に良い。
AWSの最近の対応(Fargate, docker image提供)を見ると、この選択肢が良さそう
sudo usermod -aG wheel aws-kinesis-agent-user sudo chown :wheel /tmp/app.log sudo chmod 640 /tmp/app.log
cat /etc/logrotate.d/aws-kinesis-agent /var/log/aws-kinesis-agent/*.log { missingok notifempty compress dateext copytruncate create 0664 aws-kinesis-agent-user aws-kinesis-agent-user su aws-kinesis-agent-user aws-kinesis-agent-user }
{ "cloudwatch.emitMetrics": false, "cloudwatch.endpoint": "monitoring.ap-northeast-1.amazonaws.com", "firehose.endpoint": "firehose.ap-northeast-1.amazonaws.com", "kinesis.endpoint": "", "flows": [ ] }
{ "flows": [ { "filePattern": "/tmp/app.log*", "deliveryStream": "yourdeliverystream" } ] }
{ "flows": [ { "filePattern": "/tmp/app.log*", "deliveryStream": "yourdeliverystream" } ] }
{ "flows": [ { "dataProcessingOptions": [ { "logFormat": "SYSLOG", "optionName": "LOGTOJSON" } ], "deliveryStream": "yourdeliverystream", "filePattern": "/var/log/messages*" } ] }
{ "flows": [ { "dataProcessingOptions": [ { "logFormat": "SYSLOG", "optionName": "LOGTOJSON" }, { "logFormat": "SYSLOG", "optionName": "ADDEC2METADATA" } ], "deliveryStream": "yourdeliverystream", "filePattern": "/var/log/messages*" } ] }