filebeat收集json日志 _收集

莫道桑榆晚，为霞尚满天。这篇文章主要讲述filebeat收集json日志相关的知识，希望能为你提供帮助。
filebeat收集json日志，一 Tomcat的日志配置为json格式
[root@centos2 conf]# vim /usr/local/tomcat/conf/server.xml

#找到139行,将`pattern="%h %l %u %t & quot; %r& quot; %s%b" /> 删除添加下面的

pattern="& quot; clientip& quot; :& quot; %h& quot; , & quot; clientuser& quot; :& quot; %l& quot; , & quot; authenticated& quot; :& quot; %u& quot; , & quot; AccessTime& quot; :& quot; %t& quot; , & quot; method& quot; :& quot; %r& quot; , & quot; status& quot; :& quot; %s& quot; , & quot; SendBytes& quot; :& quot; %b& quot; , & quot; Query?string& quot; :& quot; %q& quot; , & quot; partner& quot; :& quot; %Refereri& quot; , & quot; AgentVersion& quot; :& quot; %User-Agenti& quot; "/>

二 nginx日志配置为json格式
[root@centos2 nginx]# head -n 50 nginx.conf #红色字体部分

user root; worker_processes 1; events worker_connections 1024; http include mime.types; default_type application/octet-stream; map $http_upgrade $connection_upgrade default upgrade; close; #按照json格式产生日志文件 log_formatjson "@timestamp": "$time_local", "remote_addr": "$remote_addr", "referer":"$http_referer", "request":"$request", 【filebeat收集json日志】 "status":$status, "bytes":$body_bytes_sent, "agent":"$http_user_agent", "x_forwarded": "$http_x_forwarded_for", "up_addr":"$upstream_addr", "up_host":"$upstream_http_host", "up_resp_time": "$upstream_response_time", "request_time": "$request_time" ; access_log /var/log/nginx/access.log json; client_max_body_size 100m; sendfile on; keepalive_timeout 65; upstream assemble ip_hash; server 192.168.2.12:8040fail_timeout=30s; upstream websocket server 192.168.2.12:8040; upstream websocketMQ server 192.168.2.12:3872; upstream nodejs ip_hash;

三修改filebeat配置文件

#===========================Filebeat inputs ============================= filebeat.inputs: - type: log enabled: true paths: - /usr/local/tomcat/logs/access_log* json.keys_under_root: true json.overwrite_keys: true tags: ["tomcat"] #-----------------------------Logstash output -------------------------------- output.logstash: # The Logstash hosts hosts: ["192.168.2.222:5044"] indices: - index:"tomcat-access-%[beat.version]-%+yyyy.MM" when.contains: tags: "tomcat"

.重启filebeat
[root@db01 ~]# systemctl restart filebeat

收集多个日志：
[root@centos2 filebeat]# vim /etc/filebeat/filebeat.yml

#===========================Filebeat inputs ============================= filebeat.inputs: - type: log tail_files: true scan_frequency: 5s backoff: 1s max_backoff: 10s paths: - /usr/local/tomcat/logs/catalina.out - /usr/local/tomcat/logs/access_log* fields: type: tomcat ip: 192.168.2.231 fields_under_root: true - type: log tail_files: true scan_frequency: 5s backoff: 1s max_backoff: 10s paths: - /home/docker/nginx/log/access.log fields: type: nginx ip: 192.168.2.231 fields_under_root: true #-----------------------------Logstash output -------------------------------- output.logstash: # The Logstash hosts enabled: true hosts: ["192.168.2.222:5044"]

[root@master conf.d]# cat /etc/logstash/conf.d/nginx.conf

input beats host => 0.0.0.0 port => 5044 filter if [type] == "access" grok match => "message" => (?< clientip> [0-9]1,3\\.[0-9]1,3\\.[0-9]1,3\\.[0-9]1,3) -(?< user> \\S+) \\[(?< timestamp> [^ ]+ \\+[0-9]+)\\]"(?< requesttype> [A-Z]+) (?< requesturl> [^ ]+) HTTP/\\d.\\d" (?< status> \\d+)(?< bodysize> \\d+) "(?< url> \\S+)" "[^"]+" #移除不需要的字段 remove_field => ["message","@version","path"] date match => ["requesttime","dd/MMM/yyyy:HH:mm:ss Z"] target => "@timestamp" output if [type] == "nginx" elasticsearch hosts => ["??http://192.168.2.222:9200??"] index => "nginx_log-%+YYYY.MM.dd" else if [type] == "tomcat" elasticsearch hosts => ["??http://192.168.2.222:9200??"] index => "tomcat_log-%+YYYY.MM.dd" else if [type] == "access" elasticsearch hosts => ["??http://192.168.2.222:9200??"] index => "access-%+YYYY.MM.dd" stdout codec=> rubydebug