Logstash|Logstash geoip 库测试 Logstashgeoip库测试

1. nginx access log 测试文件

# cat /usr/share/logstash/nginx_access_geoip_test.log 33.55.69.101 - - [24/May/2018:01:50:26 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36" 222.174.69.101 - - [24/May/2018:01:52:11 +0800] "GET /abc HTTP/1.1" 404 3652 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36"

2. logstash 配置

input { file { # 指定文件路径 path => "/usr/share/logstash/nginx_access_geoip_test.log" type => "nginx_access" start_position => "beginning" } }filter { grok { # 匹配日志内容，分割字段 match => { "message" => '%{IPORHOST:remote_ip} - %{DATA:user_name} \[%{HTTPDATE:time}\] "%{WORD:method} %{DATA:uri} HTTP/%{NUMBER:http_version}" %{NUMBER:response_code} %{NUMBER:bytes} "%{DATA:referrer}" "%{DATA:agent}"' } } mutate { # bytes 字段由 string 类型转换为 number 类型，以便可以做 sum 之类的统计分析 convert => { "bytes" => "integer" } }date { # 用 time 字段展示日志原来的北京时间 match => [ "time", "dd/MMM/YYYY:HH:mm:ss Z" ] locale => en }geoip { # 对来源 IP 做 geoip 查询归类 source => "remote_ip" target => "geoip" }useragent { source => "agent" target => "user_agent" } }output { stdout { codec => "rubydebug" } }

3. 测试结果

# /usr/share/logstash/bin/logstash -f /usr/share/logstash/geoip.conf WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console [INFO ] 2018-06-01 23:17:06.036 [main] scaffold - Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"} [INFO ] 2018-06-01 23:17:06.044 [main] scaffold - Initializing module {:module_name=>"netflow", :directory=>"/usr/share/logstash/modules/netflow/configuration"} [WARN ] 2018-06-01 23:17:06.529 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified [INFO ] 2018-06-01 23:17:06.814 [LogStash::Runner] runner - Starting Logstash {"logstash.version"=>"6.2.3"} [INFO ] 2018-06-01 23:17:07.111 [Api Webserver] agent - Successfully started Logstash API endpoint {:port=>9601} [INFO ] 2018-06-01 23:17:09.025 [Ruby-0-Thread-1: /usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:22] pipeline - Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50} [INFO ] 2018-06-01 23:17:09.134 [[main]-pipeline-manager] geoip - Using geoip database {:path=>"/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-filter-geoip-5.0.3-java/vendor/GeoLite2-City.mmdb"} [INFO ] 2018-06-01 23:17:10.125 [Ruby-0-Thread-1: /usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:22] pipeline - Pipeline started succesfully {:pipeline_id=>"main", :thread=>"#"} [INFO ] 2018-06-01 23:17:10.180 [Ruby-0-Thread-1: /usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:22] agent - Pipelines running {:count=>1, :pipelines=>["main"]} { "uri" => "/", "bytes" => 0, "@version" => "1", "method" => "GET", "host" => "devops-01", "geoip" => { "ip" => "33.55.69.101", "country_code3" => "US", "latitude" => 37.751, "country_code2" => "US", "longitude" => -97.822, "continent_code" => "NA", "location" => { "lon" => -97.822, "lat" => 37.751 }, "country_name" => "United States" }, "http_version" => "1.1", "time" => "24/May/2018:01:50:26 +0800", "message" => "33.55.69.101 - - [24/May/2018:01:50:26 +0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36\"", "@timestamp" => 2018-05-23T17:50:26.000Z, "remote_ip" => "33.55.69.101", "response_code" => "304", "user_name" => "-", "agent" => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36", "type" => "nginx_access", "path" => "/usr/share/logstash/nginx_access_geoip_test.log", "referrer" => "-", "user_agent" => { "device" => "Other", "os_name" => "Windows 10", "major" => "64", "patch" => "3282", "build" => "", "name" => "Chrome", "minor" => "0", "os" => "Windows 10" } } { "uri" => "/abc", "bytes" => 3652, "@version" => "1", "method" => "GET", "host" => "devops-01", "geoip" => { "ip" => "222.174.69.101", "city_name" => "Jinan", "location" => { "lon" => 116.9972, "lat" => 36.6683 }, "country_name" => "China", "country_code3" => "CN", "region_name" => "Shandong", "latitude" => 36.6683, "country_code2" => "CN", "longitude" => 116.9972, "timezone" => "Asia/Shanghai", "continent_code" => "AS", "region_code" => "37" }, "http_version" => "1.1", "time" => "24/May/2018:01:52:11 +0800", "message" => "222.174.69.101 - - [24/May/2018:01:52:11 +0800] \"GET /abc HTTP/1.1\" 404 3652 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36\"", "@timestamp" => 2018-05-23T17:52:11.000Z, "remote_ip" => "222.174.69.101", "response_code" => "404", "user_name" => "-", "agent" => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36", "type" => "nginx_access", "path" => "/usr/share/logstash/nginx_access_geoip_test.log", "referrer" => "-", "user_agent" => { "device" => "Other", "os_name" => "Windows 10", "major" => "64", "patch" => "3282", "build" => "", "name" => "Chrome", "minor" => "0", "os" => "Windows 10" } }