Logstash处理数据用法示例---待完善

filter {
  mutate { 
    rename => [ "message", "blog_html" ] 
    copy => { "blog_html" => "blog_text" } 
    copy => { "published" => "@timestamp" } 
  } 
  mutate { 
    gsub => [  
      "blog_text", "<.*?>", "",
      "blog_text", "[\n\t]", " " 
    ] 
    remove_field => [ "published", "author" ] 
  } 
}

apache日志
日志格式

83.149.9.216 - - [17/May/2015:10:05:03 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"
83.149.9.216 - - [17/May/2015:10:05:43 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-dashboard3.png HTTP/1.1" 200 171717 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"

filter {
  grok { # 通过正则表达式来匹配我们的每一条log信息,把相应的值赋值给相应的变量
    match => {
      "message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}'
    }
  }
 
  date { # @timestamp表示的在运行时的当前timestamp。把@timestamp转换成来自于log里的时间信息，也就是timestamp所表述的时间。
    match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]
    locale => en
  }
 
  geoip { # 根据IP地址解析是来自哪一个地方的以及它的经纬度等信息
    source => "clientip"
  }
 
  useragent { # 添加有关useragent（如系列，操作系统，版本和设备）的信息
    source => "agent"
    target => "useragent"
  }
}

csv文件

input {
  file {
    path => "/Users/liuxg/data/cars.csv"
    start_position => "beginning"
    sincedb_path => "null"
  }
}
 
filter {
  csv {
    # 分隔符
    separator => ","
    # 表格的列名
    columns => [ "maker", "model", "mileage", "manufacture_year", "engine_displacement",
    "engine_power", "body_type", "color_slug", "stk_year", "transmission", "door_count",
    "seat_count", "fuel_type", "date_created", "date_last_seen", "price_eur" ]
  }
  # 转换数据类型
  mutate { convert => ["mileage", "integer"] }
  mutate { convert => ["price_eur", "float"] }
  mutate { convert => ["engine_power", "integer"] }
  mutate { convert => ["door_count", "integer"] }
  mutate { convert => ["seat_count", "integer"] }
}

# 在input中，path指向csv文件的位置。start_position指向beginning。对于一个实时的数据源，它通常是ending，表示它每次都是从最后拿到那个数据。sincedb_path通常指向一个文件。这个文件保存上次操作的位置。设置为/dev/null表明我们不存储这个数据
# 在filter中，column都来自于csv表格。通常Logstash会把每个数据都当做是字符串。mileage是一个整型数，price_eur是一个浮点数，可以通过mutate来完成

Logstash处理数据用法示例---待完善

猜你喜欢