flink按照processTime处理流

1、日志类型

offset = 8336399570, key = 10865354946, value = 14:36:51,465[INFO][resin-port-8573-90242 controller.StatController.init(311)]  - uid=7143aba7e035a5af544849dfaded0753|pid=864314032460198|sver=7.5.7|plat=6|partner=281|pn=OPPO+R9sk|sysver=6.0.1|code=0|isnet=5|net=|poid=1|res=1080*1920|newuser=0|ip=117.136.32.86|ua=com.sohu.sohuvideo/7005007 (Linux; U; Android 6.0.1; zh_CN; OPPO R9sk; Build/MMB29M)|sim=1|gentype=2|tkey=b4f0621dd6c9e5ac8989936aad848634ec1c7704|androidId=null|imei=864314032460198|timer=null|xid=null|sendtime=1568011011465|mfo=OPPO|coreswitch=1|appswitch=null|url=1002|abmod=null|memo=%7B%22sdcard_permission%22%3A%221%22%7D|method=post|
offset = 8336399570, key = 10865354946, value = 14:36:51,465[INFO][resin-port-8573-90242 controller.StatController.init(311)]  - uid=7143aba7e035a5af544849dfaded0753|pid=864314032460198|sver=7.5.7|plat=6|partner=281|pn=OPPO+R9sk|sysver=6.0.1|code=0|isnet=5|net=|poid=1|res=1080*1920|newuser=0|ip=117.136.32.86|ua=com.sohu.sohuvideo/7005007 (Linux; U; Android 6.0.1; zh_CN; OPPO R9sk; Build/MMB29M)|sim=1|gentype=2|tkey=b4f0621dd6c9e5ac8989936aad848634ec1c7704|androidId=null|imei=864314032460198|timer=null|xid=null|sendtime=1568011011465|mfo=OPPO|coreswitch=1|appswitch=null|url=1002|abmod=null|memo=%7B%22sdcard_permission%22%3A%221%22%7D|method=post|
offset = 8336399570, key = 10865354946, value = 14:36:51,465[INFO][resin-port-8573-90242 controller.StatController.init(311)]  - uid=7143aba7e035a5af544849dfaded0753|pid=864314032460198|sver=7.5.7|plat=6|partner=281|pn=OPPO+R9sk|sysver=6.0.1|code=0|isnet=5|net=|poid=1|res=1080*1920|newuser=0|ip=117.136.32.86|ua=com.sohu.sohuvideo/7005007 (Linux; U; Android 6.0.1; zh_CN; OPPO R9sk; Build/MMB29M)|sim=1|gentype=2|tkey=b4f0621dd6c9e5ac8989936aad848634ec1c7704|androidId=null|imei=864314032460198|timer=null|xid=null|sendtime=1568011011465|mfo=OPPO|coreswitch=1|appswitch=null|url=1002|abmod=null|memo=%7B%22sdcard_permission%22%3A%221%22%7D|method=post|
offset = 8336399570, key = 10865354946, value = 14:36:51,465[INFO][resin-port-8573-90242 controller.StatController.init(311)]  - uid=7143aba7e035a5af544849dfaded0753|pid=864314032460198|sver=7.5.7|plat=6|partner=281|pn=OPPO+R9sk|sysver=6.0.1|code=0|isnet=5|net=|poid=1|res=1080*1920|newuser=0|ip=117.136.32.86|ua=com.sohu.sohuvideo/7005007 (Linux; U; Android 6.0.1; zh_CN; OPPO R9sk; Build/MMB29M)|sim=1|gentype=2|tkey=b4f0621dd6c9e5ac8989936aad848634ec1c7704|androidId=null|imei=864314032460198|timer=null|xid=null|sendtime=1568011011465|mfo=OPPO|coreswitch=1|appswitch=null|url=1002|abmod=null|memo=%7B%22sdcard_permission%22%3A%221%22%7D|method=post|
offset = 8336399570, key = 10865354946, value = 14:36:51,465[INFO][resin-port-8573-90242 controller.StatController.init(311)]  - uid=7143aba7e035a5af544849dfaded0753|pid=864314032460198|sver=7.5.7|plat=6|partner=281|pn=OPPO+R9sk|sysver=6.0.1|code=0|isnet=5|net=|poid=1|res=1080*1920|newuser=0|ip=117.136.32.86|ua=com.sohu.sohuvideo/7005007 (Linux; U; Android 6.0.1; zh_CN; OPPO R9sk; Build/MMB29M)|sim=1|gentype=2|tkey=b4f0621dd6c9e5ac8989936aad848634ec1c7704|androidId=null|imei=864314032460198|timer=null|xid=null|sendtime=1568011011465|mfo=OPPO|coreswitch=1|appswitch=null|url=1002|abmod=null|memo=%7B%22sdcard_permission%22%3A%221%22%7D|method=post|

2、代码

import java.util.Properties
import java.util.regex.Pattern

import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
import org.apache.flink.streaming.api.windowing.time.Time
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer
import org.apache.flink.api.scala._


//1.	滚动窗口
//  Flink默认的时间窗口根据Processing Time 进行窗口的划分，将Flink获取到的数据根据进入Flink的时间划分到不同的窗口中。


object StreamingProcessTime {
  private val patternBd = Pattern.compile("^(.*),[0-9]{1,3}(.*)  - (.*)")

  def main(args: Array[String]): Unit = {

    val environment: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
    environment.setParallelism(1)
    environment.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime)


    val log = environment.socketTextStream("130.28.34.145",1111)
    val value = log.map(value => {
      val strings: Array[String] = value.split("\\|")
      var result = "--"
      if (strings(0).length > 15) {
        result = strings(0).substring(0, 10)
      }
      (result, 1)
    }).keyBy(0).
      timeWindow(Time.seconds(20)).
      sum(1).
      map(_._2.toString).print()
//      addSink(new SinkToMySQL)


    environment.execute()
  }
}

3、结果

每20s打印一次结果

5
10
20