Core code:
object TransformTest { def main(args: Array[String]): Unit = { val env = StreamExecutionEnvironment.getExecutionEnvironment env.setParallelism(1) val streamFromFile = env.readTextFile("C:\\Users\\Mi\\Documents\\project\\idea\\FlinkTitorial\\src\\main\\resources\\sensor.txt") //------------map-------------- val dataStream1: DataStream[SensorReading] = streamFromFile.map(d => { val arr = d.split(",") SensorReading(arr(0).trim, arr(1).trim.toLong, arr(2).toDouble) }) //------------keyBY-------------- //此时key是一个元组 val dataStream2: KeyedStream[SensorReading, Tuple] = dataStream1.keyBy(0) val dataStream3: KeyedStream [SensorReading, Tuple] = dataStream1.keyBy ("id") // key is the type of the field at this time val dataStream4: KeyedStream [SensorReading, String] = dataStream1.keyBy (_. id) // ------------ sum -------------- val dataStream5 = dataStream4.sum (2) val dataStream6 = dataStream4.sum ("temperature") // ------------ reduce ----------- // temperature summation. sr1 is the last result, sr2 is the current SensorReading object traversed val dataStream7 = dataStream4.reduce ((sr1, sr2) => SensorReading (sr2.id, sr2.timestamp, sr1.temperature + sr2.temperature)) dataStream7 .print env.execute ("transform test") } } case class SensorReading (id: String, timestamp: Long, temperature: Double)
sensor.txt file content:
sensor_1, 1547718199, 35.80018327300259 sensor_6, 1547718201, 15.402984393403084 sensor_7, 1547718202, 6.720945201171228 sensor_10, 1547718205, 38.101067604893444 sensor_1, 1547718200, 30.8 sensor_1, 1547718201, 40.8
Output result:
SensorReading(sensor_1,1547718199,35.80018327300259) SensorReading(sensor_6,1547718201,15.402984393403084) SensorReading(sensor_7,1547718202,6.720945201171228) SensorReading(sensor_10,1547718205,38.101067604893444) SensorReading(sensor_1,1547718200,66.60018327300259) SensorReading(sensor_1,1547718201,107.40018327300258)