canal DirectToKafka(采集数据到kafka、json解析、写入kafka)

canal DirectToKafka
环境的配置：
在mysql的配置文件中：/etc/my.cnf

linux>/etc/my.cnf 
[mysqld]
server-id=1
log-bin=mysql-bin	//binlog
binlog_format=row	//binlog形式row 一行数据
binlog-do-db=testdb //binlog 数据库testdb

在canal中中配置：
在canal.properties中的配置：

linux>vi conf/canal.properties 
canal.serverMode = kafka
canal.instance.parser.parallel = false
canal.destinations = example
canal.mq.servers = 192.168.58.201:9092,192.168.58.202:9092,192.168.58.203:9092

在example中的配置instance.properties

linux>vi conf/example/instance.properties
canal.instance.mysql.slaveId=21
#mysql的地址
canal.instance.master.address=192.168.58.203:3306
canal.instance.dbUsername=canal
canal.instance.dbPassword=canal

1.启动canal

linux>bin/startup.sh
注意环境应是java8

2.启动一个消费者

linux>bin/kafka-console-consumer.sh --topic example --from-beginning --bootstrap-server 192.168.58.201:9092,192.168.58.202:9092,192.168.58.203:9092

在mysql中使用insert 语句
如果有数据，说明环境搭建好了（kafka内）

编写spark streaming 程序：
1.使用direct模式对接kafka

 val conf = new SparkConf().setMaster("local[*]").setAppName(this.getClass.getName)
    val ssc = new StreamingContext(conf, Seconds(5))
    ssc.sparkContext.setLogLevel("error")

    val kafkaParams = Map[String, Object](
      "bootstrap.servers" -> "192.168.58.201:9092,192.168.58.202:9092,192.168.58.203:9092",
      "key.deserializer" -> classOf[StringDeserializer],
      "value.deserializer" -> classOf[StringDeserializer],
      "group.id" -> "canal-demo",
      "auto.offset.reset" -> "latest"
      //      "enable.auto.commit" -> (t: java.lang.Boolean)
    )
    val topics = Array("example")
    val stream = KafkaUtils.createDirectStream[String, String](
      ssc,
      PreferConsistent,
      Subscribe[String, String](topics, kafkaParams)
    )

使用 fastJson 解析，偏平化

 stream.map(_.value())
      .map(line=>{
    
    
        val jsonObject=JSON.parseObject(line)
        val table=jsonObject.getString("table")
        val types=jsonObject.getString("type")
//        val data=jsonObject.getJSONArray("data")
        val data = jsonObject.getJSONArray("data")
//        for( i <- 0 until jSONArray.size()){
    
    
                  val id=jSONArray.getJSONObject(i).getString("id")
                  val name=(jSONArray.getJSONObject(i).getString("name"))
                }
        (table,types,data)
      }).print()

transform 算子关联维度数据（黑名单的程序）

 filter.transform(rdd=>{
    
    
      //建立数据库的链接
      rdd.map(line=>{
    
    

      })
        //执行查询语句，关联维度数据
		（）
      })

json解析测试

package SparkTest.canal
import com.alibaba.fastjson.JSON
object JsonTest {
    
    
  //解析本地json 格式 日志分析
  //和TransformationDemo 关连
  def main(args: Array[String]): Unit = {
    
    
    // val nums = List(1, 2, 3, 4)
    val value = "{\"data\":[{\"id\":\"1\",\"name\":\"lisi\",\"score\":\"100\"},{\"id\":\"2\",\"name\":\"lisi\",\"score\":\"100\"},{\"id\":\"3\",\"name\":\"lisi\",\"score\":\"100\"}],\"database\":\"scott\",\"es\":1644282564000,\"id\":8,\"isDdl\":false,\"mysqlType\":{\"id\":\"int(11)\",\"name\":\"varchar(30)\",\"score\":\"int(11)\"},\"old\":[{\"name\":\"zhaoliu\"},{\"name\":\"zhaoliu\"},{\"name\":\"zhaoliu\"}],\"sql\":\"\",\"sqlType\":{\"id\":4,\"name\":12,\"score\":4},\"table\":\"student\",\"ts\":1644282564778,\"type\":\"UPDATE\"}"
    try {
    
    
      val jsonObject = JSON.parseObject(value)
      println(jsonObject)
      println(jsonObject.getString("table"))
      println(jsonObject.getString("type"))
      println(jsonObject.getJSONArray("data"))
      val jSONArray = jsonObject.getJSONArray("data")

      for (i <- 0 until jSONArray.size()) {
    
    
        println(jSONArray.getJSONObject(i).getString("id"))
        println(jSONArray.getJSONObject(i).getString("name"))
      }
      //      val students: = jSONArray.toJavaList[Student](Student.
      //      class)
      // println(students)
    } catch {
    
    
      case e: Exception => {
    
    
        e.printStackTrace()
      }
    }
  }
}

解析服务器上的 kafka日志分析，json 格式

package SparkTest.canal
import com.alibaba.fastjson.JSON
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.SparkConf
import org.apache.spark.streaming.{
    
    Seconds, StreamingContext}
import org.apache.spark.streaming.kafka010.ConsumerStrategies.Subscribe
import org.apache.spark.streaming.kafka010.KafkaUtils
import org.apache.spark.streaming.kafka010.LocationStrategies.PreferConsistent
//解析服务器上的 kafka日志分析，json 格式
object TransformationDemo {
    
    
  def main(args: Array[String]): Unit = {
    
    
    val conf = new SparkConf().setMaster("local[*]").setAppName(this.getClass.getName)
    val ssc = new StreamingContext(conf, Seconds(5))
    ssc.sparkContext.setLogLevel("error")

    val kafkaParams = Map[String, Object](
      "bootstrap.servers" -> "192.168.58.201:9092,192.168.58.202:9092,192.168.58.203:9092",
      "key.deserializer" -> classOf[StringDeserializer],
      "value.deserializer" -> classOf[StringDeserializer],
      "group.id" -> "canal-demo",
      "auto.offset.reset" -> "latest"
      //      "enable.auto.commit" -> (t: java.lang.Boolean)  //自动提交
    )
    
    val topics = Array("example")
    val stream = KafkaUtils.createDirectStream[String, String](
      ssc,
      PreferConsistent,
      Subscribe[String, String](topics, kafkaParams)
    )
    /**
     * 从topic中取出的数是key，value格式，一般情况只处理value
     */
     val filter = stream.map(_.value())
    stream.map(_.value())
      .map(line => {
    
    
        val jsonObject = JSON.parseObject(line)
        val table = jsonObject.getString("table")
        val types = jsonObject.getString("type")
        //        val data=jsonObject.getJSONArray("data")
        val data = jsonObject.getJSONArray("data")
        /*              for( i <- 0 until jSONArray.size()){
              //          val id=jSONArray.getJSONObject(i).getString("id")
              //          val name=(jSONArray.getJSONObject(i).getString("name"))
              //        }*/
        (table, types, data) //取值 json扁平化
      }).print()
        filter.transform(rdd=>{
    
    
          //建立数据库的链接
          rdd.map(line=>{
    
    
          })
            //执行查询语句，关联维度数据
          })
    ssc.start()
    ssc.awaitTermination()
  }
}

将数据写入kafka
1，收集到Driver端，构造一个producer 发送数据
2，在每个executor端构造一个producer 发送数据
需要在driver发送到executor 需要序列化 lazy val producer

1.定义类KafkaSink：
2.在driver端：构造一个广播变量，广播到每个executor

 val kafkaProducer: Broadcast[KafkaSink[String, String]] = {
    
    
      val kafkaProducerConfig = {
    
    
        val p = new Properties()
        p.setProperty("bootstrap.servers", "192.168.58.201:9092,192.168.58.202:9092,192.168.58.203:9092")
        p.setProperty("key.serializer", classOf[StringSerializer].getName)
        p.setProperty("value.serializer", classOf[StringSerializer].getName)
        p
      }
//      log.warn("kafka producer init done!")
      ssc.sparkContext.broadcast(KafkaSink[String, String](kafkaProducerConfig))
    }

3.在executor端使用：
	 stream.foreachRDD {
    
     rdd =>
      if (!rdd.isEmpty) {
    
    
        rdd.foreach(record => {
    
    
          kafkaProducer.value.send("example", record.key(),record.value())
          // do something else
        })
      }

1.启动一个生产者，生产数据 wordcount

linux>bin/kafka-console-producer.sh  --broker-list 192.168.58.201:9092,192.168.58.202:9092,192.168.58.203:9092 --topic wordcount

运行SparkStreamingKafkaDemo后随便输入数据
2.启动一个消费者，example

linux>bin/kafka-console-consumer.sh  --topic example --from-beginning --bootstrap-server 192.168.58.201:9092,192.168.58.202:9092,192.168.58.203:9092

DirectToKafka

package SparkTest.canal
import java.util.Properties
//6-4 最终
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.common.serialization.{
    
    StringDeserializer, StringSerializer}
import org.apache.spark.SparkConf
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.streaming.{
    
    Durations, StreamingContext}
import org.apache.spark.streaming.dstream.{
    
    DStream, InputDStream}
import org.apache.spark.streaming.kafka010.ConsumerStrategies.Subscribe
import org.apache.spark.streaming.kafka010.{
    
    CanCommitOffsets, HasOffsetRanges, KafkaUtils, OffsetRange}
import org.apache.spark.streaming.kafka010.LocationStrategies.PreferConsistent

object SparkStreamingKafkaDemo {
    
    
  def main(args: Array[String]): Unit = {
    
    
    val conf = new SparkConf()
    conf.setMaster("local[*]")
    conf.setAppName("submitoffset")
    val ssc = new StreamingContext(conf, Durations.seconds(5))
    //设置日志级别
    ssc.sparkContext.setLogLevel("Error")

    val kafkaProducer: Broadcast[KafkaSink[String, String]] = {
    
     //定义的广播变量
      val kafkaProducerConfig = {
    
    
        val p = new Properties()
        p.setProperty("bootstrap.servers", "192.168.58.201:9092,192.168.58.202:9092,192.168.58.203:9092")
        p.setProperty("key.serializer", classOf[StringSerializer].getName)
        p.setProperty("value.serializer", classOf[StringSerializer].getName)
        p
      }
      //      log.warn("kafka producer init done!")
      ssc.sparkContext.broadcast(KafkaSink[String, String](kafkaProducerConfig))  //调用方法做广播变量（List）
    }

    val kafkaParams = Map[String, Object](
      "bootstrap.servers" -> "192.168.58.201:9092,192.168.58.202:9092,192.168.58.203:9092",
      "key.deserializer" -> classOf[StringDeserializer],
      "value.deserializer" -> classOf[StringDeserializer],
      "group.id" -> "test", //

      /**
       * 当没有初始的offset，或者当前的offset不存在，如何处理数据
       * earliest ：自动重置偏移量为最小偏移量
       * latest：自动重置偏移量为最大偏移量【默认】
       * none:没有找到以前的offset,抛出异常
       */
      "auto.offset.reset" -> "earliest",
      /**
       * 当设置 enable.auto.commit为false时，不会自动向kafka中保存消费者offset.需要异步的处理完数据之后手动提交
       */
      "enable.auto.commit" -> (true: java.lang.Boolean) //默认是true 每5s提交一次
    )
    //    从wordcount读出数据，写入example：
    //      1.启动一个生产者，生产数据 wordcount
    //      2.启动一个消费者，example
    val topics = Array("wordcount")
    // 原始的stream 中包含了 offset的信息
    val stream: InputDStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream[String, String](
      ssc,
      PreferConsistent,
      Subscribe[String, String](topics, kafkaParams)
    )
    // 获取 record 中 value
    //    val valueStream: DStream[String] = stream.map(record => {
    
    
    //
    //      println("receive message key = " + record.key)
    //      println("receive message value = " + record.value)
    //      record.value
    //    })
    //    val words: DStream[String] = valueStream.flatMap(line => {
    
    
    //      line.split(" ")
    //    })
    //    val result: DStream[(String, Int)] = words.map((_, 1)).reduceByKey(_ + _)
    //    result.print()

    /**
     * 以上业务处理完成之后，异步的提交消费者offset,这里将 enable.auto.commit 设置成false,就是使用kafka 自己来管理消费者offset
     * 注意这里，获取 offsetRanges: Array[OffsetRange] 每一批次topic 中的offset时，必须从 源头读取过来的 stream中获取，不能从经过stream转换之后的DStream中获取。
     */
    //    stream.map(_.value()).transform(rdd=>{
    
    
    //      rdd
    //    })
    stream.foreachRDD {
    
     rdd =>
      if (!rdd.isEmpty) {
    
    
        rdd.foreach(record => {
    
    
          //    从wordcount读出数据，写入example：
          //    1.启动一个生产者，生产数据 wordcount
          //    2.启动一个消费者，example
          kafkaProducer.value.send("example", record.key(),record.value())
          // do something else
        })
      }
      //      val offsetRanges: Array[OffsetRange] = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
      //      // some time later, after outputs have completed
      //      // 业务处理完，收动提交offset
      //      stream.asInstanceOf[CanCommitOffsets].commitAsync(offsetRanges)
    }
    ssc.start()
    ssc.awaitTermination()
    ssc.stop()
  }
}

canal DirectToKafka(采集数据到kafka、json解析、写入kafka)

猜你喜欢