计算成交量例子，kafka/spark streaming/zk

package com.ws.streaming
import kafka.common.TopicAndPartition
import kafka.message.MessageAndMetadata
import kafka.serializer.StringDecoder
import kafka.utils.{ZKGroupTopicDirs, ZkUtils}
import org.I0Itec.zkclient.ZkClient
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.dstream.InputDStream
import org.apache.spark.streaming.kafka.{HasOffsetRanges, KafkaUtils, OffsetRange}
import org.apache.spark.streaming.{Duration, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
object OrderCount {

  def main(args: Array[String]): Unit = {

    val conf = new SparkConf().setAppName("OrderCount").setMaster("local[4]")

    val ssc = new StreamingContext(conf, Duration(5000))

    //读取ip规则
    val broadcast: Broadcast[Array[(Long, Long, String)]] = generalIpRules(ssc.sparkContext,"")

    //创建组
    val group = "group1"

    //指定消费者的topic主题
    val topic = "customerOrder"

    //指定kafka的broker地址
    val brokerList = "hadoop-01:9092,hadoop-02:9092,hadoop-03:9092"
    //指定zk地址,用来更新消费的偏移量时使用（也可以用redis，mysql）
    val zkQuorum = "hadoop-01:2181,hadoop-02:2181,hadoop-03:2181"

    //创建 stream 时使用的 topic 名字集合，SparkStreaming可同时消费多个topic
    val topics: Set[String] = Set(topic)

    //创建一个 ZKGroupTopicDirs 对象,其实是指定往zk中写入数据的目录，用于保存偏移量
    val topicDirs = new ZKGroupTopicDirs(group, topic)

    //获取 zookeeper 中的路径 "/g001/offsets/wordcount/"
    val zkTopicPath = s"$topicDirs.consumerOffsetDir"

    //准备kafka参数
    val kafkaParams = Map("metadata.broker.list" -> brokerList,
      "group.id" -> group,
      //"deserializer.encoding" -> "GB2312", //配置读取Kafka中数据的编码
      //设置从头开始读取
      "auto.offset.reset" -> kafka.api.OffsetRequest.SmallestTimeString
    )

    //zookeeper 的host 和 ip，创建一个 client,用于跟新偏移量量的
    //是zookeeper的客户端，可以从zk中读取偏移量数据，并更新偏移量
    val zkClient = new ZkClient(zkQuorum)
    val children = zkClient.countChildren(zkTopicPath)

    var kafkaStream: InputDStream[(String, String)] = null

    //如果 zookeeper 中有保存 offset，我们会利用这个 offset 作为 kafkaStream 的起始位置
    var fromOffsets: Map[TopicAndPartition, Long] = Map()

    //如果保存过 offset
    //注意：偏移量的查询是在Driver完成的
    if (children > 0) {
      for (i <- 0 until children) {
        // /g001/offsets/wordcount/0/10001

        // /g001/offsets/wordcount/0
        val partitionOffset = zkClient.readData[String](s"$zkTopicPath/${i}")
        // wordcount/0
        val tp = TopicAndPartition(topic, i)
        //将不同 partition 对应的 offset 增加到 fromOffsets 中
        // wordcount/0 -> 10001
        fromOffsets += (tp -> partitionOffset.toLong)
      }
      //Key: kafka的key   values: "hello tom hello jerry"
      //这个会将 kafka 的消息进行 transform，最终 kafak 的数据都会变成 (kafka的key, message) 这样的 tuple
      val messageHandler = (mmd: MessageAndMetadata[String, String]) => (mmd.key(), mmd.message())

      //通过KafkaUtils创建直连的DStream（fromOffsets参数的作用是:按照前面计算好了的偏移量继续消费数据）
      //[String, String, StringDecoder, StringDecoder,     (String, String)]
      //  key    value    key的解码方式   value的解码方式
      kafkaStream = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder, (String, String)](ssc, kafkaParams, fromOffsets, messageHandler)
    } else {
      //如果未保存，根据 kafkaParam 的配置使用最新(largest)或者最旧的（smallest） offset
      kafkaStream = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](ssc, kafkaParams, topics)
    }
    //偏移量的范围
    var offsetRanges = Array[OffsetRange]()

    //直连方式只有在KafkaDStream的RDD（KafkaRDD）中才能获取偏移量，那么就不能到调用DStream的Transformation
    //所以只能子在kafkaStream调用foreachRDD，获取RDD的偏移量，然后就是对RDD进行操作了
    //依次迭代KafkaDStream中的KafkaRDD
    //如果使用直连方式累加数据，那么就要在外部的数据库中进行累加（用KeyVlaue的内存数据库（NoSQL），Redis）
    //kafkaStream.foreachRDD里面的业务逻辑是在Driver端执行
    kafkaStream.foreachRDD(kafkaRdd=>{
      //判断当前的kafkaStream中的RDD是否有数据
      if(!kafkaRdd.isEmpty()){

      //获取偏移量，只有KafkaRDD可以强转成HasOffsetRanges，并获取到偏移量
      offsetRanges = kafkaRdd.asInstanceOf[HasOffsetRanges].offsetRanges

      val lineData = kafkaRdd.map(_._2)

      val fields: RDD[Array[String]] = lineData.map(_.split(" "))

      //计算总金额
      CalculateUtil.calculateTotalCount(fields)
      //计算商品分类金额
      CalculateUtil.calculateSortItem(fields)

      //计算每个地区金额
      val broadcastValue: Array[(Long, Long, String)] = broadcast.value

      CalculateUtil.calculateProvince(fields,broadcastValue)

      //更新偏移量
      for (o <- offsetRanges){
        val zkPath = s"${topicDirs.consumerOffsetDir}/${o.partition}"
        //将该 partition 的 offset 保存到 zookeeper
        //  /g001/offsets/wordcount/0/20000
        ZkUtils.updatePersistentPath(zkClient, zkPath, o.untilOffset.toString)
      }

      }

    })

    ssc.start()

    ssc.awaitTermination()

  }


  /**
    * 生成ip规则，并广播变量
    */
  def generalIpRules(sc : SparkContext, path : String): Broadcast[Array[(Long, Long, String)]] ={

    val pathData = sc.textFile(path)

    val rules: RDD[(Long, Long, String)] = pathData.map(lines => {
      val strArr = lines.split("[|]")
      val ipNum1 = strArr(2).toLong
      val ipNum2 = strArr(3).toLong
      val provnice = strArr(6)
      (ipNum1, ipNum2, provnice)
    })

    val rulesCollect = rules.collect()

    sc.broadcast(rulesCollect)
  }
}

package com.ws.streaming

import com.ws.spark.IpFromUtils
import org.apache.spark.rdd.RDD

object CalculateUtil {




  /**
    * 计算总金额
    */
  def calculateTotalCount(fields: RDD[Array[String]]): Unit = {

    if (!fields.isEmpty()) {

      val priceRdd: RDD[Double] = fields.map(arr => {
        val price = arr(arr.length - 1).toDouble
        price
      })

      //将价格累加
      val totalCount = priceRdd.reduce(_ + _)

      //获取redis连接
      val conn = RedisPool.getConnection()

      conn.incrByFloat("totalCount", totalCount)

      conn.close()
    }

  }

  /**
    * 计算商品分类金额
    */
  def calculateSortItem(fields: RDD[Array[String]]): Unit = {

    if (!fields.isEmpty()) {

      val itemAndPrice: RDD[(String, Double)] = fields.map(arr => {
        val item = arr(2)
        val price = arr(arr.length - 1).toDouble
        (item, price)
      })
      //聚合
      val itemCount: RDD[(String, Double)] = itemAndPrice.reduceByKey(_ + _)

      itemCount.foreachPartition(it => {
        //获取redis连接
        val conn = RedisPool.getConnection()

        it.foreach(f => {
          conn.incrByFloat(f._1, f._2)
        })

        conn.close()

      })
    }
  }

  /**
    * 计算地区总金额
    */
  def calculateProvince(fields: RDD[Array[String]], broadcastValue: Array[(Long, Long, String)]): Unit = {

    if (!fields.isEmpty()){

      val provinceAndPrice: RDD[(String, Double)] = fields.map(arr => {
        val ip = arr(1)
        val ipNum = IpFromUtils.ipToLong(ip)
        val provinceIndex = IpFromUtils.binarySearch(broadcastValue, ipNum)

        var province = "未知"

        if (-1 != provinceIndex) {
          province = broadcastValue(provinceIndex)._3
        }

        val price = arr(arr.length - 1).toDouble

        (province, price)
      })

      //将每个地区的金额累加写入redis
      val result = provinceAndPrice.reduceByKey(_+_)

      result.foreachPartition(f=>{
        val conn = RedisPool.getConnection()
        f.foreach(r=>{
          conn.incrByFloat(r._1,r._2)
        })
        conn.close()
      })
    }
  }
}

package com.ws.spark

import scala.io.Source
import scala.reflect.io.Path

/**
  * 查找ip归属地
  */
object IpFromUtils {
  /**
    * ip转换10进制
    *
    * @param ip
    * @return
    */
  def ipToLong(ip: String): Long = {

    val ipArr = ip.split("[.]")

    var ipNum = 0L;

    for (i <- ipArr) {
      ipNum = i.toLong | ipNum << 8
    }

    ipNum
  }

  /**
    * 生成规则
    *
    * @param filePath
    * @return
    */
  def rules(filePath: String): Array[(Long, Long, String)] = {
    val path = Source.fromFile(filePath.toString())

    val array: Array[(Long, Long, String)] = path.getLines().map(lines => {
      val strArr = lines.split("[|]")
      val ipNum1 = strArr(2).toLong
      val ipNum2 = strArr(3).toLong
      val provnice = strArr(6)
      (ipNum1, ipNum2, provnice)
    }).toArray

    array
  }

  def generalRules(line : String): (Long, Long, String)={
    val strArr = line.split("[|]")
    val ipStart = strArr(2).toLong
    val ipEnd = strArr(3).toLong
    val province = strArr(6)
    (ipStart, ipEnd, province)
  }

  /**
    * 二分法(前提有序)
    */
  def binarySearch(array: Array[(Long, Long, String)], num: Long): Int = {
    var end = array.length - 1;
    var begin = 0;

    while (begin <= end) {
      val middle = (begin + end) >> 1;


      if (num >= array(middle)._1 && num <= array(middle)._2) {
        return middle
      }

      if (num < array(middle)._1) {
        end = middle - 1
      }
      else {
        begin = middle + 1
      }

    }

    -1
  }
}

redis :

package com.ws.streaming
import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig}
object RedisPool {

  val config = new JedisPoolConfig()
  //最大空闲连接数
  config.setMaxIdle(5)
  //最大连接数
  config.setMaxTotal(20)

  val pool = new JedisPool(config,"192.168.127.12",6379,10000)

  def getConnection(): Jedis ={
    pool.getResource
  }
}