kafka-->spark-->phoenix

其他 2019-01-30 15:47:59 阅读次数: 0

一、在IDEA新建一个maven项目：

1.[pom.xml]:

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>com</groupId>
  <artifactId>SparkPhoenixDemo</artifactId>
  <version>1.0-SNAPSHOT</version>
  <name>${project.artifactId}</name>
  <description>My wonderfull scala app</description>
  <inceptionYear>2010</inceptionYear>
  <licenses>
    <license>
      <name>My License</name>
      <url>http://....</url>
      <distribution>repo</distribution>
    </license>
  </licenses>
  <properties>
    <maven.compiler.source>1.6</maven.compiler.source>
    <maven.compiler.target>1.6</maven.compiler.target>
    <encoding>UTF-8</encoding>
    <scala.tools.version>2.11</scala.tools.version>
    <scala.version>2.11.12</scala.version>
  </properties>

  <dependencies>
    <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core -->
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-core</artifactId>
      <version>2.6.3</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind -->
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-databind</artifactId>
      <version>2.6.3</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations -->
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-annotations</artifactId>
      <version>2.6.3</version>
    </dependency>    <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core -->    <dependency>      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_2.11</artifactId>
      <version>2.3.0</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql -->
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_2.11</artifactId>
      <version>2.3.0</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka-clients -->
    <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka-clients</artifactId>
      <version>0.11.0.1</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming-kafka-0-10 -->
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming-kafka-0-8_2.11</artifactId>
      <version>2.0.0</version>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
      <version>2.3.0</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming -->
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming_2.11</artifactId>
      <version>2.3.0</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-graphx -->
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-graphx_2.11</artifactId>
      <version>2.3.0</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hbase</groupId>
      <artifactId>hbase-client</artifactId>
      <version>1.4.8</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hbase</groupId>
      <artifactId>hbase-server</artifactId>
      <version>1.4.8</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.hbase/hbase-common -->
    <dependency>
      <groupId>org.apache.hbase</groupId>
      <artifactId>hbase-common</artifactId>
      <version>1.4.8</version>
    </dependency>
    <!--<dependency>-->
      <!--<groupId>org.apache.hbase</groupId>-->
      <!--<artifactId>hbase-mapreduce</artifactId>-->
      <!--<version>2.0.0</version>-->
    <!--</dependency>-->
    <dependency>
      <groupId>mysql</groupId>
      <artifactId>mysql-connector-java</artifactId>
      <version>5.1.39</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.phoenix/phoenix-spark -->
    <dependency>
      <groupId>org.apache.phoenix</groupId>
      <artifactId>phoenix-spark</artifactId>
      <version>4.14.0-HBase-1.4</version>
    </dependency>
    <dependency>
      <groupId>com.lmax</groupId>
      <artifactId>disruptor</artifactId>
      <version>3.3.8</version>
    </dependency>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-library</artifactId>
      <version>${scala.version}</version>
    </dependency>
    <!-- flink  -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-scala_2.11</artifactId>
      <version>1.6.1</version>
    </dependency>
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-streaming-scala_2.11</artifactId>
      <version>1.6.1</version>
    </dependency>
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-clients_2.11</artifactId>
      <version>1.6.1</version>
    </dependency>
    <!-- spark-mllib  -->
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-mllib_2.11</artifactId>
      <version>2.3.0</version>
    </dependency>
    <dependency>
      <groupId>org.jblas</groupId>
      <artifactId>jblas</artifactId>
      <version>1.2.3</version>
    </dependency>
  </dependencies>
  <build>
    <sourceDirectory>src/main/scala</sourceDirectory>
    <testSourceDirectory>src/test/scala</testSourceDirectory>
    <plugins>
      <plugin>
        <!-- see http://davidb.github.com/scala-maven-plugin -->
        <groupId>net.alchim31.maven</groupId>
        <artifactId>scala-maven-plugin</artifactId>
        <version>3.1.3</version>
        <executions>
          <execution>
            <goals>
              <goal>compile</goal>
              <goal>testCompile</goal>
            </goals>
            <configuration>
              <args>
                <arg>-dependencyfile</arg>
                <arg>${project.build.directory}/.scala_dependencies</arg>
              </args>
            </configuration>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-surefire-plugin</artifactId>
        <version>2.13</version>
        <configuration>
          <useFile>false</useFile>
          <disableXmlReport>true</disableXmlReport>
          <!-- If you have classpath issue like NoDefClassError,... -->
          <!-- useManifestOnlyJar>false</useManifestOnlyJar -->
          <includes>
            <include>**/*Test.*</include>
            <include>**/*Suite.*</include>
          </includes>
        </configuration>
      </plugin>
    </plugins>
  </build>
</project>


二、新建一个类SparkKafkaconsumer：
import kafka.serializer.StringDecoder
import org.apache.spark.{SparkConf}
import org.apache.spark.streaming.{Duration, StreamingContext}
import org.apache.spark.streaming.kafka.KafkaUtils

object SparkKafkaconsumer {
  def main(args: Array[String]): Unit = {
 val sparkConf = new SparkConf()
 .setMaster("local[*]")
 .setAppName("SparkStreamingKafka_Direct")
 sparkConf.set("spark.streaming.backpressure.enabled","true")
 val ssc = new StreamingContext(sparkConf,Duration(5000))
 val topics=Set("test3") //我们需要消费的kafka数据的topic
 val kafkaParams=Map(
 "metadata.broker.list"->"slave1:9092,slave2:9092,slave3:9092",// kafka的broker list地址
 "group.id"->"Kafka_Direct"
 )

 //创建一个从kafka获取数据的流
 val messages = KafkaUtils.createDirectStream[String,String,StringDecoder,StringDecoder](ssc,kafkaParams,topics)

 // 取出value
 val lines = messages.map(_._2)

 lines.foreachRDD(
 rdd => {
 val data = rdd.collect
 data.foreach(record => {
 SparkConnectionScalaNew.runmain(record,ssc)
 })
 }
 )


 ssc.start()// 真正启动程序
 ssc.awaitTermination()//阻塞等待
 }
}


三、新建一个类：SparkConnectionScalaNew
import java.util.UUID

import org.apache.spark.streaming.StreamingContext

object SparkConnectionScalaNew {
  def runmain(keyValues:String,ssc:StreamingContext): Unit ={
    val uuid = UUID.randomUUID().toString
    SparkKafkaMessNew.messMain(keyValues,ssc,uuid)
  }
}


四、新建一个类：SparkKafkaMessNew
import java.text.SimpleDateFormat
import java.util.Date

import org.apache.spark.sql.SQLContext
import org.apache.spark.streaming.StreamingContext

import scala.util.parsing.json.JSON
import org.apache.phoenix.spark._

object SparkKafkaMessNew {

  def regJson(json:Option[Any]) = json match {
    case Some(map: Map[String, Any]) => map
  }


  def messMain(mess:String,ssc:StreamingContext,uuid:String): Unit ={
    val jsonS = JSON.parseFull(mess)
    val first = regJson(jsonS)

    println(mess)//{"hello":"aa"}
    println(first)//Map(hello -> aa)

    val res = first.get("hello").toString//Some(aa)
    val name = res.replace("Some(","").replace(")","").trim//aa

    val date = new Date()
    val df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
    val dateRe = df.format(date)



    val sc = ssc.sparkContext
    val sqlContext = new SQLContext(sc)

    val dataSet = List((uuid,name,dateRe))
    sc.parallelize(dataSet).saveToPhoenix(
      "TEST1.STUDENT", //表名
      Seq("ID","NAME","CREATETIME"),//字段名
      zkUrl = Some("slave1:2181")
    )
  }
}
 

猜你喜欢

转载自www.cnblogs.com/zzmmyy/p/10338261.html
今日推荐
《美国对全球网络空间安全与发展的威胁和破坏》报告发布
火速冲上 GitHub 热榜 —— 开源编程语言、框架哪有这么可爱？
北京人形机器人创新中心发布全球首个纯电驱拟人奔跑的全尺寸人形机器人“天工”
LFOSSA 源来如此公开课 | 掌握云原生未来：CNCF 认证全面攻略与备考秘籍
周排行
让自己的头脑极度开放
CentOS 6.5(x64) 和Redhat6.5操作系误删libc
高可用注册中心
【日记】12.28/【题解】AtCoder AGC041
XML（5）_XML 约束_DTD
Java集合Map（四）
树梅派安装桌面环境教程
pipenv 的 使用和安装
小程序白屏问题和内存研究
C语言简单选择排序
每日归档
更多
2024-05-02(0)
2024-05-01(4)
2024-04-30(1)
2024-04-29(40)
2024-04-28(0)
2024-04-27(56)
2024-04-26(39)
2024-04-25(22)
2024-04-24(36)
2024-04-23(26)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022