1、添加必要的maven依赖
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>2.11</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.11</artifactId>
<version>2.3.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql-kafka-0-10 -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql-kafka-0-10_2.11</artifactId>
<version>2.3.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.codehaus.janino/janino -->
<dependency>
<groupId>org.codehaus.janino</groupId>
<artifactId>janino</artifactId>
<version>3.0.8</version>
</dependency>
2、完整代码
import conf.{ConfManager, Constans}
import org.apache.log4j.{Level, Logger}
import org.apache.spark.sql.{DataFrame, SparkSession}
object ReadKafka {
def main(args: Array[String]): Unit = {
Logger.getLogger("org.apache.spark").setLevel(Level.ERROR)
Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.ERROR)
Logger.getLogger("org.apache.kafka.clients.consumer").setLevel(Level.ERROR)
var query:org.apache.spark.sql.streaming.StreamingQuery = null
val spark = SparkSession
.builder
.appName("StructuredStreaming")
.master("local[2]")
.getOrCreate()
val topic = "test"
val broker = "hadoop01:9092"
val df = spark
.readStream
.format("kafka")
.option("kafka.bootstrap.servers",broker)
.option("subscribe",topic)
.option("startingOffsets", "latest")
.option("max.poll.records", 100)
.load()
import spark.implicits._
val word = df.selectExpr("CAST(value AS STRING)").as[String].flatMap(_.split(" "))
val wordcount = word.groupBy("value").count()
val q = wordcount
.writeStream
.outputMode("complete")
.format("console")
.start()
q.awaitTermination()
}
}