Flink 1.11 读取kafka注册为表

先上代码:

import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.TableResult;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.types.Row;

/**
 * @program: flink-tech
 * @description: 1.11情况下的sql
 * @author: Mr.Wang
 * @create: 2020-07-13 09:41
 **/
public class TableDemo {
    /**
     * 官网API网址:https://ci.apache.org/projects/flink/flink-docs-release-1.11/zh/dev/table/connectors/kafka.html#how-to-create-a-kafka-table
     *
     *所有参数:
     * connector
     * topic
     * properties.bootstrap.servers
     * properties.group.id
     * format
     *  scan.startup.mode
     *  scan.startup.specific-offsets
     *  scan.startup.timestamp-millis
     *  sink.partitioner
     *
     */
    private static final String KAFKA_SQL = "CREATE TABLE kafkaTable (\n" +
            " code STRING," +
            " total_emp INT" +
            ") WITH (" +
            " 'connector' = 'kafka'," +
            " 'topic' = 'flink_dwd_test1'," +
            " 'properties.bootstrap.servers' = 'local:9092'," +
            " 'properties.group.id' = 'test1'," +
            " 'format' = 'json'," +
            " 'scan.startup.mode' = 'earliest-offset'" +
            ")";



    public static void main(String[] args) throws Exception {


        //bink table
        StreamExecutionEnvironment bsEnv = StreamExecutionEnvironment.getExecutionEnvironment();
        EnvironmentSettings bsSettings = EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build();
        StreamTableEnvironment bsTableEnv = StreamTableEnvironment.create(bsEnv, bsSettings);

        TableResult tableResult = bsTableEnv.executeSql(KAFKA_SQL);

        Table table = bsTableEnv.sqlQuery("select * from kafkaTable");
        DataStream<Row> dsRow = bsTableEnv.toAppendStream(table, Row.class);
        dsRow.print();
//        tableResult.print();
//        bsTableEnv.execute("aaa");
        bsEnv.execute("aa");

    }
}

pom依赖:

这里我要多说一点,我开始的时候安装官网的依赖导入执行一直报错,在官网群问过很多次,后来还是通过白斩鸡确认之后,基础代码没问题,是依赖问题,自动上次误删代码之后,我的本地代码各种问题,即使我把各种依赖注释删除,只剩官网的依赖,运行代码还是报错:

Exception in thread "main" java.util.concurrent.ExecutionException: org.apache.flink.runtime.client.JobExecutionException: Job execution failed.     at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357)     at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1895)     at org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1719)     at org.apache.flink.streaming.api.environment.LocalStreamEnvironment.execute(LocalStreamEnvironment.java:74)     at org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1699)     at flinksql.TableDemo.main(TableDemo.java:45) Caused by: org.apache.flink.runtime.client.JobExecutionException: Job execution failed.     at org.apache.flink.runtime.jobmaster.JobResult.toJobExecutionResult(JobResult.java:147)     at org.apache.flink.client.program.PerJobMiniClusterFactory$PerJobMiniClusterJobClient.lambda$getJobExecutionResult$2(PerJobMiniClusterFactory.java:186)     at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:602)     at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577)     at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)     at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962)     at org.apache.flink.runtime.rpc.akka.AkkaInvocationHandler.lambda$invokeRpc$0(AkkaInvocationHandler.java:229)     at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:760)     at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:736)     at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)     at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962)     at org.apache.flink.runtime.concurrent.FutureUtils$1.onComplete(FutureUtils.java:892)     at akka.dispatch.OnComplete.internal(Future.scala:264)     at akka.dispatch.OnComplete.internal(Future.scala:261)     at akka.dispatch.japi$CallbackBridge.apply(Future.scala:191)     at akka.dispatch.japi$CallbackBridge.apply(Future.scala:188)     at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:36)     at org.apache.flink.runtime.concurrent.Executors$DirectExecutionContext.execute(Executors.java:74)     at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:44)     at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:252)     at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:572)     at akka.pattern.PipeToSupport$PipeableFuture$$anonfun$pipeTo$1.applyOrElse(PipeToSupport.scala:22)     at akka.pattern.PipeToSupport$PipeableFuture$$anonfun$pipeTo$1.applyOrElse(PipeToSupport.scala:21)     at scala.concurrent.Future$$anonfun$andThen$1.apply(Future.scala:436)     at scala.concurrent.Future$$anonfun$andThen$1.apply(Future.scala:435)     at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:36)     at akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:55)     at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply$mcV$sp(BatchingExecutor.scala:91)     at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply(BatchingExecutor.scala:91)     at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply(BatchingExecutor.scala:91)     at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72)     at akka.dispatch.BatchingExecutor$BlockableBatch.run(BatchingExecutor.scala:90)     at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:40)     at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:44)     at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)     at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)     at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)     at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Caused by: org.apache.flink.runtime.JobException: Recovery is suppressed by NoRestartBackoffTimeStrategy     at org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.handleFailure(ExecutionFailureHandler.java:116)     at org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.getFailureHandlingResult(ExecutionFailureHandler.java:78)     at org.apache.flink.runtime.scheduler.DefaultScheduler.handleTaskFailure(DefaultScheduler.java:192)     at org.apache.flink.runtime.scheduler.DefaultScheduler.maybeHandleTaskFailure(DefaultScheduler.java:185)     at org.apache.flink.runtime.scheduler.DefaultScheduler.updateTaskExecutionStateInternal(DefaultScheduler.java:179)     at org.apache.flink.runtime.scheduler.SchedulerBase.updateTaskExecutionState(SchedulerBase.java:503)     at org.apache.flink.runtime.jobmaster.JobMaster.updateTaskExecutionState(JobMaster.java:386)     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)     at java.lang.reflect.Method.invoke(Method.java:498)     at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcInvocation(AkkaRpcActor.java:284)     at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:199)     at org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:74)     at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:152)     at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:26)     at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:21)     at scala.PartialFunction$class.applyOrElse(PartialFunction.scala:123)     at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:21)     at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:170)     at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)     at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)     at akka.actor.Actor$class.aroundReceive(Actor.scala:517)     at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:225)     at akka.actor.ActorCell.receiveMessage(ActorCell.scala:592)     at akka.actor.ActorCell.invoke(ActorCell.scala:561)     at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:258)     at akka.dispatch.Mailbox.run(Mailbox.scala:225)     at akka.dispatch.Mailbox.exec(Mailbox.scala:235)     ... 4 more Caused by: java.lang.AbstractMethodError: org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase.createFetcher(Lorg/apache/flink/streaming/api/functions/source/SourceFunction$SourceContext;Ljava/util/Map;Lorg/apache/flink/util/SerializedValue;Lorg/apache/flink/streaming/api/operators/StreamingRuntimeContext;Lorg/apache/flink/streaming/connectors/kafka/config/OffsetCommitMode;Lorg/apache/flink/metrics/MetricGroup;Z)Lorg/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher;     at org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase.run(FlinkKafkaConsumerBase.java:736)     at org.apache.flink.streaming.api.operators.StreamSource.run(StreamSource.java:100)     at org.apache.flink.streaming.api.operators.StreamSource.run(StreamSource.java:63)     at org.apache.flink.streaming.runtime.tasks.SourceStreamTask$LegacySourceFunctionThread.run(SourceStreamTask.java:201)

好了,上最基础的pom依赖:

<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-json</artifactId>
    <version>${flink.version}</version>
    <!--<scope>provided</scope>-->
</dependency>


<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-core</artifactId>
    <version>${flink.version}</version>
    <!--<scope>provided</scope>-->
</dependency>
<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-table-api-java-bridge_${scala.binary.version}</artifactId>
    <version>${flink.version}</version>
</dependency>
<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-table-api-scala-bridge_${scala.binary.version}</artifactId>
    <version>${flink.version}</version>
</dependency>
<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-table-planner_${scala.binary.version}</artifactId>
    <version>${flink.version}</version>
</dependency>
<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-table-planner-blink_${scala.binary.version}</artifactId>
    <version>${flink.version}</version>
</dependency>

<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-table-common</artifactId>
    <version>1.11.0</version>
    <scope>provided</scope>
</dependency>



<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
    <version>${flink.version}</version>
</dependency>

<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-clients_${scala.binary.version}</artifactId>
    <version>${flink.version}</version>
</dependency>



<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-connector-kafka_2.11</artifactId>
    <version>1.11.0</version>
    <scope>provided</scope>
</dependency>

最后思考:如果不是官网代码问题,而是自己本地环境依赖有问题怎么解决:

1,代码找别人执行,对比验证

扫描二维码关注公众号,回复: 11477352 查看本文章

2,新建项目,导入官网基础依赖

3,使用zeppelin在web端测试代码,这样子很靠谱。

4,清空libraries所有依赖

右键项目 maven reimport:

希望对大家有用。 

猜你喜欢

转载自blog.csdn.net/qq_31866793/article/details/107312914