版权声明:本文为博主原创文章,遵循 CC 4.0 BY-SA 版权协议,转载请附上原文出处链接和本声明。
Flink 异步IO
在flink我们经常需要与外部系统打交道,由于外部系统的问题,可能导致时间耗时比较长,为了不影响flink的处理性能,flink引入了异步IO来处理这个问题
实现
需要 extends RichAsyncFunction<IN, OUT>
分析
AsyncDataStream 有两个重要的方法
返回的结果可能是乱序的
public static <IN, OUT> SingleOutputStreamOperator<OUT> unorderedWait(
DataStream<IN> in,
AsyncFunction<IN, OUT> func,
long timeout,
TimeUnit timeUnit,
int capacity //异常操作最大个数) {
return addOperator(in, func, timeUnit.toMillis(timeout), capacity, OutputMode.UNORDERED);
}
返回结果是有序的
public static <IN, OUT> SingleOutputStreamOperator<OUT> orderedWait(
DataStream<IN> in,
AsyncFunction<IN, OUT> func,
long timeout,
TimeUnit timeUnit,
int capacity) {
return addOperator(in, func, timeUnit.toMillis(timeout), capacity, OutputMode.ORDERED);
}
demo
public class FlinkAsIoTest {
public static void main(String[] args) {
StreamExecutionEnvironment en = StreamExecutionEnvironment.getExecutionEnvironment();
//自己实现的
KafkaUtil util = new KafkaUtil();
FlinkKafkaConsumer011 consumer1 = util.getConsumer("stream1", "test2");
DataStream<String> source = en.addSource(consumer1);
try {
source.print();
DataStream<String> result = AsyncDataStream.orderedWait(source, new AsIo(), 5000, TimeUnit.MILLISECONDS, 10);
result.print();
en.execute("test io");
} catch (Exception e) {
e.printStackTrace();
}
}
}
class AsIo extends RichAsyncFunction<String, String> {
@Override
public void asyncInvoke(String s, ResultFuture<String> resultFuture) throws Exception {
CompletableFuture.supplyAsync(new Supplier<String>() {
@Override
public String get() {
return queryResurlt(s);
}
}).thenAccept(new Consumer<String>() {
@Override
public void accept(String s) {
resultFuture.complete(Collections.singleton(s));
}
});
}
/**
* 假定这是一个异步操作
*
* @param params
* @return
*/
public String queryResurlt(String params) {
try {
//sleep 4s
Thread.sleep(4000);
} catch (InterruptedException e) {
e.printStackTrace();
}
if (params.equals("beijing")) {
return "this is beijing";
} else if (params.equals("chendou")) {
return "this is chendou";
} else {
return "this is other";
}
}
@Override
public void timeout(String input, ResultFuture<String> resultFuture) throws Exception {
}
}