Flink示例——Sink
其他
2020-03-04 10:51:24
阅读次数: 0
Flink示例——Sink
版本信息
产品
版本
Flink
1.7.2
Java
1.8.0_231
Scala
2.11.12
Mavan依赖
自定义SourceFunction
提供一个SourceFunction,方便后面测试public class CustomSourceFunction extends RichSourceFunction < Tuple2< String, Long> > {
private boolean flag = true ;
@Override
public void run ( SourceContext< Tuple2< String, Long> > ctx) throws Exception {
List< String> data = Arrays. asList ( "a" , "b" , "c" , "d" , "e" , "f" , "g" ) ;
Random random = new Random ( ) ;
while ( flag) {
Thread. sleep ( 100 ) ;
String key = data. get ( random. nextInt ( data. size ( ) ) ) ;
long value = System. currentTimeMillis ( ) ;
ctx. collect ( Tuple2. of ( key, value) ) ;
}
}
@Override
public void cancel ( ) {
flag = false ;
}
}
Print Sink 示例
代码 PrintSinkDemopublic class PrintSinkDemo {
public static void main ( String[ ] args) {
StreamExecutionEnvironment env = StreamExecutionEnvironment. getExecutionEnvironment ( ) ;
CustomSourceFunction sourceFunction = new CustomSourceFunction ( ) ;
DataStreamSource< Tuple2< String, Long> > customDS = env. addSource ( sourceFunction) ;
PrintSinkFunction< Tuple2< String, Long> > sinkFunction = new PrintSinkFunction < > ( "stream" , false ) ;
customDS. addSink ( sinkFunction) ;
try {
env. execute ( ) ;
} catch ( Exception e) {
e. printStackTrace ( ) ;
}
}
}
Kafka Sink 示例
Maven导包 pom.xml< dependency>
< groupId> org.apache.flink</ groupId>
< artifactId> flink-connector-kafka-0.11_2.11</ artifactId>
< version> ${flink.version}</ version>
</ dependency>
代码 KafkaSinkDemopublic class KafkaSinkDemo {
public static void main ( String[ ] args) {
StreamExecutionEnvironment env = StreamExecutionEnvironment. getExecutionEnvironment ( ) ;
CustomSourceFunction sourceFunction = new CustomSourceFunction ( ) ;
DataStreamSource< Tuple2< String, Long> > customDS = env. addSource ( sourceFunction) ;
DataStream< String> resultDS = customDS. map ( new MapFunction < Tuple2< String, Long> , String> ( ) {
@Override
public String map ( Tuple2< String, Long> value) throws Exception {
return value. f0 + "|" + value. f1;
}
} ) ;
FlinkKafkaProducer011< String> kafkaProducer011 = generateKafkaProducer ( ) ;
resultDS. addSink ( kafkaProducer011) ;
try {
env. execute ( ) ;
} catch ( Exception e) {
e. printStackTrace ( ) ;
}
}
private static FlinkKafkaProducer011< String> generateKafkaProducer ( ) {
FlinkKafkaProducer011< String> kafkaProducer011 = new FlinkKafkaProducer011 < > (
"192.168.0.101:9092" , "topic_01" , new SimpleStringSchema ( )
) ;
return kafkaProducer011;
}
}
两阶段提交,保证exactly-once
Properties props = new Properties ( ) ;
props. setProperty ( "bootstrap.servers" , "192.168.0.101:9092" ) ;
props. setProperty ( "transaction.timeout.ms" , "900000" ) ;
FlinkKafkaProducer011< String> kafkaProducer011 = new FlinkKafkaProducer011 < > (
"topic_01" ,
new SimpleStringSchema ( ) ,
props,
FlinkKafkaProducer011. Semantic. EXACTLY_ONCE
) ;
ElasticSearch Sink 示例
Maven导包 pom.xml< dependency>
< groupId> org.apache.flink</ groupId>
< artifactId> flink-connector-elasticsearch6_2.11</ artifactId>
< version> ${flink.version}</ version>
</ dependency>
代码 ESSinkDemopublic class ESSinkDemo {
public static void main ( String[ ] args) {
StreamExecutionEnvironment env = StreamExecutionEnvironment. getExecutionEnvironment ( ) ;
CustomSourceFunction sourceFunction = new CustomSourceFunction ( ) ;
DataStreamSource< Tuple2< String, Long> > customDS = env. addSource ( sourceFunction) ;
ElasticsearchSink< Tuple2< String, Long> > esSink = generateESSink ( ) ;
customDS. addSink ( esSink) ;
try {
env. execute ( ) ;
} catch ( Exception e) {
e. printStackTrace ( ) ;
}
}
private static ElasticsearchSink< Tuple2< String, Long> > generateESSink ( ) {
List< HttpHost> httpHosts = Collections. singletonList (
new HttpHost ( "192.168.0.120" , 9200 )
) ;
ElasticsearchSinkFunction< Tuple2< String, Long> > sinkFunction = new ElasticsearchSinkFunction < Tuple2< String, Long> > ( ) {
@Override
public void process ( Tuple2< String, Long> tuple2, RuntimeContext runtimeContext, RequestIndexer requestIndexer) {
HashMap< String, String> map = new HashMap < > ( ) ;
map. put ( "content" , tuple2. f0) ;
map. put ( "eventTime" , tuple2. f1. toString ( ) ) ;
map. put ( "processTime" , String. valueOf ( System. currentTimeMillis ( ) ) ) ;
IndexRequest request = Requests. indexRequest ( )
. index ( "my_index" )
. type ( "my_data" )
. source ( map) ;
requestIndexer. add ( request) ;
}
} ;
ElasticsearchSink< Tuple2< String, Long> > esSink = new ElasticsearchSink. Builder < > (
httpHosts, sinkFunction
) . build ( ) ;
return esSink;
}
}
Redis Sink 示例
Maven导包 pom.xml< dependency>
< groupId> org.apache.bahir</ groupId>
< artifactId> flink-connector-redis_2.11</ artifactId>
< version> 1.0</ version>
</ dependency>
代码 RedisSinkDemopublic class RedisSinkDemo {
public static void main ( String[ ] args) {
StreamExecutionEnvironment env = StreamExecutionEnvironment. getExecutionEnvironment ( ) ;
CustomSourceFunction sourceFunction = new CustomSourceFunction ( ) ;
DataStreamSource< Tuple2< String, Long> > customDS = env. addSource ( sourceFunction) ;
RedisSink< Tuple2< String, Long> > redisSink = generateRedisSink ( ) ;
customDS. addSink ( redisSink) ;
try {
env. execute ( ) ;
} catch ( Exception e) {
e. printStackTrace ( ) ;
}
}
private static RedisSink< Tuple2< String, Long> > generateRedisSink ( ) {
FlinkJedisPoolConfig config = new FlinkJedisPoolConfig. Builder ( )
. setMaxTotal ( 8 )
. setMaxIdle ( 4 )
. setMinIdle ( 2 )
. setHost ( "192.168.0.110" )
. setPort ( 6379 )
. build ( ) ;
RedisMapper< Tuple2< String, Long> > redisMapper = new RedisMapper < Tuple2< String, Long> > ( ) {
@Override
public RedisCommandDescription getCommandDescription ( ) {
return new RedisCommandDescription (
RedisCommand. HSET,
"my_hash"
) ;
}
@Override
public String getKeyFromData ( Tuple2< String, Long> tuple2) {
return tuple2. f0;
}
@Override
public String getValueFromData ( Tuple2< String, Long> tuple2) {
return tuple2. f1. toString ( ) ;
}
} ;
return new RedisSink < > ( config, redisMapper) ;
}
}
自定义 Sink 示例 (JDBC)
Maven导包 pom.xml< dependency>
< groupId> mysql</ groupId>
< artifactId> mysql-connector-java</ artifactId>
< version> 5.1.38</ version>
</ dependency>
代码 JDBCUtilspublic class JDBCUtils {
static {
try {
Class. forName ( "com.mysql.jdbc.Driver" ) ;
} catch ( ClassNotFoundException e) {
e. printStackTrace ( ) ;
}
}
public static Connection getConnection ( ) {
String url = "jdbc:mysql://192.168.0.130:3306/test_db" ;
String user = "user_name" ;
String password = "123456" ;
Connection conn = null;
try {
conn = DriverManager. getConnection ( url, user, password) ;
} catch ( SQLException e) {
e. printStackTrace ( ) ;
}
if ( conn == null) {
throw new RuntimeException ( "JDBC创建连接失败!" ) ;
}
return conn;
}
}
代码 JDBCSinkFunctionpublic class JDBCSinkFunction extends RichSinkFunction < Tuple2< String, Long> > {
private Connection conn = null;
private PreparedStatement pst;
@Override
public void open ( Configuration parameters) throws Exception {
conn = JDBCUtils. getConnection ( ) ;
pst = conn. prepareStatement (
"INSERT INTO tb_name (content, event_time, update_time) VALUES (?, ?, ?)"
) ;
}
@Override
public void close ( ) throws Exception {
conn. close ( ) ;
}
@Override
public void invoke ( Tuple2< String, Long> value, Context context) throws Exception {
pst. setString ( 1 , value. f0) ;
pst. setLong ( 2 , value. f1) ;
pst. setLong ( 3 , System. currentTimeMillis ( ) ) ;
pst. execute ( ) ;
}
}
代码 CustomSinkDemopublic class CustomSinkDemo {
public static void main ( String[ ] args) {
StreamExecutionEnvironment env = StreamExecutionEnvironment. getExecutionEnvironment ( ) ;
CustomSourceFunction sourceFunction = new CustomSourceFunction ( ) ;
DataStreamSource< Tuple2< String, Long> > dataDS = env. addSource ( sourceFunction) ;
JDBCSinkFunction jdbcSink = new JDBCSinkFunction ( ) ;
dataDS. addSink ( jdbcSink) ;
try {
env. execute ( ) ;
} catch ( Exception e) {
e. printStackTrace ( ) ;
}
}
}
发布了146 篇原创文章 ·
获赞 54 ·
访问量 17万+
转载自 blog.csdn.net/alionsss/article/details/104248083