1.Flink侧输入流–> ODS–>DWD
场景：Flink 接收kakfa 信息 通过 侧输入流--> ODS-->DWD
代码：
主类：
package com.wudl.flink.app.dwd;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.wudl.flink.utils.MyKafkaUtil;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;
/**
* @author wudl
* @date 2021-11-28 22:26:27
* @description 日志
*/
public class BaseLogApp {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
env.setParallelism(1);
String SourceTopic = "ods_base_log";
String groupId = "base_log_app_2021";
DataStreamSource<String> kafkaDs = env.addSource(MyKafkaUtil.getKafkaConsumer(SourceTopic, groupId));
//TODO 3.将每行数据转换为JSON对象
OutputTag<String> outputTag = new OutputTag<String>("Dirty") {
};
SingleOutputStreamOperator<JSONObject> jsonObjDS = kafkaDs.process(new ProcessFunction<String, JSONObject>() {
@Override
public void processElement(String value, Context ctx, Collector<JSONObject> out) throws Exception {
try {
JSONObject jsonObject = JSON.parseObject(value);
System.out.println("start-------"+jsonObject.get("start"));
out.collect(jsonObject);
} catch (Exception e) {
System.out.println(e.getMessage());
System.out.println("---------------------------------------------");
//发生异常,将数据写入侧输出流
e.printStackTrace();
ctx.output(outputTag, value);
}
}
});
// 打印脏数据
jsonObjDS.getSideOutput(outputTag).print(" 脏数据>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
jsonObjDS.print("-----------");
//TODO 4.新老用户校验 状态编程
SingleOutputStreamOperator<JSONObject> jsonObjWithNewFlagDS = jsonObjDS.keyBy(jsonObj -> jsonObj.getJSONObject("common").getString("mid"))
.map(new RichMapFunction<JSONObject, JSONObject>() {
private ValueState<String> valueState;
@Override
public void open(Configuration parameters) throws Exception {
valueState = getRuntimeContext().getState(new ValueStateDescriptor<String>("value-state", String.class));
}
@Override
public JSONObject map(JSONObject jsonObject) throws Exception {
String isNew = jsonObject.getJSONObject("common").getString("is_new");
if ("1".equals(isNew)) {
//获取状态数据
String state = valueState.value();
if (state != null) {
jsonObject.getJSONObject("common").put("is_new", "0");
} else {
valueState.update("1");
}
}
return jsonObject;
}
});
//TODO 5.分流 侧输出流 页面：主流 启动：侧输出流 曝光：侧输出流
OutputTag<String> startTag = new OutputTag<String>("start") {
};
OutputTag<String> displayTag = new OutputTag<String>("display") {
};
SingleOutputStreamOperator<String> pageDS = jsonObjWithNewFlagDS.process(new ProcessFunction<JSONObject, String>() {
@Override
public void processElement(JSONObject value, Context ctx, Collector<String> out) throws Exception {
//获取启动日志字段
String start = value.getString("start");
if (start != null && start.length() > 0) {
//将数据写入启动日志侧输出流
ctx.output(startTag, value.toJSONString());
} else {
//将数据写入页面日志主流
out.collect(value.toJSONString());
//取出数据中的曝光数据
JSONArray displays = value.getJSONArray("displays");
if (displays != null && displays.size() > 0) {
//获取页面ID
String pageId = value.getJSONObject("page").getString("page_id");
for (int i = 0; i < displays.size(); i++) {
JSONObject display = displays.getJSONObject(i);
//添加页面id
display.put("page_id", pageId);
//将输出写出到曝光侧输出流
ctx.output(displayTag, display.toJSONString());
}
}
}
}
});
//TODO 6.提取侧输出流
DataStream<String> startDS = pageDS.getSideOutput(startTag);
DataStream<String> displayDS = pageDS.getSideOutput(displayTag);
//TODO 7.将三个流进行打印并输出到对应的Kafka主题中
startDS.print("Start>>>>>>>>>>>");
pageDS.print("Page>>>>>>>>>>>");
displayDS.print("Display>>>>>>>>>>>>");
startDS.addSink(MyKafkaUtil.getKafkaProducer("dwd_start_log"));
pageDS.addSink(MyKafkaUtil.getKafkaProducer("dwd_page_log"));
displayDS.addSink(MyKafkaUtil.getKafkaProducer("dwd_display_log"));
//TODO 8.启动任务
env.execute("BaseLogApp");
}
}
kafka 工具类
package com.wudl.flink.utils;
import org.apache.calcite.util.Static;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
import org.apache.flink.streaming.connectors.kafka.KafkaSerializationSchema;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import java.util.Properties;
/**
* @ClassName : MyKafkaUtil
* @Description : kakfa 工具类
* @Author :wudl
* @Date: 2021-11-28 22:25:27
*/
public class MyKafkaUtil {
private static String brokers = "192.168.1.130:9092,192.168.1.128:9092,192.168.1.129:9092";
private static String default_topic = "DWD_DEFAULT_TOPIC";
public static FlinkKafkaProducer<String> getKafkaProducer(String topic) {
return new FlinkKafkaProducer<String>(brokers,
topic,
new SimpleStringSchema());
}
public static <T> FlinkKafkaProducer<T> getKafkaProducer(KafkaSerializationSchema<T> kafkaSerializationSchema) {
Properties properties = new Properties();
properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokers);
return new FlinkKafkaProducer<T>(default_topic,
kafkaSerializationSchema,
properties,
FlinkKafkaProducer.Semantic.EXACTLY_ONCE);
}
public static FlinkKafkaConsumer<String> getKafkaConsumer(String topic, String groupId) {
Properties properties = new Properties();
properties.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokers);
return new FlinkKafkaConsumer<String>(topic,
new SimpleStringSchema(),
properties);
}
/**
* 拼接kafka 相关属性到ddl
* @param topic
* @param groupId
* @return
*/
//拼接Kafka相关属性到DDL
public static String getKafkaDDL(String topic, String groupId) {
return " 'connector' = 'kafka', " +
" 'topic' = '" + topic + "'," +
" 'properties.bootstrap.servers' = '" + brokers + "', " +
" 'properties.group.id' = '" + groupId + "', " +
" 'format' = 'json', " +
" 'scan.startup.mode' = 'latest-offset' ";
}
}
kakfa 命令：
./kafka-console-producer.sh --broker-list 192.168.1.130:9092 --topic ods_base_log
./kafka-console-consumer.sh --bootstrap-server 192.168.1.130:9092 --from-beginning --topic dwd_start_log
./kafka-console-consumer.sh --bootstrap-server 192.168.1.130:9092 --from-beginning --topic dwd_page_log
./kafka-console-consumer.sh --bootstrap-server 192.168.1.130:9092 --from-beginning --topic dwd_display_log
控制台输出
"C:\Program Files\Java\jdk1.8.0_241\bin\java.exe" -agentlib:jdwp=transport=dt_socket,address=127.0.0.1:64700,suspend=y,server=n -Dvisualvm.id=31740293873700 -javaagent:C:\Users\Administrator\AppData\Local\JetBrains\IntelliJIdea2020.3\captureAgent\debugger-agent.jar -Dfile.encoding=UTF-8 -classpath "C:\Program Files\Java\jdk1.8.0_241\jre\lib\charsets.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\deploy.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\access-bridge-64.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\cldrdata.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\dnsns.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\jaccess.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\jfxrt.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\localedata.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\nashorn.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\sunec.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\sunjce_provider.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\sunmscapi.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\sunpkcs11.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\zipfs.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\javaws.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\jce.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\jfr.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\jfxswt.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\jsse.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\management-agent.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\plugin.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\resources.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\rt.jar;D:\ideaWorkSpace\learning\Flinklearning\Flink-gmall-realtime\flink-wudl-gmall-realtime\target\classes;D:\software\maven_jar\org\apache\flink\flink-java\1.12.0\flink-java-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-core\1.12.0\flink-core-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-annotations\1.12.0\flink-annotations-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-metrics-core\1.12.0\flink-metrics-core-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-shaded-asm-7\7.1-12.0\flink-shaded-asm-7-7.1-12.0.jar;D:\software\maven_jar\com\esotericsoftware\kryo\kryo\2.24.0\kryo-2.24.0.jar;D:\software\maven_jar\com\esotericsoftware\minlog\minlog\1.2\minlog-1.2.jar;D:\software\maven_jar\org\objenesis\objenesis\2.1\objenesis-2.1.jar;D:\software\maven_jar\org\apache\commons\commons-compress\1.20\commons-compress-1.20.jar;D:\software\maven_jar\org\apache\commons\commons-lang3\3.3.2\commons-lang3-3.3.2.jar;D:\software\maven_jar\org\apache\commons\commons-math3\3.5\commons-math3-3.5.jar;D:\software\maven_jar\com\google\code\findbugs\jsr305\1.3.9\jsr305-1.3.9.jar;D:\software\maven_jar\org\apache\flink\force-shading\1.12.0\force-shading-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-streaming-java_2.11\1.12.0\flink-streaming-java_2.11-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-file-sink-common\1.12.0\flink-file-sink-common-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-runtime_2.11\1.12.0\flink-runtime_2.11-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-queryable-state-client-java\1.12.0\flink-queryable-state-client-java-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-hadoop-fs\1.12.0\flink-hadoop-fs-1.12.0.jar;D:\software\maven_jar\commons-io\commons-io\2.7\commons-io-2.7.jar;D:\software\maven_jar\org\apache\flink\flink-shaded-netty\4.1.49.Final-12.0\flink-shaded-netty-4.1.49.Final-12.0.jar;D:\software\maven_jar\org\apache\flink\flink-shaded-jackson\2.10.1-12.0\flink-shaded-jackson-2.10.1-12.0.jar;D:\software\maven_jar\org\apache\flink\flink-shaded-zookeeper-3\3.4.14-12.0\flink-shaded-zookeeper-3-3.4.14-12.0.jar;D:\software\maven_jar\org\javassist\javassist\3.24.0-GA\javassist-3.24.0-GA.jar;D:\software\maven_jar\org\scala-lang\scala-library\2.11.12\scala-library-2.11.12.jar;D:\software\maven_jar\com\typesafe\akka\akka-actor_2.11\2.5.21\akka-actor_2.11-2.5.21.jar;D:\software\maven_jar\com\typesafe\config\1.3.3\config-1.3.3.jar;D:\software\maven_jar\org\scala-lang\modules\scala-java8-compat_2.11\0.7.0\scala-java8-compat_2.11-0.7.0.jar;D:\software\maven_jar\com\typesafe\akka\akka-stream_2.11\2.5.21\akka-stream_2.11-2.5.21.jar;D:\software\maven_jar\org\reactivestreams\reactive-streams\1.0.2\reactive-streams-1.0.2.jar;D:\software\maven_jar\com\typesafe\ssl-config-core_2.11\0.3.7\ssl-config-core_2.11-0.3.7.jar;D:\software\maven_jar\com\typesafe\akka\akka-protobuf_2.11\2.5.21\akka-protobuf_2.11-2.5.21.jar;D:\software\maven_jar\com\typesafe\akka\akka-slf4j_2.11\2.5.21\akka-slf4j_2.11-2.5.21.jar;D:\software\maven_jar\org\clapper\grizzled-slf4j_2.11\1.3.2\grizzled-slf4j_2.11-1.3.2.jar;D:\software\maven_jar\com\github\scopt\scopt_2.11\3.5.0\scopt_2.11-3.5.0.jar;D:\software\maven_jar\org\xerial\snappy\snappy-java\1.1.4\snappy-java-1.1.4.jar;D:\software\maven_jar\com\twitter\chill_2.11\0.7.6\chill_2.11-0.7.6.jar;D:\software\maven_jar\com\twitter\chill-java\0.7.6\chill-java-0.7.6.jar;D:\software\maven_jar\org\lz4\lz4-java\1.6.0\lz4-java-1.6.0.jar;D:\software\maven_jar\org\apache\flink\flink-shaded-guava\18.0-12.0\flink-shaded-guava-18.0-12.0.jar;D:\software\maven_jar\org\apache\flink\flink-connector-kafka_2.11\1.12.0\flink-connector-kafka_2.11-1.12.0.jar;D:\software\maven_jar\org\apache\kafka\kafka-clients\2.4.1\kafka-clients-2.4.1.jar;D:\software\maven_jar\com\github\luben\zstd-jni\1.4.3-1\zstd-jni-1.4.3-1.jar;D:\software\maven_jar\org\apache\flink\flink-clients_2.11\1.12.0\flink-clients_2.11-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-optimizer_2.11\1.12.0\flink-optimizer_2.11-1.12.0.jar;D:\software\maven_jar\commons-cli\commons-cli\1.3.1\commons-cli-1.3.1.jar;D:\software\maven_jar\org\apache\flink\flink-cep_2.11\1.12.0\flink-cep_2.11-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-json\1.12.0\flink-json-1.12.0.jar;D:\software\maven_jar\com\alibaba\fastjson\1.2.78\fastjson-1.2.78.jar;D:\software\maven_jar\org\apache\hadoop\hadoop-client\3.1.3\hadoop-client-3.1.3.jar;D:\software\maven_jar\org\apache\hadoop\hadoop-common\3.1.3\hadoop-common-3.1.3.jar;D:\software\maven_jar\commons-codec\commons-codec\1.11\commons-codec-1.11.jar;D:\software\maven_jar\commons-net\commons-net\3.6\commons-net-3.6.jar;D:\software\maven_jar\org\eclipse\jetty\jetty-servlet\9.3.24.v20180605\jetty-servlet-9.3.24.v20180605.jar;D:\software\maven_jar\org\eclipse\jetty\jetty-webapp\9.3.24.v20180605\jetty-webapp-9.3.24.v20180605.jar;D:\software\maven_jar\org\eclipse\jetty\jetty-xml\9.3.24.v20180605\jetty-xml-9.3.24.v20180605.jar;D:\software\maven_jar\javax\servlet\jsp\jsp-api\2.1\jsp-api-2.1.jar;D:\software\maven_jar\com\sun\jersey\jersey-servlet\1.19\jersey-servlet-1.19.jar;D:\software\maven_jar\commons-lang\commons-lang\2.6\commons-lang-2.6.jar;D:\software\maven_jar\org\apache\commons\commons-configuration2\2.1.1\commons-configuration2-2.1.1.jar;D:\software\maven_jar\org\apache\avro\avro\1.7.7\avro-1.7.7.jar;D:\software\maven_jar\com\thoughtworks\paranamer\paranamer\2.3\paranamer-2.3.jar;D:\software\maven_jar\com\google\re2j\re2j\1.1\re2j-1.1.jar;D:\software\maven_jar\com\google\protobuf\protobuf-java\2.5.0\protobuf-java-2.5.0.jar;D:\software\maven_jar\com\google\code\gson\gson\2.2.4\gson-2.2.4.jar;D:\software\maven_jar\org\apache\hadoop\hadoop-auth\3.1.3\hadoop-auth-3.1.3.jar;D:\software\maven_jar\com\nimbusds\nimbus-jose-jwt\4.41.1\nimbus-jose-jwt-4.41.1.jar;D:\software\maven_jar\net\minidev\json-smart\2.3\json-smart-2.3.jar;D:\software\maven_jar\net\minidev\accessors-smart\1.2\accessors-smart-1.2.jar;D:\software\maven_jar\org\ow2\asm\asm\5.0.4\asm-5.0.4.jar;D:\software\maven_jar\org\apache\curator\curator-framework\2.13.0\curator-framework-2.13.0.jar;D:\software\maven_jar\org\apache\curator\curator-client\2.13.0\curator-client-2.13.0.jar;D:\software\maven_jar\org\apache\curator\curator-recipes\2.13.0\curator-recipes-2.13.0.jar;D:\software\maven_jar\org\apache\htrace\htrace-core4\4.1.0-incubating\htrace-core4-4.1.0-incubating.jar;D:\software\maven_jar\org\apache\kerby\kerb-simplekdc\1.0.1\kerb-simplekdc-1.0.1.jar;D:\software\maven_jar\org\apache\kerby\kerb-client\1.0.1\kerb-client-1.0.1.jar;D:\software\maven_jar\org\apache\kerby\kerby-config\1.0.1\kerby-config-1.0.1.jar;D:\software\maven_jar\org\apache\kerby\kerb-core\1.0.1\kerb-core-1.0.1.jar;D:\software\maven_jar\org\apache\kerby\kerby-pkix\1.0.1\kerby-pkix-1.0.1.jar;D:\software\maven_jar\org\apache\kerby\kerby-asn1\1.0.1\kerby-asn1-1.0.1.jar;D:\software\maven_jar\org\apache\kerby\kerby-util\1.0.1\kerby-util-1.0.1.jar;D:\software\maven_jar\org\apache\kerby\kerb-common\1.0.1\kerb-common-1.0.1.jar;D:\software\maven_jar\org\apache\kerby\kerb-crypto\1.0.1\kerb-crypto-1.0.1.jar;D:\software\maven_jar\org\apache\kerby\kerb-util\1.0.1\kerb-util-1.0.1.jar;D:\software\maven_jar\org\apache\kerby\token-provider\1.0.1\token-provider-1.0.1.jar;D:\software\maven_jar\org\apache\kerby\kerb-admin\1.0.1\kerb-admin-1.0.1.jar;D:\software\maven_jar\org\apache\kerby\kerb-server\1.0.1\kerb-server-1.0.1.jar;D:\software\maven_jar\org\apache\kerby\kerb-identity\1.0.1\kerb-identity-1.0.1.jar;D:\software\maven_jar\org\apache\kerby\kerby-xdr\1.0.1\kerby-xdr-1.0.1.jar;D:\software\maven_jar\com\fasterxml\jackson\core\jackson-databind\2.7.8\jackson-databind-2.7.8.jar;D:\software\maven_jar\com\fasterxml\jackson\core\jackson-core\2.7.8\jackson-core-2.7.8.jar;D:\software\maven_jar\org\codehaus\woodstox\stax2-api\3.1.4\stax2-api-3.1.4.jar;D:\software\maven_jar\com\fasterxml\woodstox\woodstox-core\5.0.3\woodstox-core-5.0.3.jar;D:\software\maven_jar\org\apache\hadoop\hadoop-hdfs-client\3.1.3\hadoop-hdfs-client-3.1.3.jar;D:\software\maven_jar\com\squareup\okhttp\okhttp\2.7.5\okhttp-2.7.5.jar;D:\software\maven_jar\com\squareup\okio\okio\1.6.0\okio-1.6.0.jar;D:\software\maven_jar\com\fasterxml\jackson\core\jackson-annotations\2.7.8\jackson-annotations-2.7.8.jar;D:\software\maven_jar\org\apache\hadoop\hadoop-yarn-api\3.1.3\hadoop-yarn-api-3.1.3.jar;D:\software\maven_jar\org\apache\hadoop\hadoop-yarn-client\3.1.3\hadoop-yarn-client-3.1.3.jar;D:\software\maven_jar\org\apache\hadoop\hadoop-mapreduce-client-core\3.1.3\hadoop-mapreduce-client-core-3.1.3.jar;D:\software\maven_jar\org\apache\hadoop\hadoop-yarn-common\3.1.3\hadoop-yarn-common-3.1.3.jar;D:\software\maven_jar\javax\servlet\javax.servlet-api\3.1.0\javax.servlet-api-3.1.0.jar;D:\software\maven_jar\org\eclipse\jetty\jetty-util\9.3.24.v20180605\jetty-util-9.3.24.v20180605.jar;D:\software\maven_jar\com\sun\jersey\jersey-core\1.19\jersey-core-1.19.jar;D:\software\maven_jar\javax\ws\rs\jsr311-api\1.1.1\jsr311-api-1.1.1.jar;D:\software\maven_jar\com\sun\jersey\jersey-client\1.19\jersey-client-1.19.jar;D:\software\maven_jar\com\fasterxml\jackson\module\jackson-module-jaxb-annotations\2.7.8\jackson-module-jaxb-annotations-2.7.8.jar;D:\software\maven_jar\com\fasterxml\jackson\jaxrs\jackson-jaxrs-json-provider\2.7.8\jackson-jaxrs-json-provider-2.7.8.jar;D:\software\maven_jar\com\fasterxml\jackson\jaxrs\jackson-jaxrs-base\2.7.8\jackson-jaxrs-base-2.7.8.jar;D:\software\maven_jar\org\apache\hadoop\hadoop-mapreduce-client-jobclient\3.1.3\hadoop-mapreduce-client-jobclient-3.1.3.jar;D:\software\maven_jar\org\apache\hadoop\hadoop-mapreduce-client-common\3.1.3\hadoop-mapreduce-client-common-3.1.3.jar;D:\software\maven_jar\org\apache\hadoop\hadoop-annotations\3.1.3\hadoop-annotations-3.1.3.jar;D:\software\maven_jar\mysql\mysql-connector-java\5.1.49\mysql-connector-java-5.1.49.jar;D:\software\maven_jar\com\alibaba\ververica\flink-connector-mysql-cdc\1.2.0\flink-connector-mysql-cdc-1.2.0.jar;D:\software\maven_jar\org\slf4j\slf4j-api\1.7.25\slf4j-api-1.7.25.jar;D:\software\maven_jar\org\slf4j\slf4j-log4j12\1.7.25\slf4j-log4j12-1.7.25.jar;D:\software\maven_jar\log4j\log4j\1.2.17\log4j-1.2.17.jar;D:\software\maven_jar\org\apache\logging\log4j\log4j-to-slf4j\2.14.0\log4j-to-slf4j-2.14.0.jar;D:\software\maven_jar\org\apache\logging\log4j\log4j-api\2.14.0\log4j-api-2.14.0.jar;D:\software\maven_jar\org\projectlombok\lombok\1.18.12\lombok-1.18.12.jar;D:\software\maven_jar\org\apache\flink\flink-connector-jdbc_2.11\1.12.0\flink-connector-jdbc_2.11-1.12.0.jar;D:\software\maven_jar\org\apache\phoenix\phoenix-spark\5.0.0-HBase-2.0\phoenix-spark-5.0.0-HBase-2.0.jar;D:\software\maven_jar\org\apache\phoenix\phoenix-core\5.0.0-HBase-2.0\phoenix-core-5.0.0-HBase-2.0.jar;D:\software\maven_jar\org\apache\tephra\tephra-api\0.14.0-incubating\tephra-api-0.14.0-incubating.jar;D:\software\maven_jar\org\apache\tephra\tephra-core\0.14.0-incubating\tephra-core-0.14.0-incubating.jar;D:\software\maven_jar\com\google\inject\guice\3.0\guice-3.0.jar;D:\software\maven_jar\javax\inject\javax.inject\1\javax.inject-1.jar;D:\software\maven_jar\aopalliance\aopalliance\1.0\aopalliance-1.0.jar;D:\software\maven_jar\com\google\inject\extensions\guice-assistedinject\3.0\guice-assistedinject-3.0.jar;D:\software\maven_jar\org\apache\thrift\libthrift\0.9.0\libthrift-0.9.0.jar;D:\software\maven_jar\it\unimi\dsi\fastutil\6.5.6\fastutil-6.5.6.jar;D:\software\maven_jar\org\apache\twill\twill-common\0.8.0\twill-common-0.8.0.jar;D:\software\maven_jar\org\apache\twill\twill-core\0.8.0\twill-core-0.8.0.jar;D:\software\maven_jar\org\apache\twill\twill-api\0.8.0\twill-api-0.8.0.jar;D:\software\maven_jar\org\ow2\asm\asm-all\5.0.2\asm-all-5.0.2.jar;D:\software\maven_jar\org\apache\twill\twill-discovery-api\0.8.0\twill-discovery-api-0.8.0.jar;D:\software\maven_jar\org\apache\twill\twill-discovery-core\0.8.0\twill-discovery-core-0.8.0.jar;D:\software\maven_jar\org\apache\twill\twill-zookeeper\0.8.0\twill-zookeeper-0.8.0.jar;D:\software\maven_jar\org\apache\tephra\tephra-hbase-compat-2.0\0.14.0-incubating\tephra-hbase-compat-2.0-0.14.0-incubating.jar;D:\software\maven_jar\org\antlr\antlr-runtime\3.5.2\antlr-runtime-3.5.2.jar;D:\software\maven_jar\jline\jline\2.11\jline-2.11.jar;D:\software\maven_jar\sqlline\sqlline\1.2.0\sqlline-1.2.0.jar;D:\software\maven_jar\joda-time\joda-time\1.6\joda-time-1.6.jar;D:\software\maven_jar\com\github\stephenc\findbugs\findbugs-annotations\1.3.9-1\findbugs-annotations-1.3.9-1.jar;D:\software\maven_jar\com\github\stephenc\jcip\jcip-annotations\1.0-1\jcip-annotations-1.0-1.jar;D:\software\maven_jar\org\codehaus\jackson\jackson-core-asl\1.9.2\jackson-core-asl-1.9.2.jar;D:\software\maven_jar\org\codehaus\jackson\jackson-mapper-asl\1.9.2\jackson-mapper-asl-1.9.2.jar;D:\software\maven_jar\junit\junit\4.12\junit-4.12.jar;D:\software\maven_jar\org\hamcrest\hamcrest-core\1.3\hamcrest-core-1.3.jar;D:\software\maven_jar\org\iq80\snappy\snappy\0.3\snappy-0.3.jar;D:\software\maven_jar\org\apache\htrace\htrace-core\3.1.0-incubating\htrace-core-3.1.0-incubating.jar;D:\software\maven_jar\org\apache\commons\commons-csv\1.0\commons-csv-1.0.jar;D:\software\maven_jar\org\apache\hbase\hbase-annotations\2.0.0\hbase-annotations-2.0.0.jar;D:\software\maven_jar\org\apache\hbase\hbase-mapreduce\2.0.0\hbase-mapreduce-2.0.0.jar;D:\software\maven_jar\org\apache\hbase\hbase-zookeeper\2.0.0\hbase-zookeeper-2.0.0.jar;D:\software\maven_jar\org\apache\hbase\hbase-metrics\2.0.0\hbase-metrics-2.0.0.jar;D:\software\maven_jar\org\apache\hbase\hbase-metrics-api\2.0.0\hbase-metrics-api-2.0.0.jar;D:\software\maven_jar\org\apache\hbase\hbase-replication\2.0.0\hbase-replication-2.0.0.jar;D:\software\maven_jar\org\apache\hadoop\hadoop-hdfs\2.7.4\hadoop-hdfs-2.7.4.jar;D:\software\maven_jar\org\mortbay\jetty\jetty\6.1.26\jetty-6.1.26.jar;D:\software\maven_jar\org\mortbay\jetty\jetty-util\6.1.26\jetty-util-6.1.26.jar;D:\software\maven_jar\com\sun\jersey\jersey-server\1.9\jersey-server-1.9.jar;D:\software\maven_jar\asm\asm\3.1\asm-3.1.jar;D:\software\maven_jar\commons-daemon\commons-daemon\1.0.13\commons-daemon-1.0.13.jar;D:\software\maven_jar\xmlenc\xmlenc\0.52\xmlenc-0.52.jar;D:\software\maven_jar\io\netty\netty-all\4.0.23.Final\netty-all-4.0.23.Final.jar;D:\software\maven_jar\org\fusesource\leveldbjni\leveldbjni-all\1.8\leveldbjni-all-1.8.jar;D:\software\maven_jar\org\apache\hbase\hbase-common\2.0.0\hbase-common-2.0.0.jar;D:\software\maven_jar\org\apache\hbase\hbase-protocol\2.0.0\hbase-protocol-2.0.0.jar;D:\software\maven_jar\org\apache\hbase\hbase-server\2.0.0\hbase-server-2.0.0.jar;D:\software\maven_jar\org\apache\hbase\hbase-http\2.0.0\hbase-http-2.0.0.jar;D:\software\maven_jar\org\eclipse\jetty\jetty-util-ajax\9.3.19.v20170502\jetty-util-ajax-9.3.19.v20170502.jar;D:\software\maven_jar\org\glassfish\jersey\core\jersey-server\2.25.1\jersey-server-2.25.1.jar;D:\software\maven_jar\org\glassfish\jersey\core\jersey-common\2.25.1\jersey-common-2.25.1.jar;D:\software\maven_jar\org\glassfish\jersey\bundles\repackaged\jersey-guava\2.25.1\jersey-guava-2.25.1.jar;D:\software\maven_jar\org\glassfish\hk2\osgi-resource-locator\1.0.1\osgi-resource-locator-1.0.1.jar;D:\software\maven_jar\org\glassfish\jersey\core\jersey-client\2.25.1\jersey-client-2.25.1.jar;D:\software\maven_jar\org\glassfish\jersey\media\jersey-media-jaxb\2.25.1\jersey-media-jaxb-2.25.1.jar;D:\software\maven_jar\javax\annotation\javax.annotation-api\1.2\javax.annotation-api-1.2.jar;D:\software\maven_jar\org\glassfish\hk2\hk2-api\2.5.0-b32\hk2-api-2.5.0-b32.jar;D:\software\maven_jar\org\glassfish\hk2\hk2-utils\2.5.0-b32\hk2-utils-2.5.0-b32.jar;D:\software\maven_jar\org\glassfish\hk2\external\aopalliance-repackaged\2.5.0-b32\aopalliance-repackaged-2.5.0-b32.jar;D:\software\maven_jar\org\glassfish\hk2\external\javax.inject\2.5.0-b32\javax.inject-2.5.0-b32.jar;D:\software\maven_jar\org\glassfish\hk2\hk2-locator\2.5.0-b32\hk2-locator-2.5.0-b32.jar;D:\software\maven_jar\javax\validation\validation-api\1.1.0.Final\validation-api-1.1.0.Final.jar;D:\software\maven_jar\org\glassfish\jersey\containers\jersey-container-servlet-core\2.25.1\jersey-container-servlet-core-2.25.1.jar;D:\software\maven_jar\org\apache\hbase\hbase-procedure\2.0.0\hbase-procedure-2.0.0.jar;D:\software\maven_jar\org\apache\hbase\hbase-common\2.0.0\hbase-common-2.0.0-tests.jar;D:\software\maven_jar\org\glassfish\web\javax.servlet.jsp\2.3.2\javax.servlet.jsp-2.3.2.jar;D:\software\maven_jar\javax\servlet\jsp\javax.servlet.jsp-api\2.3.1\javax.servlet.jsp-api-2.3.1.jar;D:\software\maven_jar\org\codehaus\jettison\jettison\1.3.8\jettison-1.3.8.jar;D:\software\maven_jar\org\jamon\jamon-runtime\2.4.1\jamon-runtime-2.4.1.jar;D:\software\maven_jar\javax\ws\rs\javax.ws.rs-api\2.0.1\javax.ws.rs-api-2.0.1.jar;D:\software\maven_jar\org\apache\hadoop\hadoop-distcp\2.7.4\hadoop-distcp-2.7.4.jar;D:\software\maven_jar\org\eclipse\jetty\jetty-http\9.3.19.v20170502\jetty-http-9.3.19.v20170502.jar;D:\software\maven_jar\org\eclipse\jetty\jetty-security\9.3.19.v20170502\jetty-security-9.3.19.v20170502.jar;D:\software\maven_jar\org\eclipse\jetty\jetty-server\9.3.19.v20170502\jetty-server-9.3.19.v20170502.jar;D:\software\maven_jar\org\eclipse\jetty\jetty-io\9.3.19.v20170502\jetty-io-9.3.19.v20170502.jar;D:\software\maven_jar\org\apache\hbase\hbase-hadoop-compat\2.0.0\hbase-hadoop-compat-2.0.0.jar;D:\software\maven_jar\org\apache\hbase\hbase-hadoop2-compat\2.0.0\hbase-hadoop2-compat-2.0.0.jar;D:\software\maven_jar\org\jruby\joni\joni\2.1.2\joni-2.1.2.jar;D:\software\maven_jar\com\clearspring\analytics\stream\2.9.5\stream-2.9.5.jar;D:\software\maven_jar\com\salesforce\i18n\i18n-util\1.0.4\i18n-util-1.0.4.jar;D:\software\maven_jar\com\ibm\icu\icu4j\60.2\icu4j-60.2.jar;D:\software\maven_jar\com\ibm\icu\icu4j-localespi\60.2\icu4j-localespi-60.2.jar;D:\software\maven_jar\com\ibm\icu\icu4j-charset\60.2\icu4j-charset-60.2.jar;D:\software\maven_jar\com\lmax\disruptor\3.3.6\disruptor-3.3.6.jar;D:\software\maven_jar\org\apache\hbase\hbase-client\2.0.0\hbase-client-2.0.0.jar;D:\software\maven_jar\org\apache\hbase\thirdparty\hbase-shaded-protobuf\2.1.0\hbase-shaded-protobuf-2.1.0.jar;D:\software\maven_jar\org\apache\hbase\hbase-protocol-shaded\2.0.0\hbase-protocol-shaded-2.0.0.jar;D:\software\maven_jar\org\apache\hbase\thirdparty\hbase-shaded-miscellaneous\2.1.0\hbase-shaded-miscellaneous-2.1.0.jar;D:\software\maven_jar\org\apache\hbase\thirdparty\hbase-shaded-netty\2.1.0\hbase-shaded-netty-2.1.0.jar;D:\software\maven_jar\org\apache\zookeeper\zookeeper\3.4.10\zookeeper-3.4.10.jar;D:\software\maven_jar\org\jruby\jcodings\jcodings\1.0.18\jcodings-1.0.18.jar;D:\software\maven_jar\io\dropwizard\metrics\metrics-core\3.2.1\metrics-core-3.2.1.jar;D:\software\maven_jar\org\apache\commons\commons-crypto\1.0.0\commons-crypto-1.0.0.jar;D:\software\maven_jar\org\apache\yetus\audience-annotations\0.5.0\audience-annotations-0.5.0.jar;D:\software\maven_jar\commons-beanutils\commons-beanutils\1.9.3\commons-beanutils-1.9.3.jar;D:\software\maven_jar\commons-logging\commons-logging\1.2\commons-logging-1.2.jar;D:\software\maven_jar\commons-collections\commons-collections\3.2.2\commons-collections-3.2.2.jar;D:\software\maven_jar\com\google\guava\guava\29.0-jre\guava-29.0-jre.jar;D:\software\maven_jar\com\google\guava\failureaccess\1.0.1\failureaccess-1.0.1.jar;D:\software\maven_jar\com\google\guava\listenablefuture\9999.0-empty-to-avoid-conflict-with-guava\listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar;D:\software\maven_jar\org\checkerframework\checker-qual\2.11.1\checker-qual-2.11.1.jar;D:\software\maven_jar\com\google\errorprone\error_prone_annotations\2.3.4\error_prone_annotations-2.3.4.jar;D:\software\maven_jar\com\google\j2objc\j2objc-annotations\1.3\j2objc-annotations-1.3.jar;D:\software\maven_jar\redis\clients\jedis\3.3.0\jedis-3.3.0.jar;D:\software\maven_jar\org\apache\commons\commons-pool2\2.6.2\commons-pool2-2.6.2.jar;D:\software\maven_jar\ru\yandex\clickhouse\clickhouse-jdbc\0.2.4\clickhouse-jdbc-0.2.4.jar;D:\software\maven_jar\org\apache\httpcomponents\httpclient\4.5.2\httpclient-4.5.2.jar;D:\software\maven_jar\org\apache\httpcomponents\httpcore\4.4.4\httpcore-4.4.4.jar;D:\software\maven_jar\org\apache\httpcomponents\httpmime\4.5.2\httpmime-4.5.2.jar;D:\software\maven_jar\net\jpountz\lz4\lz4\1.3.0\lz4-1.3.0.jar;D:\software\maven_jar\javax\xml\bind\jaxb-api\2.3.0\jaxb-api-2.3.0.jar;D:\software\maven_jar\org\apache\flink\flink-table-api-java-bridge_2.11\1.12.0\flink-table-api-java-bridge_2.11-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-table-api-java\1.12.0\flink-table-api-java-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-table-planner-blink_2.11\1.12.0\flink-table-planner-blink_2.11-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-table-common\1.12.0\flink-table-common-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-connector-files\1.12.0\flink-connector-files-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-connector-base\1.12.0\flink-connector-base-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-table-api-scala_2.11\1.12.0\flink-table-api-scala_2.11-1.12.0.jar;D:\software\maven_jar\org\scala-lang\scala-reflect\2.11.12\scala-reflect-2.11.12.jar;D:\software\maven_jar\org\scala-lang\scala-compiler\2.11.12\scala-compiler-2.11.12.jar;D:\software\maven_jar\org\scala-lang\modules\scala-xml_2.11\1.0.5\scala-xml_2.11-1.0.5.jar;D:\software\maven_jar\org\scala-lang\modules\scala-parser-combinators_2.11\1.0.4\scala-parser-combinators_2.11-1.0.4.jar;D:\software\maven_jar\org\apache\flink\flink-table-api-scala-bridge_2.11\1.12.0\flink-table-api-scala-bridge_2.11-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-scala_2.11\1.12.0\flink-scala_2.11-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-streaming-scala_2.11\1.12.0\flink-streaming-scala_2.11-1.12.0.jar;D:\software\maven_jar\org\apache\flink\flink-table-runtime-blink_2.11\1.12.0\flink-table-runtime-blink_2.11-1.12.0.jar;D:\software\maven_jar\org\codehaus\janino\janino\3.0.11\janino-3.0.11.jar;D:\software\maven_jar\org\codehaus\janino\commons-compiler\3.0.11\commons-compiler-3.0.11.jar;D:\software\maven_jar\org\apache\calcite\avatica\avatica-core\1.17.0\avatica-core-1.17.0.jar;D:\software\maven_jar\org\reflections\reflections\0.9.10\reflections-0.9.10.jar;D:\software\maven_jar\com\janeluo\ikanalyzer\2012_u6\ikanalyzer-2012_u6.jar;D:\software\maven_jar\org\apache\lucene\lucene-core\4.7.2\lucene-core-4.7.2.jar;D:\software\maven_jar\org\apache\lucene\lucene-queryparser\4.7.2\lucene-queryparser-4.7.2.jar;D:\software\maven_jar\org\apache\lucene\lucene-queries\4.7.2\lucene-queries-4.7.2.jar;D:\software\maven_jar\org\apache\lucene\lucene-sandbox\4.7.2\lucene-sandbox-4.7.2.jar;D:\software\maven_jar\org\apache\lucene\lucene-analyzers-common\4.7.2\lucene-analyzers-common-4.7.2.jar;D:\Program Files\JetBrains\IntelliJ IDEA 2020.3.4\lib\idea_rt.jar" com.wudl.flink.app.dwd.BaseLogApp
Connected to the target VM, address: '127.0.0.1:64700', transport: 'socket'
start-------null
-----------> {
"common":{
"ar":"440000","uid":"47","os":"iOS 13.3.1","ch":"Appstore","is_new":"0","md":"iPhone Xs","mid":"mid_4","vc":"v2.1.134","ba":"iPhone"},"page":{
"page_id":"home","during_time":6574},"displays":[{
"display_type":"activity","item":"2","item_type":"activity_id","pos_id":4,"order":1},{
"display_type":"query","item":"3","item_type":"sku_id","pos_id":2,"order":2},{
"display_type":"query","item":"10","item_type":"sku_id","pos_id":1,"order":3},{
"display_type":"query","item":"2","item_type":"sku_id","pos_id":3,"order":4},{
"display_type":"promotion","item":"5","item_type":"sku_id","pos_id":2,"order":5},{
"display_type":"query","item":"9","item_type":"sku_id","pos_id":2,"order":6}],"ts":1633620181000}
Page>>>>>>>>>>>> {
"common":{
"ar":"440000","uid":"47","os":"iOS 13.3.1","ch":"Appstore","is_new":"0","md":"iPhone Xs","mid":"mid_4","vc":"v2.1.134","ba":"iPhone"},"page":{
"page_id":"home","during_time":6574},"displays":[{
"display_type":"activity","item":"2","item_type":"activity_id","pos_id":4,"order":1},{
"display_type":"query","item":"3","item_type":"sku_id","pos_id":2,"order":2},{
"display_type":"query","item":"10","item_type":"sku_id","pos_id":1,"order":3},{
"display_type":"query","item":"2","item_type":"sku_id","pos_id":3,"order":4},{
"display_type":"promotion","item":"5","item_type":"sku_id","pos_id":2,"order":5},{
"display_type":"query","item":"9","item_type":"sku_id","pos_id":2,"order":6}],"ts":1633620181000}
Display>>>>>>>>>>>>> {
"display_type":"activity","page_id":"home","item":"2","item_type":"activity_id","pos_id":4,"order":1}
Display>>>>>>>>>>>>> {
"display_type":"query","page_id":"home","item":"3","item_type":"sku_id","pos_id":2,"order":2}
Display>>>>>>>>>>>>> {
"display_type":"query","page_id":"home","item":"10","item_type":"sku_id","pos_id":1,"order":3}
Display>>>>>>>>>>>>> {
"display_type":"query","page_id":"home","item":"2","item_type":"sku_id","pos_id":3,"order":4}
Display>>>>>>>>>>>>> {
"display_type":"promotion","page_id":"home","item":"5","item_type":"sku_id","pos_id":2,"order":5}
Display>>>>>>>>>>>>> {
"display_type":"query","page_id":"home","item":"9","item_type":"sku_id","pos_id":2,"order":6}
