前言:
在上一篇博客中,我们做了flume实时收集数据,然后发送到kafka的不同topic。每个topic会有对应的分区,需要将数据发送到topic不同的分区,这样数据的存储可以负载均衡,后续sparkstreaming实时消费kafka数据也能够有序(一个消费者只能消费一个topic的一个分区即有序消费)。
flume 实时收集数据到kafka 不同topic:
https://blog.csdn.net/hg_harvey/article/details/80852763
想要实现上述功能,flume自带的kafka sink不能满足我们的需要,需要自定义kafka sink,参考官网:http://flume.apache.org/FlumeDeveloperGuide.html#sink
新建一个maven工程,删除resource目录,pom.xml中添加flume和kafka依赖,完整pom.xml文件配置如下
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.sqyc.bigdata</groupId>
<artifactId>solar-kafka-sink</artifactId>
<version>1.0</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<kafka.client.version>0.11.0.0</kafka.client.version>
<flume.version>1.8.0</flume.version>
<okhttp.version>3.10.0</okhttp.version>
<gson.version>2.8.2</gson.version>
</properties>
<dependencies>
<!-- kafka client -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>${kafka.client.version}</version>
</dependency>
<!-- flume -->
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-core</artifactId>
<version>${flume.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-sdk</artifactId>
<version>${flume.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-configuration</artifactId>
<version>${flume.version}</version>
</dependency>
<!-- okhttp -->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>${okhttp.version}</version>
</dependency>
<!-- gson -->
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>${gson.version}</version>
</dependency>
</dependencies>
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
</plugins>
</pluginManagement>
</build>
</project>
代码
- kafka sink
package com.sqyc.bigdata.sink;
import com.sqyc.bigdata.exception.SolarException;
import com.sqyc.bigdata.model.LogModel;
import com.sqyc.bigdata.partitioner.SolarPartitioner;
import com.sqyc.bigdata.util.LogModelSerializer;
import org.apache.commons.lang.StringUtils;
import org.apache.flume.*;
import org.apache.flume.conf.Configurable;
import org.apache.flume.sink.AbstractSink;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
/**
* @description
* @auther huhanwei
* @created 2018/6/14 09:42
*/
public class SolarKafkaSink extends AbstractSink implements Configurable {
private static final int LOG_START_INDEX = 24;
private Logger logger = LoggerFactory.getLogger(this.getClass());
private String topicName;
private String kafkaServers;
/**
* 错误日志保存的路径
*/
private String errorLogPath;
/**
* 错误日志保存的文件名称
*/
private String errorLogFileName;
/**
* 告警错误发送人手机号码
*/
private List<String> warningMobileList;
private KafkaProducer<String, String> producer;
@Override
public Status process() throws EventDeliveryException {
Status status = null;
// Start transaction
Channel ch = getChannel();
Transaction txn = ch.getTransaction();
txn.begin();
String body = null;
try {
Event event = ch.take();
if(event != null){
body = new String(event.getBody(), "utf-8");
String log = body.substring(LOG_START_INDEX);
LogModel logModel = LogModelSerializer.deserializer(log);
String warehouseTableName = logModel.getWarehouseTableName();
logger.debug(
"headers" + event.getHeaders() + ",topicName = " + topicName + ",warehouseTableName = "
+ warehouseTableName);
// 发送数据kafka
this.send(warehouseTableName, LogModelSerializer.serializer(logModel));
}
status = Status.READY;
txn.commit();
} catch (Throwable t) {
logger.error("传递数据出现异常:" + body, t);
txn.rollback();
status = Status.BACKOFF;
if (t instanceof Error) {
throw (Error) t;
}
} finally {
if (txn != null) {
txn.close();
}
}
return status;
}
private static final String KAFKA_TOPIC_KEY = "kafka.topic";
private static final String KAFKA_BOOTSTRAP_SERVERS_KEY = "kafka.bootstrap.servers";
private static final String ERROR_LOG_PATH_KEY = "errorLogPath";
private static final String ERROR_LOG_FILE_NAME_KEY = "errorLogFileName";
private static final String WARNING_MOBILES_KEY = "warningMobiles";
private static final String COMMA = ",";
@Override
public void configure(Context context) {
topicName = context.getString(KAFKA_TOPIC_KEY);
if ((topicName == null) || (topicName.isEmpty())) {
throw new SolarException("SolarKafkaSink 需要配置" + KAFKA_TOPIC_KEY + "参数作为指定数据的存储的Topic");
}
kafkaServers = context.getString(KAFKA_BOOTSTRAP_SERVERS_KEY);
if ((kafkaServers == null) || (kafkaServers.isEmpty())) {
throw new SolarException("SolarKafkaSink 需要配置" + KAFKA_BOOTSTRAP_SERVERS_KEY +"参数作为Kafka服务器的地址");
}
this.errorLogPath = context.getString(ERROR_LOG_PATH_KEY);
if(StringUtils.isEmpty(this.errorLogPath)){
throw new SolarException("SolarKafkaSink 需要配置" + ERROR_LOG_PATH_KEY +"参数确认错误数据文件存储路径");
}
this.errorLogFileName = context.getString(ERROR_LOG_FILE_NAME_KEY);
if(StringUtils.isEmpty(this.errorLogFileName)){
throw new SolarException("SolarKafkaSink 需要配置" + ERROR_LOG_FILE_NAME_KEY + "参数确认错误数据存储路径的文件名称");
}
String warningMobiles = context.getString(WARNING_MOBILES_KEY);
if(StringUtils.isEmpty(warningMobiles)){
throw new SolarException("SolarKafkaSink 需要配置" + WARNING_MOBILES_KEY + "参数确认告警短信接收人");
}else{
try{
this.warningMobileList = Arrays.asList(warningMobiles.split(COMMA));
}catch(Exception e){
throw new SolarException("读取warningMobiles出现异常", e);
}
}
}
@Override
public synchronized void start() {
Properties kafkaProps = new Properties();
kafkaProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, this.kafkaServers);
kafkaProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,StringSerializer.class.getCanonicalName());
kafkaProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName());
kafkaProps.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, SolarPartitioner.class.getCanonicalName());
kafkaProps.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384);
kafkaProps.put(ProducerConfig.LINGER_MS_CONFIG, 100);
kafkaProps.put(ProducerConfig.ACKS_CONFIG, "1");
this.producer = new KafkaProducer<String, String>(kafkaProps);
super.start();
}
@Override
public synchronized void stop() {
try {
this.producer.close();
} catch (Exception e) {
}
super.stop();
}
private void send(String warehouseTableName, String msg) {
try {
ProducerRecord<String, String> record = new ProducerRecord<>(this.topicName,
warehouseTableName, msg);
this.producer.send(record, new SolarKafkaSinkCallback(this.warningMobileList, this.errorLogPath, this.errorLogFileName, msg));
} catch (Exception e) {
throw new SolarException(e);
}
}
}
- kafka callback
说明:回调类,用于记录发送失败的数据,offset,然后发送短信提示
package com.sqyc.bigdata.sink;
import com.sqyc.bigdata.util.SMSMessageSender;
import org.apache.commons.io.IOUtils;
import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedWriter;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.util.List;
/**
* @description 自定义 kafka Callback
* @auther huhanwei
*/
public class SolarKafkaSinkCallback implements Callback {
private Logger logger = LoggerFactory.getLogger(this.getClass());
private String errorLogPath;
private String errorLogFileName;
private List<String> waringMobileList;
private String message;
public SolarKafkaSinkCallback(List<String> waringMobileList, String errorLogPath, String errorLogFileName,
String message) {
this.waringMobileList = waringMobileList;
this.errorLogPath = errorLogPath;
this.errorLogFileName = errorLogFileName;
this.message = message;
}
@Override
public void onCompletion(RecordMetadata recordMetadata, Exception e) {
if(recordMetadata == null || e != null){
logger.error("发送消息到Kafka失败,消息内容为:" + this.message, e);
BufferedWriter writer = null;
try{
new File(this.errorLogPath).mkdirs();
File errorFile = new File(this.errorLogPath + File.separator + this.errorLogFileName);
if(errorFile.exists() == false){
errorFile.createNewFile();
}
writer = Files.newBufferedWriter(errorFile.toPath(), StandardOpenOption.APPEND);
writer.write(this.message);
writer.write(System.lineSeparator());
writer.flush();
}catch(Exception e1){
logger.error("记录发送到Kafaka失败的消息到本地文件出错,消息内容为:" + this.message, e1);
}finally{
new SMSMessageSender().send(this.waringMobileList, "发送数据到Kafka出现异常,日志数据已经写入Log文件");
IOUtils.closeQuietly(writer);
}
}else{
logger.debug("发送到offset=" + recordMetadata.offset());
}
}
}
- 自定义partitioner
package com.sqyc.bigdata.partitioner;
import com.sqyc.bigdata.exception.SolarException;
import org.apache.kafka.clients.producer.Partitioner;
import org.apache.kafka.common.Cluster;
import org.apache.kafka.common.PartitionInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
/**
* @description 自定义Partitioner(warehouseTableName的hash值模分区数量)
* @auther huhanwei
*/
public class SolarPartitioner implements Partitioner{
private Logger logger = LoggerFactory.getLogger(this.getClass());
private Map<String, Integer> partitionNumberMap = new HashMap<String, Integer>();
@Override
public int partition(String topicName, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
List<PartitionInfo> partitionList = cluster.availablePartitionsForTopic(topicName);
// partition排序,升序(默认倒序,不排序,在消费者段获取的分区号和数据存放的分区号不一致)
List<Integer> partitonNoList = new ArrayList<Integer>();
for(int i = 0, len = partitionList.size();i < len; i++){
partitonNoList.add(partitionList.get(i).partition());
}
Collections.sort(partitonNoList);
// 第一次初始化将topic对应的partition数量存放到map中
if(this.partitionNumberMap.containsKey(topicName) == false){
this.partitionNumberMap.put(topicName, partitonNoList.size());
}
// 判断partition数量是否发生变化,如果发生变化抛出异常
// 为什么需要判断partition的数量
// 分析1:canal组件实时同步mysql数据然后发送到kafka,最后存放到hdfs,load到hive,最终hive中的数据是有先后顺序的,如果数据乱序,那么仓库离线数据处理是没办法处理的
// 分析2:flume实时采集数据发送到kafka不同topic不同partition,后续sparkstreaming实时消费数据,一个消费者同一时刻只能同一个topic的partition消费,保证数据有序
// 如果此时某个topic宕掉,或者partition数量减少,那么上面离线和实时处理就会受到影响,但是partition的数量是可以增加的
if(this.partitionNumberMap.get(topicName) != partitionList.size()){
throw new SolarException("topic:" + topicName + "中的Partition数量发生变化");
}
if(partitionList.isEmpty()){
throw new SolarException("topic:" + topicName + "当前无可用的Partition");
}else{
String warehouseTableName = key.toString();
int hash = Math.abs(warehouseTableName.hashCode());
int index = hash % partitonNoList.size();
logger.debug("send data to kafka topic[" + topicName + "] partition[" + index + "]");
return partitonNoList.get(index);
}
}
@Override
public void close() {
}
@Override
public void configure(Map<String, ?> map) {
}
}
- 自定义异常类
package com.sqyc.bigdata.exception;
import org.apache.commons.lang.exception.NestableRuntimeException;
/**
* @description 公共异常类
* @author huhanwei
*/
public class SolarException extends NestableRuntimeException {
public SolarException(String errorCode) {
super(errorCode);
}
public SolarException(String errorCode, Throwable cause) {
super(errorCode, cause);
}
public SolarException(String errorCode, String errorDesc) {
super(errorCode + ":" + errorDesc);
}
public SolarException(String errorCode, String errorDesc, Throwable cause) {
super(errorCode + ":" + errorDesc, cause);
}
public SolarException(Throwable cause) {
super(cause);
}
}
- 模型类
LogModel.java
package com.sqyc.bigdata.model;
/**
* @description 日志模型父类
* @auther huhanwei
*/
public class LogModel {
protected static final String LOG_MODEL = "log";
private String warehouseTableName;
public String getWarehouseTableName() {
return warehouseTableName;
}
public void setWarehouseTableName(String warehouseTableName) {
this.warehouseTableName = warehouseTableName;
}
}
BizlogModel.java
package com.sqyc.bigdata.model;
/**
* @description Log日志类
* @auther wangtie
* @created 2018/5/28 17:57
*/
public class BizlogModel extends LogModel {
private String bizKey;
private String sign;
private String logStr;
private String logType = LOG_MODEL;
public String getBizKey() {
return bizKey;
}
public void setBizKey(String bizKey) {
this.bizKey = bizKey;
}
public String getLogStr() {
return logStr;
}
public void setLogStr(String logStr) {
this.logStr = logStr;
}
public String getSign() {
return sign;
}
public void setSign(String sign) {
this.sign = sign;
}
}
- 工具类
DataConvert.java
package com.sqyc.bigdata.util;
import org.apache.commons.codec.digest.DigestUtils;
/**
* Created by zhoulingjiang on 2018/6/13.
*/
public class DataConvert {
public static String createSign(String logStr){
return DigestUtils.md5Hex(logStr).toUpperCase();
}
public static String StrReplace(String str){
String tempStr=str.replaceAll("\\n|\\t","");
return tempStr;
}
}
LogModelSerializer.java
package com.sqyc.bigdata.util;
import com.google.gson.Gson;
import com.sqyc.bigdata.exception.SolarException;
import com.sqyc.bigdata.model.BizlogModel;
import com.sqyc.bigdata.model.LogModel;
/**
* @description 业务数据序列化工具类
* @author huhanwei
*/
public class LogModelSerializer {
public static String serializer(LogModel model){
Gson gson = new Gson();
if(model instanceof BizlogModel){
BizlogModel logModel = (BizlogModel)model;
String sign =DataConvert.createSign(logModel.getLogStr());
logModel.setSign(sign);
}
return gson.toJson(model);
}
private static final String LOG_MODEL_KEY = "\"logType\":\"log\"";
/**
* @param str
* @return
*/
public static LogModel deserializer(String str){
Gson g = new Gson();
if (str.contains(LOG_MODEL_KEY)) {
return g.fromJson(str, BizlogModel.class);
} else {
throw new SolarException("未知的数据类型,无法转换:" + str);
}
}
}
- 发送短信工具类
发送短信的工具类笔者这就不提供了,工具类中使用到的依赖是okhttp,短信发送需要网站上购买,封装一个方法,两个参数一个手机号列表一个错误告警内容,这个自己找度娘解决吧!
将上述代码打成jar,上传到flume的lib目录下,切记不要有第三方依赖,否则启动flume时会报找不到自定义的kafka sink类,如果有第三方依赖,把jar也上传到flume的lib目录下即可,笔者将flume下的kafka和kafka-client gson jar做了替换,并上传了自定义kafka sink 的jar
- 模拟日志
package com.sqyc.bigdata.log;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.text.SimpleDateFormat;
import java.util.Date;
/**
* @description 模拟生成日志
* @author huhanwei
*/
public class LogGenerate {
private static final Logger logger = LoggerFactory.getLogger(LogGenerate.class);
public static void main(String[] args) throws InterruptedException {
logger.info("----------------------------------------> log generate start...");
for (int i = 1; i<= 100000; i++) {
Thread.sleep(200);
System.out.println(getData());
}
logger.info("----------------------------------------> log generate end...");
}
private static String getData() {
String msg = "";
/* 分topic
int num = (int)(Math.random() * 3) + 1;
switch (num) {
case 1:
msg = getCurrDate() + " " + "{\"bizKey\":\"KCAELBDDCYOXSLGR\",\"sign\":\"F038492CC90210167E275977FFBE5A0C\"," +
"\"logStr\":\"{\\\"id\\\":\\\"1003392469380538370\\\",\\\"acode\\\":\\\"CLICK\\\",\\\"app\\\":\\\"4.3.4\\\",\\\"apprunid\\\":\\\"5DB78A8FE3AE_1528272457316\\\",\\\"ctime\\\":\\\"2018-06-25 13:32:17\\\",\\\"create_time\\\":\\\"2018-06-25 13:32:17\\\",\\\"nt\\\":\\\"WIFI\\\",\\\"platform\\\":\\\"0\\\",\\\"r\\\":\\\"\\\",\\\"stime\\\":\\\"2018-06-25 13:32:17\\\",\\\"phone\\\":\\\"18519087423\\\",\\\"p1\\\":\\\"SQHomePageViewController\\\",\\\"p2\\\":\\\"SQRecommendViewController\\\",\\\"p3\\\":\\\"\\\",\\\"android_id\\\":\\\"94e3fcc30e90edf2\\\",\\\"auid\\\":\\\"7da009b4b7878c179d65e20e4d1d589d\\\",\\\"idfa\\\":\\\"\\\",\\\"imei\\\":\\\"860797036717585\\\",\\\"imsi\\\":\\\"460000864499542\\\",\\\"install_id\\\":\\\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\\\",\\\"mac\\\":\\\"\\\",\\\"wmac\\\":\\\"90:ad:f7:86:28:f0\\\",\\\"pcode\\\":\\\"\\\",\\\"uid\\\":\\\"\\\",\\\"property\\\":\\\"22C25529510391525315227D\\\",\\\"ar\\\":\\\"\\\",\\\"at\\\":\\\"291366\\\",\\\"deviceid\\\":\\\"\\\",\\\"lat\\\":\\\"38.92499077690972\\\",\\\"lg\\\":\\\"121.65503987630208\\\",\\\"os\\\":\\\"Android\\\",\\\"os_ver\\\":\\\"7.0\\\",\\\"brand\\\":\\\"HUAWEI\\\",\\\"xh\\\":\\\"HUAWEI NXT-AL10\\\",\\\"coordinate\\\":\\\"GAODE\\\",\\\"city\\\":\\\"大连市\\\",\\\"trigger_man_phone\\\":\\\"18018990297\\\",\\\"app_code\\\":\\\"1002\\\",\\\"cookie\\\":\\\"\\\",\\\"serialno\\\":\\\"CJL5T16217005509\\\"}\"," +
"\"logType\":\"log\",\"warehouseTableName\":\"risk_tab\"}";
break;
case 2:
msg = getCurrDate() + " " + "{\"bizKey\":\"DXADXHZWEXTGCFQB\",\"sign\":\"F068492CC90210167E275977FFBE5A0C\"," +
"\"logStr\":\"{\\\"id\\\":\\\"1004392469380538370\\\",\\\"acode\\\":\\\"CLICK\\\",\\\"app\\\":\\\"4.3.4\\\",\\\"apprunid\\\":\\\"5DB78A8FE3AE_1528272457316\\\",\\\"ctime\\\":\\\"2018-06-25 13:32:17\\\",\\\"create_time\\\":\\\"2018-06-25 13:32:17\\\",\\\"nt\\\":\\\"WIFI\\\",\\\"platform\\\":\\\"0\\\",\\\"r\\\":\\\"\\\",\\\"stime\\\":\\\"2018-06-25 13:32:17\\\",\\\"phone\\\":\\\"18519087423\\\",\\\"p1\\\":\\\"SQHomePageViewController\\\",\\\"p2\\\":\\\"SQRecommendViewController\\\",\\\"p3\\\":\\\"\\\",\\\"android_id\\\":\\\"94e3fcc30e90edf2\\\",\\\"auid\\\":\\\"7da009b4b7878c179d65e20e4d1d589d\\\",\\\"idfa\\\":\\\"\\\",\\\"imei\\\":\\\"860797036717585\\\",\\\"imsi\\\":\\\"460000864499542\\\",\\\"install_id\\\":\\\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\\\",\\\"mac\\\":\\\"\\\",\\\"wmac\\\":\\\"90:ad:f7:86:28:f0\\\",\\\"pcode\\\":\\\"\\\",\\\"uid\\\":\\\"\\\",\\\"property\\\":\\\"22C25529510391525315227D\\\",\\\"ar\\\":\\\"\\\",\\\"at\\\":\\\"291366\\\",\\\"deviceid\\\":\\\"\\\",\\\"lat\\\":\\\"38.92499077690972\\\",\\\"lg\\\":\\\"121.65503987630208\\\",\\\"os\\\":\\\"Android\\\",\\\"os_ver\\\":\\\"7.0\\\",\\\"brand\\\":\\\"HUAWEI\\\",\\\"xh\\\":\\\"HUAWEI NXT-AL10\\\",\\\"coordinate\\\":\\\"GAODE\\\",\\\"city\\\":\\\"大连市\\\",\\\"trigger_man_phone\\\":\\\"18018990297\\\",\\\"app_code\\\":\\\"1002\\\",\\\"cookie\\\":\\\"\\\",\\\"serialno\\\":\\\"CJL5T16217005509\\\"}\"," +
"\"logType\":\"log\",\"warehouseTableName\":\"user_tab\"}";
break;
default:
msg = getCurrDate() + " " + "{\"bizKey\":\"TEST\",\"sign\":\"F028492CC90210167E275977FFBE5A0C\",\"logStr\":\"test\",\"logType\":\"log\",\"warehouseTableName\":\"test\"}";
} */
// 分topic 分partitioner
int num = (int)(Math.random() * 8) + 1;
switch (num){
case 1:
msg = getCurrDate() + " " + "{\"bizKey\":\"KCAELBDDCYOXSLGR\",\"sign\":\"F018492CC90210167E275977FFBE5A0C\"," +
"\"logStr\":\"{\\\"id\\\":\\\"1001392469380538370\\\",\\\"acode\\\":\\\"CLICK\\\",\\\"app\\\":\\\"4.3.4\\\",\\\"apprunid\\\":\\\"5DB78A8FE3AE_1528272457316\\\",\\\"ctime\\\":\\\"2018-06-25 13:32:17\\\",\\\"create_time\\\":\\\"2018-06-25 13:32:17\\\",\\\"nt\\\":\\\"WIFI\\\",\\\"platform\\\":\\\"0\\\",\\\"r\\\":\\\"\\\",\\\"stime\\\":\\\"2018-06-25 13:32:17\\\",\\\"phone\\\":\\\"18519087423\\\",\\\"p1\\\":\\\"SQHomePageViewController\\\",\\\"p2\\\":\\\"SQRecommendViewController\\\",\\\"p3\\\":\\\"\\\",\\\"android_id\\\":\\\"94e3fcc30e90edf2\\\",\\\"auid\\\":\\\"7da009b4b7878c179d65e20e4d1d589d\\\",\\\"idfa\\\":\\\"\\\",\\\"imei\\\":\\\"860797036717585\\\",\\\"imsi\\\":\\\"460000864499542\\\",\\\"install_id\\\":\\\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\\\",\\\"mac\\\":\\\"\\\",\\\"wmac\\\":\\\"90:ad:f7:86:28:f0\\\",\\\"pcode\\\":\\\"\\\",\\\"uid\\\":\\\"\\\",\\\"property\\\":\\\"22C25529510391525315227D\\\",\\\"ar\\\":\\\"\\\",\\\"at\\\":\\\"291366\\\",\\\"deviceid\\\":\\\"\\\",\\\"lat\\\":\\\"38.92499077690972\\\",\\\"lg\\\":\\\"121.65503987630208\\\",\\\"os\\\":\\\"Android\\\",\\\"os_ver\\\":\\\"7.0\\\",\\\"brand\\\":\\\"HUAWEI\\\",\\\"xh\\\":\\\"HUAWEI NXT-AL10\\\",\\\"coordinate\\\":\\\"GAODE\\\",\\\"city\\\":\\\"大连市\\\",\\\"trigger_man_phone\\\":\\\"18018990297\\\",\\\"app_code\\\":\\\"1002\\\",\\\"cookie\\\":\\\"\\\",\\\"serialno\\\":\\\"CJL5T16217005509\\\"}\"," +
"\"logType\":\"log\",\"warehouseTableName\":\"risk_user_tab\"}";
break;
case 2:
msg = getCurrDate() + " " + "{\"bizKey\":\"KCAELBDDCYOXSLGR\",\"sign\":\"F028492CC90210167E275977FFBE5A0C\"," +
"\"logStr\":\"{\\\"id\\\":\\\"1002392469380538370\\\",\\\"acode\\\":\\\"CLICK\\\",\\\"app\\\":\\\"4.3.4\\\",\\\"apprunid\\\":\\\"5DB78A8FE3AE_1528272457316\\\",\\\"ctime\\\":\\\"2018-06-25 13:32:17\\\",\\\"create_time\\\":\\\"2018-06-25 13:32:17\\\",\\\"nt\\\":\\\"WIFI\\\",\\\"platform\\\":\\\"0\\\",\\\"r\\\":\\\"\\\",\\\"stime\\\":\\\"2018-06-25 13:32:17\\\",\\\"phone\\\":\\\"18519087423\\\",\\\"p1\\\":\\\"SQHomePageViewController\\\",\\\"p2\\\":\\\"SQRecommendViewController\\\",\\\"p3\\\":\\\"\\\",\\\"android_id\\\":\\\"94e3fcc30e90edf2\\\",\\\"auid\\\":\\\"7da009b4b7878c179d65e20e4d1d589d\\\",\\\"idfa\\\":\\\"\\\",\\\"imei\\\":\\\"860797036717585\\\",\\\"imsi\\\":\\\"460000864499542\\\",\\\"install_id\\\":\\\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\\\",\\\"mac\\\":\\\"\\\",\\\"wmac\\\":\\\"90:ad:f7:86:28:f0\\\",\\\"pcode\\\":\\\"\\\",\\\"uid\\\":\\\"\\\",\\\"property\\\":\\\"22C25529510391525315227D\\\",\\\"ar\\\":\\\"\\\",\\\"at\\\":\\\"291366\\\",\\\"deviceid\\\":\\\"\\\",\\\"lat\\\":\\\"38.92499077690972\\\",\\\"lg\\\":\\\"121.65503987630208\\\",\\\"os\\\":\\\"Android\\\",\\\"os_ver\\\":\\\"7.0\\\",\\\"brand\\\":\\\"HUAWEI\\\",\\\"xh\\\":\\\"HUAWEI NXT-AL10\\\",\\\"coordinate\\\":\\\"GAODE\\\",\\\"city\\\":\\\"大连市\\\",\\\"trigger_man_phone\\\":\\\"18018990297\\\",\\\"app_code\\\":\\\"1002\\\",\\\"cookie\\\":\\\"\\\",\\\"serialno\\\":\\\"CJL5T16217005509\\\"}\"," +
"\"logType\":\"log\",\"warehouseTableName\":\"risk_driver_tab\"}";
break;
case 3:
msg = getCurrDate() + " " + "{\"bizKey\":\"KCAELBDDCYOXSLGR\",\"sign\":\"F038492CC90210167E275977FFBE5A0C\"," +
"\"logStr\":\"{\\\"id\\\":\\\"1003392469380538370\\\",\\\"acode\\\":\\\"CLICK\\\",\\\"app\\\":\\\"4.3.4\\\",\\\"apprunid\\\":\\\"5DB78A8FE3AE_1528272457316\\\",\\\"ctime\\\":\\\"2018-06-25 13:32:17\\\",\\\"create_time\\\":\\\"2018-06-25 13:32:17\\\",\\\"nt\\\":\\\"WIFI\\\",\\\"platform\\\":\\\"0\\\",\\\"r\\\":\\\"\\\",\\\"stime\\\":\\\"2018-06-25 13:32:17\\\",\\\"phone\\\":\\\"18519087423\\\",\\\"p1\\\":\\\"SQHomePageViewController\\\",\\\"p2\\\":\\\"SQRecommendViewController\\\",\\\"p3\\\":\\\"\\\",\\\"android_id\\\":\\\"94e3fcc30e90edf2\\\",\\\"auid\\\":\\\"7da009b4b7878c179d65e20e4d1d589d\\\",\\\"idfa\\\":\\\"\\\",\\\"imei\\\":\\\"860797036717585\\\",\\\"imsi\\\":\\\"460000864499542\\\",\\\"install_id\\\":\\\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\\\",\\\"mac\\\":\\\"\\\",\\\"wmac\\\":\\\"90:ad:f7:86:28:f0\\\",\\\"pcode\\\":\\\"\\\",\\\"uid\\\":\\\"\\\",\\\"property\\\":\\\"22C25529510391525315227D\\\",\\\"ar\\\":\\\"\\\",\\\"at\\\":\\\"291366\\\",\\\"deviceid\\\":\\\"\\\",\\\"lat\\\":\\\"38.92499077690972\\\",\\\"lg\\\":\\\"121.65503987630208\\\",\\\"os\\\":\\\"Android\\\",\\\"os_ver\\\":\\\"7.0\\\",\\\"brand\\\":\\\"HUAWEI\\\",\\\"xh\\\":\\\"HUAWEI NXT-AL10\\\",\\\"coordinate\\\":\\\"GAODE\\\",\\\"city\\\":\\\"大连市\\\",\\\"trigger_man_phone\\\":\\\"18018990297\\\",\\\"app_code\\\":\\\"1002\\\",\\\"cookie\\\":\\\"\\\",\\\"serialno\\\":\\\"CJL5T16217005509\\\"}\"," +
"\"logType\":\"log\",\"warehouseTableName\":\"risk_tab\"}";
break;
case 4:
msg = getCurrDate() + " " + "{\"bizKey\":\"DXADXHZWEXTGCFQB\",\"sign\":\"F048492CC90210167E275977FFBE5A0C\"," +
"\"logStr\":\"{\\\"id\\\":\\\"1004392469380538370\\\",\\\"acode\\\":\\\"CLICK\\\",\\\"app\\\":\\\"4.3.4\\\",\\\"apprunid\\\":\\\"5DB78A8FE3AE_1528272457316\\\",\\\"ctime\\\":\\\"2018-06-25 13:32:17\\\",\\\"create_time\\\":\\\"2018-06-25 13:32:17\\\",\\\"nt\\\":\\\"WIFI\\\",\\\"platform\\\":\\\"0\\\",\\\"r\\\":\\\"\\\",\\\"stime\\\":\\\"2018-06-25 13:32:17\\\",\\\"phone\\\":\\\"18519087423\\\",\\\"p1\\\":\\\"SQHomePageViewController\\\",\\\"p2\\\":\\\"SQRecommendViewController\\\",\\\"p3\\\":\\\"\\\",\\\"android_id\\\":\\\"94e3fcc30e90edf2\\\",\\\"auid\\\":\\\"7da009b4b7878c179d65e20e4d1d589d\\\",\\\"idfa\\\":\\\"\\\",\\\"imei\\\":\\\"860797036717585\\\",\\\"imsi\\\":\\\"460000864499542\\\",\\\"install_id\\\":\\\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\\\",\\\"mac\\\":\\\"\\\",\\\"wmac\\\":\\\"90:ad:f7:86:28:f0\\\",\\\"pcode\\\":\\\"\\\",\\\"uid\\\":\\\"\\\",\\\"property\\\":\\\"22C25529510391525315227D\\\",\\\"ar\\\":\\\"\\\",\\\"at\\\":\\\"291366\\\",\\\"deviceid\\\":\\\"\\\",\\\"lat\\\":\\\"38.92499077690972\\\",\\\"lg\\\":\\\"121.65503987630208\\\",\\\"os\\\":\\\"Android\\\",\\\"os_ver\\\":\\\"7.0\\\",\\\"brand\\\":\\\"HUAWEI\\\",\\\"xh\\\":\\\"HUAWEI NXT-AL10\\\",\\\"coordinate\\\":\\\"GAODE\\\",\\\"city\\\":\\\"大连市\\\",\\\"trigger_man_phone\\\":\\\"18018990297\\\",\\\"app_code\\\":\\\"1002\\\",\\\"cookie\\\":\\\"\\\",\\\"serialno\\\":\\\"CJL5T16217005509\\\"}\"," +
"\"logType\":\"log\",\"warehouseTableName\":\"user_login_tab\"}";
break;
case 5:
msg = getCurrDate() + " " + "{\"bizKey\":\"DXADXHZWEXTGCFQB\",\"sign\":\"F058492CC90210167E275977FFBE5A0C\"," +
"\"logStr\":\"{\\\"id\\\":\\\"1004392469380538370\\\",\\\"acode\\\":\\\"CLICK\\\",\\\"app\\\":\\\"4.3.4\\\",\\\"apprunid\\\":\\\"5DB78A8FE3AE_1528272457316\\\",\\\"ctime\\\":\\\"2018-06-25 13:32:17\\\",\\\"create_time\\\":\\\"2018-06-25 13:32:17\\\",\\\"nt\\\":\\\"WIFI\\\",\\\"platform\\\":\\\"0\\\",\\\"r\\\":\\\"\\\",\\\"stime\\\":\\\"2018-06-25 13:32:17\\\",\\\"phone\\\":\\\"18519087423\\\",\\\"p1\\\":\\\"SQHomePageViewController\\\",\\\"p2\\\":\\\"SQRecommendViewController\\\",\\\"p3\\\":\\\"\\\",\\\"android_id\\\":\\\"94e3fcc30e90edf2\\\",\\\"auid\\\":\\\"7da009b4b7878c179d65e20e4d1d589d\\\",\\\"idfa\\\":\\\"\\\",\\\"imei\\\":\\\"860797036717585\\\",\\\"imsi\\\":\\\"460000864499542\\\",\\\"install_id\\\":\\\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\\\",\\\"mac\\\":\\\"\\\",\\\"wmac\\\":\\\"90:ad:f7:86:28:f0\\\",\\\"pcode\\\":\\\"\\\",\\\"uid\\\":\\\"\\\",\\\"property\\\":\\\"22C25529510391525315227D\\\",\\\"ar\\\":\\\"\\\",\\\"at\\\":\\\"291366\\\",\\\"deviceid\\\":\\\"\\\",\\\"lat\\\":\\\"38.92499077690972\\\",\\\"lg\\\":\\\"121.65503987630208\\\",\\\"os\\\":\\\"Android\\\",\\\"os_ver\\\":\\\"7.0\\\",\\\"brand\\\":\\\"HUAWEI\\\",\\\"xh\\\":\\\"HUAWEI NXT-AL10\\\",\\\"coordinate\\\":\\\"GAODE\\\",\\\"city\\\":\\\"大连市\\\",\\\"trigger_man_phone\\\":\\\"18018990297\\\",\\\"app_code\\\":\\\"1002\\\",\\\"cookie\\\":\\\"\\\",\\\"serialno\\\":\\\"CJL5T16217005509\\\"}\"," +
"\"logType\":\"log\",\"warehouseTableName\":\"user_order_tab\"}";
break;
case 6:
msg = getCurrDate() + " " + "{\"bizKey\":\"DXADXHZWEXTGCFQB\",\"sign\":\"F068492CC90210167E275977FFBE5A0C\"," +
"\"logStr\":\"{\\\"id\\\":\\\"1004392469380538370\\\",\\\"acode\\\":\\\"CLICK\\\",\\\"app\\\":\\\"4.3.4\\\",\\\"apprunid\\\":\\\"5DB78A8FE3AE_1528272457316\\\",\\\"ctime\\\":\\\"2018-06-25 13:32:17\\\",\\\"create_time\\\":\\\"2018-06-25 13:32:17\\\",\\\"nt\\\":\\\"WIFI\\\",\\\"platform\\\":\\\"0\\\",\\\"r\\\":\\\"\\\",\\\"stime\\\":\\\"2018-06-25 13:32:17\\\",\\\"phone\\\":\\\"18519087423\\\",\\\"p1\\\":\\\"SQHomePageViewController\\\",\\\"p2\\\":\\\"SQRecommendViewController\\\",\\\"p3\\\":\\\"\\\",\\\"android_id\\\":\\\"94e3fcc30e90edf2\\\",\\\"auid\\\":\\\"7da009b4b7878c179d65e20e4d1d589d\\\",\\\"idfa\\\":\\\"\\\",\\\"imei\\\":\\\"860797036717585\\\",\\\"imsi\\\":\\\"460000864499542\\\",\\\"install_id\\\":\\\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\\\",\\\"mac\\\":\\\"\\\",\\\"wmac\\\":\\\"90:ad:f7:86:28:f0\\\",\\\"pcode\\\":\\\"\\\",\\\"uid\\\":\\\"\\\",\\\"property\\\":\\\"22C25529510391525315227D\\\",\\\"ar\\\":\\\"\\\",\\\"at\\\":\\\"291366\\\",\\\"deviceid\\\":\\\"\\\",\\\"lat\\\":\\\"38.92499077690972\\\",\\\"lg\\\":\\\"121.65503987630208\\\",\\\"os\\\":\\\"Android\\\",\\\"os_ver\\\":\\\"7.0\\\",\\\"brand\\\":\\\"HUAWEI\\\",\\\"xh\\\":\\\"HUAWEI NXT-AL10\\\",\\\"coordinate\\\":\\\"GAODE\\\",\\\"city\\\":\\\"大连市\\\",\\\"trigger_man_phone\\\":\\\"18018990297\\\",\\\"app_code\\\":\\\"1002\\\",\\\"cookie\\\":\\\"\\\",\\\"serialno\\\":\\\"CJL5T16217005509\\\"}\"," +
"\"logType\":\"log\",\"warehouseTableName\":\"user_tab\"}";
break;
default:
msg = getCurrDate() + " " + "{\"bizKey\":\"TEST\",\"sign\":\"F028492CC90210167E275977FFBE5A0C\",\"logStr\":\"test\",\"logType\":\"log\",\"warehouseTableName\":\"test\"}";
}
return msg;
}
private static String getCurrDate() {
return new SimpleDateFormat("yyyy-MM-dd.HH:mm:ss.SSS").format(new Date());
}
}
- Kafka 消费者
package com.sqyc.bigdata.customer;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.common.TopicPartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
/**
* @description kafka 消费者
* @author huhanwei
*/
public class KafkaConsumer {
private static final Logger logger = LoggerFactory.getLogger(KafkaConsumer.class);
private org.apache.kafka.clients.consumer.KafkaConsumer<String,String> consumer;
// 初始化
public void init(List<String> topicList){
Properties prop = new Properties();
prop.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "写你自己的kafka集群地址"); // kafka 集群地址,多个用逗号分隔
prop.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
prop.setProperty(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "500");
prop.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
prop.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
prop.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest"); // latest earliest
prop.setProperty(ConsumerConfig.GROUP_ID_CONFIG,"a");
prop.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "a");
this.consumer = new org.apache.kafka.clients.consumer.KafkaConsumer<String, String>(prop);
this.consumer.subscribe(topicList);
while(true){
this.consume();
try {
Thread.currentThread().sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
// 消费者
public void consume() {
ConsumerRecords<String, String> records = this.consumer.poll(100);
Set<TopicPartition> partitions = records.partitions();
Iterator<ConsumerRecord<String, String>> it = records.iterator();
while(it.hasNext()){
ConsumerRecord<String, String> record = it.next();
try{
String key = record.key();
String value = record.value(); // 数据msg
String topic = record.topic(); // kafka topic
int partition = record.partition(); // 数据存放分区id
int expectPartition = Math.abs(key.hashCode()) % 3; // 计算期望数据存放的分区id
// partition 数据存放分区id
// expected partition 期望数据存放分区id
// 两者必须一致
logger.info("消费数据----> topic[" + topic + "] partition [" + partition + "] expect partition[" + expectPartition + "] msg[" + value + "]");
}catch(Exception e){
e.printStackTrace();
}
}
}
// 设置topic
private static List<String> getTopicList(){
List<String> topicList = new ArrayList<String>();
topicList.add("risk_test");
topicList.add("user_test");
return topicList;
}
public static void main(String[] arg) {
KafkaConsumer t = new KafkaConsumer();
t.init(getTopicList());
}
}
配置flume
# Name the components on this agent
logcollect.sources = taildir-source
logcollect.channels = risk-channel user-channel
logcollect.sinks = risk-sink user-sink
# Describe/configure the source
logcollect.sources.taildir-source.type = TAILDIR
logcollect.sources.taildir-source.positionFile = /home/bigdata/flume/partition/patition.json
logcollect.sources.taildir-source.filegroups = f1
logcollect.sources.taildir-source.filegroups.f1 = /home/bigdata/flume/data/.*.log
logcollect.sources.taildir-source.channels = risk-channel user-channel
# interceptor
logcollect.sources.taildir-source.interceptors = interceptor
logcollect.sources.taildir-source.interceptors.interceptor.type = regex_extractor
logcollect.sources.taildir-source.interceptors.interceptor.regex = .*(KCAELBDDCYOXSLGR|DXADXHZWEXTGCFQB).*
logcollect.sources.taildir-source.interceptors.interceptor.serializers = s1
logcollect.sources.taildir-source.interceptors.interceptor.serializers.s1.name = key
# selector
logcollect.sources.taildir-source.selector.type = multiplexing
logcollect.sources.taildir-source.selector.header = key
logcollect.sources.taildir-source.selector.mapping.KCAELBDDCYOXSLGR = risk-channel
logcollect.sources.taildir-source.selector.mapping.DXADXHZWEXTGCFQB = user-channel
# Describe/configure the channel
# risk-channel
logcollect.channels.risk-channel.type = memory
logcollect.channels.risk-channel.capacity=10000
logcollect.channels.risk-channel.byteCapacityBufferPercentage=2000
# user-channel
logcollect.channels.user-channel.type = memory
logcollect.channels.user-channel.capacity=10000
logcollect.channels.user-channel.byteCapacityBufferPercentage=2000
# Describe/configure the sink
# risk-sink
logcollect.sinks.risk-sink.type = com.sqyc.bigdata.sink.SolarKafkaSink
logcollect.sinks.risk-sink.kafka.topic = risk_test
logcollect.sinks.risk-sink.kafka.bootstrap.servers = kafka集群地址
logcollect.sinks.risk-sink.warningMobiles=phone1,phone2
logcollect.sinks.risk-sink.errorLogPath=/home/bigdata/flume/errorLogPath
logcollect.sinks.risk-sink.errorLogFileName=risk_error.log
logcollect.sinks.risk-sink.kafka.flumeBatchSize = 2000
logcollect.sinks.risk-sink.kafka.producer.acks = 1
logcollect.sinks.risk-sink.channel = risk-channel
# user-sink
logcollect.sinks.user-sink.type = com.sqyc.bigdata.sink.SolarKafkaSink
logcollect.sinks.user-sink.kafka.topic = user_test
logcollect.sinks.user-sink.warningMobiles=phone1,phone2
logcollect.sinks.user-sink.kafka.bootstrap.servers = kafka集群地址
logcollect.sinks.user-sink.errorLogPath=/home/bigdata/flume/errorLogPath
logcollect.sinks.user-sink.errorLogFileName=user_error.log
logcollect.sinks.user-sink.kafka.flumeBatchSize = 2000
logcollect.sinks.user-sink.kafka.producer.acks = 1
logcollect.sinks.user-sink.channel = user-channel
测试
启动flume,后台启动,日志级别设置为DEBUG,查看flume运行日志
nohup flume-ng agent \
--conf $FLUME_HOME/conf \
--conf-file $FLUME_HOME/conf/logcollect.conf \
--name logcollect \
-Dflume.root.logger=DEBUG,console >> /home/bigdata/flume/logs/flume.log 2>&1 &
- 模拟日志产生数据
java -jar log-generate.jar >> /home/bigdata/flume/data/data.log
- 启动kafka消费者,观察控制台
消费数据----> topic[user_test] partition [2] expect partition[2] msg[{"bizKey":"DXADXHZWEXTGCFQB","sign":"F028492CC90210167E275977FFBE5A0C","logStr":"{\"id\":\"1004392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"user_tab"}]
消费数据----> topic[user_test] partition [1] expect partition[1] msg[{"bizKey":"DXADXHZWEXTGCFQB","sign":"F028492CC90210167E275977FFBE5A0C","logStr":"{\"id\":\"1004392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"user_order_tab"}]
消费数据----> topic[risk_test] partition [2] expect partition[2] msg[{"bizKey":"KCAELBDDCYOXSLGR","sign":"946D813455B354029919A45255A2033C","logStr":"{\"id\":\"1003392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"risk_tab"}]
消费数据----> topic[risk_test] partition [2] expect partition[2] msg[{"bizKey":"KCAELBDDCYOXSLGR","sign":"946D813455B354029919A45255A2033C","logStr":"{\"id\":\"1003392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"risk_tab"}]
消费数据----> topic[user_test] partition [2] expect partition[2] msg[{"bizKey":"DXADXHZWEXTGCFQB","sign":"F028492CC90210167E275977FFBE5A0C","logStr":"{\"id\":\"1004392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"user_tab"}]
消费数据----> topic[user_test] partition [1] expect partition[1] msg[{"bizKey":"DXADXHZWEXTGCFQB","sign":"F028492CC90210167E275977FFBE5A0C","logStr":"{\"id\":\"1004392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"user_order_tab"}]
消费数据----> topic[user_test] partition [2] expect partition[2] msg[{"bizKey":"DXADXHZWEXTGCFQB","sign":"F028492CC90210167E275977FFBE5A0C","logStr":"{\"id\":\"1004392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"user_tab"}]
消费数据----> topic[risk_test] partition [1] expect partition[1] msg[{"bizKey":"KCAELBDDCYOXSLGR","sign":"83EC99B5ADE20A7179977F0D0513EC68","logStr":"{\"id\":\"1002392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"risk_driver_tab"}]
消费数据----> topic[risk_test] partition [1] expect partition[1] msg[{"bizKey":"KCAELBDDCYOXSLGR","sign":"83EC99B5ADE20A7179977F0D0513EC68","logStr":"{\"id\":\"1002392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"risk_driver_tab"}]
消费数据----> topic[user_test] partition [2] expect partition[2] msg[{"bizKey":"DXADXHZWEXTGCFQB","sign":"F028492CC90210167E275977FFBE5A0C","logStr":"{\"id\":\"1004392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"user_tab"}]
消费数据----> topic[user_test] partition [1] expect partition[1] msg[{"bizKey":"DXADXHZWEXTGCFQB","sign":"F028492CC90210167E275977FFBE5A0C","logStr":"{\"id\":\"1004392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"user_order_tab"}]
消费数据----> topic[user_test] partition [1] expect partition[1] msg[{"bizKey":"DXADXHZWEXTGCFQB","sign":"F028492CC90210167E275977FFBE5A0C","logStr":"{\"id\":\"1004392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"user_order_tab"}]
消费数据----> topic[risk_test] partition [1] expect partition[1] msg[{"bizKey":"KCAELBDDCYOXSLGR","sign":"E4EBB38858E9B5AB6995C34511303512","logStr":"{\"id\":\"1001392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"risk_user_tab"}]
消费数据----> topic[risk_test] partition [1] expect partition[1] msg[{"bizKey":"KCAELBDDCYOXSLGR","sign":"E4EBB38858E9B5AB6995C34511303512","logStr":"{\"id\":\"1001392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"risk_user_tab"}]
消费数据----> topic[risk_test] partition [2] expect partition[2] msg[{"bizKey":"KCAELBDDCYOXSLGR","sign":"946D813455B354029919A45255A2033C","logStr":"{\"id\":\"1003392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"risk_tab"}]
消费数据----> topic[user_test] partition [1] expect partition[1] msg[{"bizKey":"DXADXHZWEXTGCFQB","sign":"F028492CC90210167E275977FFBE5A0C","logStr":"{\"id\":\"1004392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"user_order_tab"}]
消费数据----> topic[risk_test] partition [1] expect partition[1] msg[{"bizKey":"KCAELBDDCYOXSLGR","sign":"E4EBB38858E9B5AB6995C34511303512","logStr":"{\"id\":\"1001392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"risk_user_tab"}]
消费数据----> topic[risk_test] partition [1] expect partition[1] msg[{"bizKey":"KCAELBDDCYOXSLGR","sign":"83EC99B5ADE20A7179977F0D0513EC68","logStr":"{\"id\":\"1002392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"risk_driver_tab"}]
消费数据----> topic[user_test] partition [0] expect partition[0] msg[{"bizKey":"DXADXHZWEXTGCFQB","sign":"F028492CC90210167E275977FFBE5A0C","logStr":"{\"id\":\"1004392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"user_login_tab"}]
消费数据----> topic[risk_test] partition [1] expect partition[1] msg[{"bizKey":"KCAELBDDCYOXSLGR","sign":"83EC99B5ADE20A7179977F0D0513EC68","logStr":"{\"id\":\"1002392469380538370\",\"acode\":\"CLICK\",\"app\":\"4.3.4\",\"apprunid\":\"5DB78A8FE3AE_1528272457316\",\"ctime\":\"2018-06-25 13:32:17\",\"create_time\":\"2018-06-25 13:32:17\",\"nt\":\"WIFI\",\"platform\":\"0\",\"r\":\"\",\"stime\":\"2018-06-25 13:32:17\",\"phone\":\"18519087423\",\"p1\":\"SQHomePageViewController\",\"p2\":\"SQRecommendViewController\",\"p3\":\"\",\"android_id\":\"94e3fcc30e90edf2\",\"auid\":\"7da009b4b7878c179d65e20e4d1d589d\",\"idfa\":\"\",\"imei\":\"860797036717585\",\"imsi\":\"460000864499542\",\"install_id\":\"7da009b4b7878c179d65e20e4d1d589d_1515973088836\",\"mac\":\"\",\"wmac\":\"90:ad:f7:86:28:f0\",\"pcode\":\"\",\"uid\":\"\",\"property\":\"22C25529510391525315227D\",\"ar\":\"\",\"at\":\"291366\",\"deviceid\":\"\",\"lat\":\"38.92499077690972\",\"lg\":\"121.65503987630208\",\"os\":\"Android\",\"os_ver\":\"7.0\",\"brand\":\"HUAWEI\",\"xh\":\"HUAWEI NXT-AL10\",\"coordinate\":\"GAODE\",\"city\":\"大连市\",\"trigger_man_phone\":\"18018990297\",\"app_code\":\"1002\",\"cookie\":\"\",\"serialno\":\"CJL5T16217005509\"}","logType":"log","warehouseTableName":"risk_driver_tab"}]
说明:partition为数据实际存放的分区,expect partition是我们期望数据存放在那个分区,测试和我们预期一致,flume实时收集到kafka不同topic不同partiton到此结束。
踩坑记
kafka 消费数据获取数据所在分区和期望分区不一致
这个问题已经解决,在SolarParttioner类中,获取到partitionList后,他是倒序的,借助一个新的list对它做一次升序即可。file-channel使用异常
上面flume实时收集数据使用的channel是memery,即内存,其实最初用的是file-channel,使用同上,但是在使用file-channel的过程中遇到了两个问题,一个是关闭flume报如下错误
java.lang.IllegalStateException: Channel closed [channel=risk-channel]
at org.apache.flume.channel.file.FileChannel.createTransaction(FileChannel.java:359)
at org.apache.flume.channel.BasicChannelSemantics.getTransaction(BasicChannelSemantics.java:122)
at com.sqyc.bigdata.solar.kuiper.sink.SolarKafkaSink.process(SolarKafkaSink.java:41)
at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:67)
at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:145)
at java.lang.Thread.run(Thread.java:748)
解决方法:把配置文件中的关于channel配置checkpointDir和dataDir目录清空
还有一个问题就是,上述配置修改为file-channel,执行如下操作会发生一个问题,有时某一个topic就收不到数据
(1).多次启停flume
(2).多次启停产生日志
最后没有采用file-channel,一是存在上述问题,二是file-channel效率低,最终采用的memery