1.场景
提供了一个接近1G的二进制文件，内容为振动数据（包含头数据 、有效数据、尾数据），要求解析入库
2.思路
运行在flink上，从文件中增量抽数据，flatmap处理，最后sink到redis中
public class SourceFromFile extends RichSourceFunction<byte[]> {
private volatile Boolean isRunning = true;
@Override
public void run(SourceContext ctx) throws Exception {
File file = new File("/root/Ca.bin");
FileInputStream fi = new FileInputStream(file);
byte[] buffer = new byte[413264];
fi.read(new byte[10]);
int count=0;
while (isRunning) {
int read = fi.read(buffer, 0, 413264);
if(read==413264){
ctx.collect(buffer);
}else {
isRunning=false;
}
}
fi.close();
}
@Override
public void cancel() {
isRunning = false;
}
}
public class ReadF {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStreamSource<byte[]> dataStream = env.addSource(new SourceFromFile());
SingleOutputStreamOperator<Tuple3<String, String, Object>> flatMap = dataStream.flatMap(new FlatMapFunction<byte[], Tuple3<String, String, Object>>() {
@Override
public void flatMap(byte[] buf, Collector<Tuple3<String, String, Object>> out) throws Exception {
StringBuffer sb = new StringBuffer();
for (int i = 0; i < 3; i++) {
String hex = Integer.toHexString(buf[i] & 0xFF);
if (hex.length() == 1) {
hex = '0' + hex;
}
sb.append(hex.toUpperCase());
}
String s = UUID.randomUUID().toString();
Tuple3<String, String, Object> modify = Tuple3.of(s, "modify", sb.toString());
out.collect(modify);
Tuple3<String, String, Object> num = Tuple3.of(s, "num", (buf[3]&0xFF) + "");
out.collect(num);
Tuple3<String, String, Object> car = Tuple3.of(s, "car", (buf[5] & 0xFF)+ "");
out.collect(car);
Tuple3<String, String, Object> sign = Tuple3.of(s, "sign", (buf[6]& 0xFF) + "");
out.collect(sign);
Tuple3<String, String, Object> time = Tuple3.of(s, "time", (2000 + (buf[7]& 0xFF)) + "年" + (buf[8]& 0xFF) + "月" + (buf[9]& 0xFF) + "日" + (buf[10]& 0xFF) + "时" + (buf[11]& 0xFF) + "分" + (buf[12]& 0xFF) + "秒");
out.collect(time);
ArrayList<Integer> list = new ArrayList<>();
for (int i = 1; i < 409933 / 16; i++) {
int sum=0;
for(int j=0;j<16;j++){
sum+=(buf[16*i+j]& 0xFF);
}
list.add(sum);
}
Tuple3<String, String, Object> of = Tuple3.of(s, "振动" , list);
out.collect(of);
}
}).setParallelism(2);
flatMap.addSink(new MyRedisSink());
env.execute();
}
}
class MyRedisSink extends RichSinkFunction<Tuple3<String, String, Object>> {
private transient Jedis jedis;
@Override
public void open(Configuration config) {
jedis = new Jedis("192.168.1.127", 16379, 20000);
jedis.select(2);
}
@Override
public void invoke(Tuple3<String, String, Object> value, Context context) {
if (!jedis.isConnected()) {
jedis.connect();
}
//保存
jedis.hset(value.f0, value.f1, String.valueOf(value.f2));
}
@Override
public void close() {
jedis.close();
}
public static void main(String[] args) {
long l = System.currentTimeMillis();
Jedis jedis = new Jedis("192.168.1.127", 16379, 20000);
jedis.select(2);
String hget = jedis.hget("test", "振动");
ArrayList list=new ArrayList(Arrays.asList(hget.split(",")));
String s = (String) list.get(2);
System.out.println(s);
long l1 = System.currentTimeMillis();
System.out.println(l1-l);
}
}