完整代码:https://gitee.com/firewolf/java-io/tree/master/java-io/netty-03-codec
在实际开发中,我们经常需要操作POJO而非只是简单的基本数据类型,对于我们自己编写POJO来说,在网络传输和文件存储的时候,我们需要对POJO进行编码,在JDK中,采用了序列化进行处理。但是,JDK的序列化具有一些缺点,所以这里介绍几种比较流行的编解码技术。
一、JDK序列化的缺点
jdk的序列化使用起来非常简单,但是有一些缺点,如下:
- 无法跨语言;
- 序列化之后的字节码流太大;
- 序列化性能较低;
由于JDK序列化的缺点,市面上产生了比较多的编码解码技术,用于传递数据。这里列举几种比较流行的编解码技术:
- MessagePack
- ProtoBuf
- Marshalling(暂时不做讲解)
二、MessagePack
MessagePack是一款高效的二进制序列化框架,它像json一样支持不同语言的数据交换,但是它的性能更快,序列化之后码流更小。
官网地址:https://msgpack.org/
(一)MessagePack基本使用
MessagePack的最新版本和以前的变化比较大,在网上能看到大量的旧版本的例子,我这里使用最新版本的MessagePack。
官网参考地址:
https://github.com/msgpack/msgpack-java
https://github.com/msgpack/msgpack-java/blob/develop/msgpack-core/src/test/java/org/msgpack/core/example/MessagePackExample.java
1. 引入相关依赖
<dependency>
<groupId>org.msgpack</groupId>
<artifactId>msgpack-core</artifactId>
<version>0.8.16</version>
</dependency>
2.常见数据类型的编码解码
package com.firewolf.java.io.msgpack;
import com.firewolf.java.io.entities.User;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.junit.Test;
import org.msgpack.core.MessageBufferPacker;
import org.msgpack.core.MessagePack;
import org.msgpack.core.MessageUnpacker;
/**
* 作者:刘兴 时间:2019-05-28
**/
public class MsgPackTest {
/**
* 基本数据类型的编码解码
*/
@Test
public void baseType() throws Exception {
MessageBufferPacker messageBufferPacker = MessagePack.newDefaultBufferPacker();
messageBufferPacker.packInt(12)
.packString("liuxing");
messageBufferPacker.close();
byte[] bytes = messageBufferPacker.toByteArray();
System.out.println(bytes.length);
MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(bytes);
int age = messageUnpacker.unpackInt();
String name = messageUnpacker.unpackString();
System.out.println(age + "," + name);
}
/**
* 基本数据类型数组处理
*/
@Test
public void baseList() throws Exception {
List<Integer> list = Arrays.asList(34, 56, 90);
//编码
MessageBufferPacker messageBufferPacker = MessagePack.newDefaultBufferPacker();
messageBufferPacker.packArrayHeader(list.size());
list.forEach(
x -> {
try {
messageBufferPacker.packInt(x);
} catch (IOException e) {
e.printStackTrace();
}
}
);
messageBufferPacker.close();
byte[] bytes = messageBufferPacker.toByteArray();
System.out.println(bytes.length);
//解码
MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(bytes);
int listSize = messageUnpacker.unpackArrayHeader();
List<Integer> els = new ArrayList<>(listSize);
for (int i = 0; i < listSize; i++) {
int el = messageUnpacker.unpackInt();
els.add(el);
}
System.out.println(els);
}
@Test
public void baseMap() throws Exception {
Map<Integer, String> stus = new HashMap<>();
stus.put(1001, "张三");
stus.put(1002, "李四");
stus.put(1003, "王五");
//编码
MessageBufferPacker messageBufferPacker = MessagePack.newDefaultBufferPacker();
messageBufferPacker.packMapHeader(stus.size());
stus.entrySet().forEach(x -> {
try {
messageBufferPacker.packInt(x.getKey()).packString(x.getValue());
} catch (IOException e) {
e.printStackTrace();
}
});
messageBufferPacker.close();
//解码
MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(messageBufferPacker.toByteArray());
int mapSize = messageUnpacker.unpackMapHeader();
Map<Integer, String> stus2 = new HashMap<>(mapSize);
for (int i = 0; i < mapSize; i++) {
stus2.put(messageUnpacker.unpackInt(), messageUnpacker.unpackString());
}
System.out.println(stus2);
}
/**
* POJO,新版本的API里面,只能自己对每个字段进行处理了,
*/
@Test
public void pojo() throws Exception {
User user = new User();
user.setName("zhangsan");
user.setAge(34);
MessageBufferPacker bufferPacker = MessagePack.newDefaultBufferPacker();
bufferPacker.packInt(user.getAge()).packString(user.getName());
bufferPacker.close();
MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(bufferPacker.toByteArray());
User u = new User(messageUnpacker.unpackInt(), messageUnpacker.unpackString());
System.out.println(u);
}
}
其中User类定义如下:
package com.firewolf.java.io.entities;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* 作者:刘兴 时间:2019-05-28
**/
@Data
@AllArgsConstructor
@NoArgsConstructor
public class User {
private Integer age;
private String name;
}
这些测试用例包括了基本数据类型(int作为代表)、String类型、POJO、List、Map等的处理,应该能满足我们大部分要求。
注:这里为了简单,用到了Lombok插件,具体使用这里不再赘述。
(二)Netty使用MessagePack传递POJO
上面介绍了messagepack的常用API之后,我们在Netty中使用MessagePack进行编码和解码,从而来传递POJO
1. 编写编码解码器
package com.firewolf.java.io.msgpack;
import com.firewolf.java.io.entities.User;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.ByteToMessageDecoder;
import io.netty.handler.codec.MessageToByteEncoder;
import java.util.List;
import org.msgpack.core.MessageBufferPacker;
import org.msgpack.core.MessagePack;
import org.msgpack.core.MessageUnpacker;
/**
* 作者:刘兴 时间:2019-05-29
**/
public class MessagePackCODEC {
/**
* 编码器,这里处理的是User
*/
public class MessagePackEncoder extends MessageToByteEncoder<User> {
@Override
protected void encode(ChannelHandlerContext channelHandlerContext, User user, ByteBuf byteBuf) throws Exception {
MessageBufferPacker packer = MessagePack.newDefaultBufferPacker();
packer.packInt(user.getAge());
packer.packString(user.getName());
packer.close();
byteBuf.writeBytes(packer.toByteArray());
}
}
/**
* 解码器,这里处理的是User
*/
public class MessagePackDecoder extends ByteToMessageDecoder {
@Override
protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List<Object> list)
throws Exception {
byte[] bytes = new byte[byteBuf.readableBytes()];
byteBuf.readBytes(bytes);
MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes);
User u = new User();
u.setAge(unpacker.unpackInt());
u.setName(unpacker.unpackString());
list.add(u);
}
@Override
public void channelInactive(ChannelHandlerContext ctx) throws Exception {
System.out.println("有连接断开了....");
}
}
}
这里被继承的MessageToByteEncoder和ByteToMessageDecoder是Netty提供的API。
2. 给服务端和设置解码器
package com.firewolf.java.io.msgpack;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelHandlerAdapter;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import io.netty.handler.codec.LengthFieldPrepender;
/**
* 作者:刘兴 时间:2019/5/15
**/
public class UserServer {
public UserServer(int port) {
EventLoopGroup bossGroup = new NioEventLoopGroup();
EventLoopGroup workGroup = new NioEventLoopGroup();
try {
ServerBootstrap bootstrap = new ServerBootstrap();
bootstrap.group(bossGroup, workGroup)
.channel(NioServerSocketChannel.class)
.option(ChannelOption.SO_BACKLOG, 1024)
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel socketChannel) throws Exception {
socketChannel.pipeline()
//配合LengthFieldBasedFrameDecoder和LengthFieldPrepender解决半粘包和拆包问题。
.addLast("frame", new LengthFieldBasedFrameDecoder(65535, 0, 2, 0, 2))
.addLast("lenth", new LengthFieldPrepender(2))
//设置解码器
.addLast("message decoder", new MessagePackCODEC().new MessagePackDecoder())
.addLast(new MessageServerHandler());
}
});
ChannelFuture future = bootstrap.bind(port).sync();
System.out.println("启动服务端监听端口:" + port);
future.channel().closeFuture().sync();
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
bossGroup.shutdownGracefully();
workGroup.shutdownGracefully();
}
}
class MessageServerHandler extends ChannelInboundHandlerAdapter {
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
User u = (User) msg; //这里可以直接读取到User对象
System.out.println(u);
}
}
public static void main(String[] args) {
new UserServer(9999);
}
}
3. 给客户端设置编码器
package com.firewolf.java.io.msgpack;
import io.netty.bootstrap.Bootstrap;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import io.netty.handler.codec.LengthFieldPrepender;
import java.net.InetSocketAddress;
/**
* 作者:刘兴 时间:2019/5/15
**/
public class UserClient {
public UserClient(String host, int port) {
NioEventLoopGroup group = new NioEventLoopGroup();
try {
Bootstrap bootstrap = new Bootstrap();
bootstrap.group(group)
.channel(NioSocketChannel.class)
.option(ChannelOption.TCP_NODELAY, true)
.handler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel socketChannel) throws Exception {
socketChannel.pipeline()
//配合LengthFieldBasedFrameDecoder和LengthFieldPrepender解决半粘包和拆包问题。
.addLast("frame", new LengthFieldBasedFrameDecoder(65535, 0, 2, 0, 2))
.addLast("lenth", new LengthFieldPrepender(2))
//设置编码器
.addLast("message encoder", new MessagePackCODEC().new MessagePackEncoder())
.addLast(new MessageClientHandler());
}
});
ChannelFuture f = bootstrap.connect(new InetSocketAddress(host, port)).sync();
System.out.println("连接服务器成功-----");
f.channel().closeFuture().sync();
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
group.shutdownGracefully();
}
}
class MessageClientHandler extends ChannelInboundHandlerAdapter {
@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
for (int i = 0; i < 10; i++) {
User u = new User(1001 + 10 * i, "张三");
ctx.writeAndFlush(u); //直接写出对象
}
}
}
public static void main(String[] args) {
new UserClient("127.0.0.1", 9999);
}
}
细心的同学可以看到:
- 客户端可以直接写出对象了,而服务端可以直接接受到对象了,这个就是咱们自定义的编码解码器的功劳;
- 除了设置编码解码器之外,还多余添加了LengthFieldBasedFrameDecoder和LengthFieldPrepender,这两个是为了解决拆包和粘包问题的,如果去掉了,你会发现不能打印10个User
- 本例中,我们只是在客户端发消息,在服务端接受消息,如果实际开发中双方相互通讯,那么就应该给服务端和客户端都设置编码解码器。
4. 执行结果
执行后服务端打印如下结果:
启动服务端监听端口:9999
User(age=1001, name=张三)
User(age=1011, name=张三)
User(age=1021, name=张三)
User(age=1031, name=张三)
User(age=1041, name=张三)
User(age=1051, name=张三)
User(age=1061, name=张三)
User(age=1071, name=张三)
User(age=1081, name=张三)
User(age=1091, name=张三)
三、ProtoBuf
Google的ProtoBuf在业界非常流行,很多商业项目都选择ProtoBuf作为序列化框架,下面我们对它进行一些讲解。
ps:谷歌已经对华为使用谷歌的操作系统进行打击了,使用谷歌别的技术会不会有隐患呢?
官网地址:https://developers.google.cn/protocol-buffers/
(一)、ProtoBuf优点
- 在谷歌内部长期使用,产品成熟度很高;
- 跨语言、支持多种语言;
- 编码后的消息更小、更有利于传输和存储;
- 编解码的性能非常高;
- 支持不同版本协议的向前兼容;
- 支持定义可选和必选字段;
(二)protobuf基本使用
protobuf需要通过对proto文件进行编译生产对应语言的文件,然后进行操作,这里我们关注的是Java。需要相应的编译工具,我们采用插件进行处理,这样的话,可以跟随项目移动而不需要做额外操作。
1.引入maven依赖的jar包
<properties>
<grpc.version>1.13.1</grpc.version>
<protobuf.version>3.5.1</protobuf.version>
<protoc.version>3.5.1-1</protoc.version>
</properties>
<dependencies>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-stub</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
<version>${protobuf.version}</version>
</dependency>
</dependencies>
2.引入编译proto的插件
<build>
<extensions>
<extension>
<groupId>kr.motd.maven</groupId>
<artifactId>os-maven-plugin</artifactId>
<version>1.5.0.Final</version>
</extension>
</extensions>
<plugins>
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
<version>0.5.1</version>
<configuration>
<protocArtifact>com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier}</protocArtifact>
<pluginId>grpc-java</pluginId>
<pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
</configuration>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>compile-custom</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.4.1</version>
<executions>
<execution>
<id>enforce</id>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<requireUpperBoundDeps/>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
3. 编写proto文件,定义实体类
注意需要在~/src/main/proto下面:
user2.proto:
syntax = "proto3";
option java_multiple_files = true;
option java_package = "com.firewolf.java.io.protobuf";
option java_outer_classname = "User2Proto";
package user2;
message User2 {
string name = 1;
int32 age = 2;
}
这个文件的语法格式可以参考官网地址即可。
4. 编译proto生产java文件
执行maven的compile插件,生成如下文件:
5. 编码解码测试
package com.firewolf.java.io.protobuf;
import org.junit.Test;
/**
* 作者:刘兴 时间:2019-05-29
**/
public class ProtoBufTest {
@Test
public void tesCODEC() throws Exception {
User2 user = User2.newBuilder().setAge(10).setName("张三").build();
byte[] bytes = user.toByteArray();
final User2 user2 = User2.parseFrom(bytes);
System.out.println(user2.getName()+","+user2.getAge());
}
}
结果如下:
张三,10
(三)Netty使用ProtoBuf进行编码解码
步骤其实和上一节使用MessagePack差不多,
1. 编写编码解码器
package com.firewolf.java.io.protobuf;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.ByteToMessageDecoder;
import io.netty.handler.codec.MessageToByteEncoder;
import java.util.List;
/**
* 作者:刘兴 时间:2019-05-29
**/
public class ProtoBufCODEC {
public class Encoder extends MessageToByteEncoder<User2> {
@Override
protected void encode(ChannelHandlerContext channelHandlerContext, User2 user2, ByteBuf byteBuf) throws Exception {
byteBuf.writeBytes(user2.toByteArray());
}
}
public class Decoder extends ByteToMessageDecoder {
@Override
protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List<Object> list)
throws Exception {
byte[] bytes = new byte[byteBuf.readableBytes()];
byteBuf.readBytes(bytes);
User2 user = User2.parseFrom(bytes);
list.add(user);
}
}
}
2. 服务端使用解码器
package com.firewolf.java.io.protobuf;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelHandlerAdapter;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import io.netty.handler.codec.LengthFieldPrepender;
/**
* 作者:刘兴 时间:2019/5/15
**/
public class User2Server {
public User2Server(int port) {
EventLoopGroup bossGroup = new NioEventLoopGroup();
EventLoopGroup workGroup = new NioEventLoopGroup();
try {
ServerBootstrap bootstrap = new ServerBootstrap();
bootstrap.group(bossGroup, workGroup)
.channel(NioServerSocketChannel.class)
.option(ChannelOption.SO_BACKLOG, 1024)
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel socketChannel) throws Exception {
socketChannel.pipeline()
//配合LengthFieldBasedFrameDecoder和LengthFieldPrepender解决半粘包和拆包问题。
.addLast("frame", new LengthFieldBasedFrameDecoder(65535, 0, 2, 0, 2))
.addLast("lenth", new LengthFieldPrepender(2))
//设置解码器
.addLast("message decoder", new ProtoBufCODEC().new Decoder())
.addLast(new MessageServerHandler());
}
});
ChannelFuture future = bootstrap.bind(port).sync();
System.out.println("启动服务端监听端口:" + port);
future.channel().closeFuture().sync();
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
bossGroup.shutdownGracefully();
workGroup.shutdownGracefully();
}
}
class MessageServerHandler extends ChannelInboundHandlerAdapter {
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
User2 u = (User2) msg; //这里可以直接读取到User对象
System.out.println(u.getName() + "," + u.getAge());
}
}
public static void main(String[] args) {
new User2Server(9999);
}
}
3. 客户端使用编码器
package com.firewolf.java.io.protobuf;
import io.netty.bootstrap.Bootstrap;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelHandlerAdapter;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import io.netty.handler.codec.LengthFieldPrepender;
import java.net.InetSocketAddress;
/**
* 作者:刘兴 时间:2019/5/15
**/
public class User2Client {
public User2Client(String host, int port) {
NioEventLoopGroup group = new NioEventLoopGroup();
try {
Bootstrap bootstrap = new Bootstrap();
bootstrap.group(group)
.channel(NioSocketChannel.class)
.option(ChannelOption.TCP_NODELAY, true)
.handler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel socketChannel) throws Exception {
socketChannel.pipeline()
//配合LengthFieldBasedFrameDecoder和LengthFieldPrepender解决半粘包和拆包问题。
.addLast("frame", new LengthFieldBasedFrameDecoder(65535, 0, 2, 0, 2))
.addLast("lenth", new LengthFieldPrepender(2))
//设置编码器
.addLast("message encoder", new ProtoBufCODEC().new Encoder())
.addLast(new MessageClientHandler());
}
});
ChannelFuture f = bootstrap.connect(new InetSocketAddress(host, port)).sync();
System.out.println("连接服务器成功-----");
f.channel().closeFuture().sync();
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
group.shutdownGracefully();
}
}
class MessageClientHandler extends ChannelInboundHandlerAdapter {
@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
for (int i = 0; i < 10; i++) {
User2 user2 = User2.newBuilder().setAge(10 + 10 * i).setName("李四" + i).build();
ctx.writeAndFlush(user2); //直接写出对象
}
}
}
public static void main(String[] args) {
new User2Client("127.0.0.1", 9999);
}
}
效果如下:
李四0,10
李四1,20
李四2,30
李四3,40
李四4,50
李四5,60
李四6,70
李四7,80
李四8,90
李四9,100