Netty-5 编解码技术

完整代码:https://gitee.com/firewolf/java-io/tree/master/java-io/netty-03-codec

在实际开发中,我们经常需要操作POJO而非只是简单的基本数据类型,对于我们自己编写POJO来说,在网络传输和文件存储的时候,我们需要对POJO进行编码,在JDK中,采用了序列化进行处理。但是,JDK的序列化具有一些缺点,所以这里介绍几种比较流行的编解码技术。

一、JDK序列化的缺点

jdk的序列化使用起来非常简单,但是有一些缺点,如下:

  1. 无法跨语言;
  2. 序列化之后的字节码流太大;
  3. 序列化性能较低;

由于JDK序列化的缺点,市面上产生了比较多的编码解码技术,用于传递数据。这里列举几种比较流行的编解码技术:

  • MessagePack
  • ProtoBuf
  • Marshalling(暂时不做讲解)

二、MessagePack

MessagePack是一款高效的二进制序列化框架,它像json一样支持不同语言的数据交换,但是它的性能更快,序列化之后码流更小。
官网地址:https://msgpack.org/

(一)MessagePack基本使用

MessagePack的最新版本和以前的变化比较大,在网上能看到大量的旧版本的例子,我这里使用最新版本的MessagePack。
官网参考地址:
https://github.com/msgpack/msgpack-java
https://github.com/msgpack/msgpack-java/blob/develop/msgpack-core/src/test/java/org/msgpack/core/example/MessagePackExample.java

1. 引入相关依赖

    <dependency>
      <groupId>org.msgpack</groupId>
      <artifactId>msgpack-core</artifactId>
      <version>0.8.16</version>
    </dependency>

2.常见数据类型的编码解码

package com.firewolf.java.io.msgpack;

import com.firewolf.java.io.entities.User;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.junit.Test;
import org.msgpack.core.MessageBufferPacker;
import org.msgpack.core.MessagePack;
import org.msgpack.core.MessageUnpacker;

/**
 * 作者:刘兴 时间:2019-05-28
 **/
public class MsgPackTest {


  /**
   * 基本数据类型的编码解码
   */
  @Test
  public void baseType() throws Exception {

    MessageBufferPacker messageBufferPacker = MessagePack.newDefaultBufferPacker();
    messageBufferPacker.packInt(12)
        .packString("liuxing");
    messageBufferPacker.close();

    byte[] bytes = messageBufferPacker.toByteArray();
    System.out.println(bytes.length);

    MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(bytes);

    int age = messageUnpacker.unpackInt();
    String name = messageUnpacker.unpackString();
    System.out.println(age + "," + name);
  }


  /**
   * 基本数据类型数组处理
   */
  @Test
  public void baseList() throws Exception {
    List<Integer> list = Arrays.asList(34, 56, 90);
    //编码
    MessageBufferPacker messageBufferPacker = MessagePack.newDefaultBufferPacker();
    messageBufferPacker.packArrayHeader(list.size());
    list.forEach(
        x -> {
          try {
            messageBufferPacker.packInt(x);
          } catch (IOException e) {
            e.printStackTrace();
          }
        }
    );
    messageBufferPacker.close();
    byte[] bytes = messageBufferPacker.toByteArray();
    System.out.println(bytes.length);

    //解码
    MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(bytes);
    int listSize = messageUnpacker.unpackArrayHeader();
    List<Integer> els = new ArrayList<>(listSize);
    for (int i = 0; i < listSize; i++) {
      int el = messageUnpacker.unpackInt();
      els.add(el);
    }

    System.out.println(els);
  }


  @Test
  public void baseMap() throws Exception {

    Map<Integer, String> stus = new HashMap<>();
    stus.put(1001, "张三");
    stus.put(1002, "李四");
    stus.put(1003, "王五");

    //编码
    MessageBufferPacker messageBufferPacker = MessagePack.newDefaultBufferPacker();
    messageBufferPacker.packMapHeader(stus.size());
    stus.entrySet().forEach(x -> {
      try {
        messageBufferPacker.packInt(x.getKey()).packString(x.getValue());
      } catch (IOException e) {
        e.printStackTrace();
      }
    });
    messageBufferPacker.close();

    //解码
    MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(messageBufferPacker.toByteArray());
    int mapSize = messageUnpacker.unpackMapHeader();
    Map<Integer, String> stus2 = new HashMap<>(mapSize);
    for (int i = 0; i < mapSize; i++) {
      stus2.put(messageUnpacker.unpackInt(), messageUnpacker.unpackString());
    }
    System.out.println(stus2);
  }

  /**
   * POJO,新版本的API里面,只能自己对每个字段进行处理了,
   */
  @Test
  public void pojo() throws Exception {
    User user = new User();
    user.setName("zhangsan");
    user.setAge(34);
    MessageBufferPacker bufferPacker = MessagePack.newDefaultBufferPacker();

    bufferPacker.packInt(user.getAge()).packString(user.getName());
    bufferPacker.close();

    MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(bufferPacker.toByteArray());
    User u = new User(messageUnpacker.unpackInt(), messageUnpacker.unpackString());
    System.out.println(u);
  }
}

其中User类定义如下:

package com.firewolf.java.io.entities;

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

/**
 * 作者:刘兴 时间:2019-05-28
 **/
@Data
@AllArgsConstructor
@NoArgsConstructor
public class User {
  private Integer age;
  private String name;
}

这些测试用例包括了基本数据类型(int作为代表)、String类型、POJO、List、Map等的处理,应该能满足我们大部分要求。
注:这里为了简单,用到了Lombok插件,具体使用这里不再赘述。

(二)Netty使用MessagePack传递POJO

上面介绍了messagepack的常用API之后,我们在Netty中使用MessagePack进行编码和解码,从而来传递POJO

1. 编写编码解码器

package com.firewolf.java.io.msgpack;

import com.firewolf.java.io.entities.User;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.ByteToMessageDecoder;
import io.netty.handler.codec.MessageToByteEncoder;
import java.util.List;
import org.msgpack.core.MessageBufferPacker;
import org.msgpack.core.MessagePack;
import org.msgpack.core.MessageUnpacker;

/**
 * 作者:刘兴 时间:2019-05-29
 **/
public class MessagePackCODEC {

  /**
   * 编码器,这里处理的是User
   */
  public class MessagePackEncoder extends MessageToByteEncoder<User> {

    @Override
    protected void encode(ChannelHandlerContext channelHandlerContext, User user, ByteBuf byteBuf) throws Exception {
      MessageBufferPacker packer = MessagePack.newDefaultBufferPacker();
      packer.packInt(user.getAge());
      packer.packString(user.getName());
      packer.close();
      byteBuf.writeBytes(packer.toByteArray());
    }
  }


  /**
   * 解码器,这里处理的是User
   */
  public class MessagePackDecoder extends ByteToMessageDecoder {

    @Override
    protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List<Object> list)
        throws Exception {
      byte[] bytes = new byte[byteBuf.readableBytes()];
      byteBuf.readBytes(bytes);
      MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bytes);
      User u = new User();
      u.setAge(unpacker.unpackInt());
      u.setName(unpacker.unpackString());
      list.add(u);
    }

    @Override
    public void channelInactive(ChannelHandlerContext ctx) throws Exception {
      System.out.println("有连接断开了....");
    }
  }

}

这里被继承的MessageToByteEncoder和ByteToMessageDecoder是Netty提供的API。

2. 给服务端和设置解码器

package com.firewolf.java.io.msgpack;

import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelHandlerAdapter;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import io.netty.handler.codec.LengthFieldPrepender;

/**
 * 作者:刘兴 时间:2019/5/15
 **/
public class UserServer {


  public UserServer(int port) {

    EventLoopGroup bossGroup = new NioEventLoopGroup();
    EventLoopGroup workGroup = new NioEventLoopGroup();
    try {
      ServerBootstrap bootstrap = new ServerBootstrap();
      bootstrap.group(bossGroup, workGroup)
          .channel(NioServerSocketChannel.class)
          .option(ChannelOption.SO_BACKLOG, 1024)
          .childHandler(new ChannelInitializer<SocketChannel>() {
            @Override
            protected void initChannel(SocketChannel socketChannel) throws Exception {
              socketChannel.pipeline()
                  //配合LengthFieldBasedFrameDecoder和LengthFieldPrepender解决半粘包和拆包问题。
                  .addLast("frame", new LengthFieldBasedFrameDecoder(65535, 0, 2, 0, 2))
                  .addLast("lenth", new LengthFieldPrepender(2))
                  //设置解码器
                  .addLast("message decoder", new MessagePackCODEC().new MessagePackDecoder())
                  .addLast(new MessageServerHandler());
            }
          });
      ChannelFuture future = bootstrap.bind(port).sync();
      System.out.println("启动服务端监听端口:" + port);
      future.channel().closeFuture().sync();

    } catch (InterruptedException e) {
      e.printStackTrace();
    } finally {
      bossGroup.shutdownGracefully();
      workGroup.shutdownGracefully();
    }

  }

  class MessageServerHandler extends ChannelInboundHandlerAdapter {
    @Override
    public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
      User u = (User) msg; //这里可以直接读取到User对象
      System.out.println(u);
    }

  }


  public static void main(String[] args) {
    new UserServer(9999);
  }
}

3. 给客户端设置编码器

package com.firewolf.java.io.msgpack;

import io.netty.bootstrap.Bootstrap;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import io.netty.handler.codec.LengthFieldPrepender;
import java.net.InetSocketAddress;

/**
 * 作者:刘兴 时间:2019/5/15
 **/
public class UserClient {


  public UserClient(String host, int port) {
    NioEventLoopGroup group = new NioEventLoopGroup();
    try {
      Bootstrap bootstrap = new Bootstrap();
      bootstrap.group(group)
          .channel(NioSocketChannel.class)
          .option(ChannelOption.TCP_NODELAY, true)
          .handler(new ChannelInitializer<SocketChannel>() {
            @Override
            protected void initChannel(SocketChannel socketChannel) throws Exception {
              socketChannel.pipeline()
                  //配合LengthFieldBasedFrameDecoder和LengthFieldPrepender解决半粘包和拆包问题。
                  .addLast("frame", new LengthFieldBasedFrameDecoder(65535, 0, 2, 0, 2))
                  .addLast("lenth", new LengthFieldPrepender(2))
                  //设置编码器
                  .addLast("message encoder", new MessagePackCODEC().new MessagePackEncoder())
                  .addLast(new MessageClientHandler());
            }
          });

      ChannelFuture f = bootstrap.connect(new InetSocketAddress(host, port)).sync();
      System.out.println("连接服务器成功-----");
      f.channel().closeFuture().sync();
    } catch (InterruptedException e) {
      e.printStackTrace();
    } finally {
      group.shutdownGracefully();
    }
  }


  class MessageClientHandler extends ChannelInboundHandlerAdapter {

    @Override
    public void channelActive(ChannelHandlerContext ctx) throws Exception {
      for (int i = 0; i < 10; i++) {
        User u = new User(1001 + 10 * i, "张三");
        ctx.writeAndFlush(u); //直接写出对象
      }
    }
  }
  public static void main(String[] args) {
    new UserClient("127.0.0.1", 9999);
  }
}

细心的同学可以看到:

  1. 客户端可以直接写出对象了,而服务端可以直接接受到对象了,这个就是咱们自定义的编码解码器的功劳;
  2. 除了设置编码解码器之外,还多余添加了LengthFieldBasedFrameDecoder和LengthFieldPrepender,这两个是为了解决拆包和粘包问题的,如果去掉了,你会发现不能打印10个User
  3. 本例中,我们只是在客户端发消息,在服务端接受消息,如果实际开发中双方相互通讯,那么就应该给服务端和客户端都设置编码解码器。

4. 执行结果

执行后服务端打印如下结果:
启动服务端监听端口:9999
User(age=1001, name=张三)
User(age=1011, name=张三)
User(age=1021, name=张三)
User(age=1031, name=张三)
User(age=1041, name=张三)
User(age=1051, name=张三)
User(age=1061, name=张三)
User(age=1071, name=张三)
User(age=1081, name=张三)
User(age=1091, name=张三)

三、ProtoBuf

Google的ProtoBuf在业界非常流行,很多商业项目都选择ProtoBuf作为序列化框架,下面我们对它进行一些讲解。
ps:谷歌已经对华为使用谷歌的操作系统进行打击了,使用谷歌别的技术会不会有隐患呢?

官网地址:https://developers.google.cn/protocol-buffers/

(一)、ProtoBuf优点

  1. 在谷歌内部长期使用,产品成熟度很高;
  2. 跨语言、支持多种语言;
  3. 编码后的消息更小、更有利于传输和存储;
  4. 编解码的性能非常高;
  5. 支持不同版本协议的向前兼容;
  6. 支持定义可选和必选字段;

(二)protobuf基本使用

protobuf需要通过对proto文件进行编译生产对应语言的文件,然后进行操作,这里我们关注的是Java。需要相应的编译工具,我们采用插件进行处理,这样的话,可以跟随项目移动而不需要做额外操作。

1.引入maven依赖的jar包

<properties>
    <grpc.version>1.13.1</grpc.version>
    <protobuf.version>3.5.1</protobuf.version>
    <protoc.version>3.5.1-1</protoc.version>
  </properties>

  <dependencies>
    <dependency>
      <groupId>io.grpc</groupId>
      <artifactId>grpc-netty</artifactId>
      <version>${grpc.version}</version>
    </dependency>
    <dependency>
      <groupId>io.grpc</groupId>
      <artifactId>grpc-protobuf</artifactId>
      <version>${grpc.version}</version>
    </dependency>
    <dependency>
      <groupId>io.grpc</groupId>
      <artifactId>grpc-stub</artifactId>
      <version>${grpc.version}</version>
    </dependency>
    <dependency>
      <groupId>com.google.protobuf</groupId>
      <artifactId>protobuf-java-util</artifactId>
      <version>${protobuf.version}</version>
    </dependency>
  </dependencies>

2.引入编译proto的插件

<build>
    <extensions>
      <extension>
        <groupId>kr.motd.maven</groupId>
        <artifactId>os-maven-plugin</artifactId>
        <version>1.5.0.Final</version>
      </extension>
    </extensions>
    <plugins>
      <plugin>
        <groupId>org.xolstice.maven.plugins</groupId>
        <artifactId>protobuf-maven-plugin</artifactId>
        <version>0.5.1</version>
        <configuration>
          <protocArtifact>com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier}</protocArtifact>
          <pluginId>grpc-java</pluginId>
          <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
        </configuration>
        <executions>
          <execution>
            <goals>
              <goal>compile</goal>
              <goal>compile-custom</goal>
            </goals>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-enforcer-plugin</artifactId>
        <version>1.4.1</version>
        <executions>
          <execution>
            <id>enforce</id>
            <goals>
              <goal>enforce</goal>
            </goals>
            <configuration>
              <rules>
                <requireUpperBoundDeps/>
              </rules>
            </configuration>
          </execution>
        </executions>
      </plugin>
    </plugins>
  </build>

3. 编写proto文件,定义实体类

注意需要在~/src/main/proto下面:
user2.proto:

syntax = "proto3";

option java_multiple_files = true;
option java_package = "com.firewolf.java.io.protobuf";
option java_outer_classname = "User2Proto";

package user2;

message User2 {
  string name = 1;
  int32 age = 2;
}

这个文件的语法格式可以参考官网地址即可。

4. 编译proto生产java文件

执行maven的compile插件,生成如下文件:
在这里插入图片描述

5. 编码解码测试

package com.firewolf.java.io.protobuf;

import org.junit.Test;

/**
 * 作者:刘兴 时间:2019-05-29
 **/
public class ProtoBufTest {

  @Test
  public void tesCODEC() throws Exception {
    User2 user = User2.newBuilder().setAge(10).setName("张三").build();
    byte[] bytes = user.toByteArray();

    final User2 user2 = User2.parseFrom(bytes);
    System.out.println(user2.getName()+","+user2.getAge());
  }

}

结果如下:
张三,10

(三)Netty使用ProtoBuf进行编码解码

步骤其实和上一节使用MessagePack差不多,

1. 编写编码解码器

package com.firewolf.java.io.protobuf;

import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.ByteToMessageDecoder;
import io.netty.handler.codec.MessageToByteEncoder;
import java.util.List;

/**
 * 作者:刘兴 时间:2019-05-29
 **/
public class ProtoBufCODEC {


  public class Encoder extends MessageToByteEncoder<User2> {

    @Override
    protected void encode(ChannelHandlerContext channelHandlerContext, User2 user2, ByteBuf byteBuf) throws Exception {
      byteBuf.writeBytes(user2.toByteArray());
    }
  }

  public class Decoder extends ByteToMessageDecoder {

    @Override
    protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List<Object> list)
        throws Exception {
      byte[] bytes = new byte[byteBuf.readableBytes()];
      byteBuf.readBytes(bytes);
      User2 user = User2.parseFrom(bytes);
      list.add(user);

    }
  }

}

2. 服务端使用解码器

package com.firewolf.java.io.protobuf;

import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelHandlerAdapter;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import io.netty.handler.codec.LengthFieldPrepender;

/**
 * 作者:刘兴 时间:2019/5/15
 **/
public class User2Server {


  public User2Server(int port) {

    EventLoopGroup bossGroup = new NioEventLoopGroup();
    EventLoopGroup workGroup = new NioEventLoopGroup();
    try {
      ServerBootstrap bootstrap = new ServerBootstrap();
      bootstrap.group(bossGroup, workGroup)
          .channel(NioServerSocketChannel.class)
          .option(ChannelOption.SO_BACKLOG, 1024)
          .childHandler(new ChannelInitializer<SocketChannel>() {
            @Override
            protected void initChannel(SocketChannel socketChannel) throws Exception {
              socketChannel.pipeline()
                  //配合LengthFieldBasedFrameDecoder和LengthFieldPrepender解决半粘包和拆包问题。
                  .addLast("frame", new LengthFieldBasedFrameDecoder(65535, 0, 2, 0, 2))
                  .addLast("lenth", new LengthFieldPrepender(2))
                  //设置解码器
                  .addLast("message decoder", new ProtoBufCODEC().new Decoder())
                  .addLast(new MessageServerHandler());
            }
          });
      ChannelFuture future = bootstrap.bind(port).sync();
      System.out.println("启动服务端监听端口:" + port);
      future.channel().closeFuture().sync();

    } catch (InterruptedException e) {
      e.printStackTrace();
    } finally {
      bossGroup.shutdownGracefully();
      workGroup.shutdownGracefully();
    }

  }
  class MessageServerHandler extends ChannelInboundHandlerAdapter {


    @Override
    public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
      User2 u = (User2) msg; //这里可以直接读取到User对象
      System.out.println(u.getName() + "," + u.getAge());
    }

  }


  public static void main(String[] args) {
    new User2Server(9999);
  }

}

3. 客户端使用编码器

package com.firewolf.java.io.protobuf;

import io.netty.bootstrap.Bootstrap;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelHandlerAdapter;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import io.netty.handler.codec.LengthFieldPrepender;
import java.net.InetSocketAddress;

/**
 * 作者:刘兴 时间:2019/5/15
 **/
public class User2Client {


  public User2Client(String host, int port) {
    NioEventLoopGroup group = new NioEventLoopGroup();
    try {
      Bootstrap bootstrap = new Bootstrap();
      bootstrap.group(group)
          .channel(NioSocketChannel.class)
          .option(ChannelOption.TCP_NODELAY, true)
          .handler(new ChannelInitializer<SocketChannel>() {
            @Override
            protected void initChannel(SocketChannel socketChannel) throws Exception {
              socketChannel.pipeline()
                  //配合LengthFieldBasedFrameDecoder和LengthFieldPrepender解决半粘包和拆包问题。
                  .addLast("frame", new LengthFieldBasedFrameDecoder(65535, 0, 2, 0, 2))
                  .addLast("lenth", new LengthFieldPrepender(2))
                  //设置编码器
                  .addLast("message encoder", new ProtoBufCODEC().new Encoder())
                  .addLast(new MessageClientHandler());
            }
          });

      ChannelFuture f = bootstrap.connect(new InetSocketAddress(host, port)).sync();
      System.out.println("连接服务器成功-----");
      f.channel().closeFuture().sync();
    } catch (InterruptedException e) {
      e.printStackTrace();
    } finally {
      group.shutdownGracefully();
    }
  }


  class MessageClientHandler extends ChannelInboundHandlerAdapter {

    @Override
    public void channelActive(ChannelHandlerContext ctx) throws Exception {
      for (int i = 0; i < 10; i++) {
        User2 user2 = User2.newBuilder().setAge(10 + 10 * i).setName("李四" + i).build();
        ctx.writeAndFlush(user2); //直接写出对象
      }
    }
  }

  public static void main(String[] args) {
    new User2Client("127.0.0.1", 9999);
  }
}

效果如下:
李四0,10
李四1,20
李四2,30
李四3,40
李四4,50
李四5,60
李四6,70
李四7,80
李四8,90
李四9,100

猜你喜欢

转载自blog.csdn.net/mail_liuxing/article/details/90640470
今日推荐