Avro序列化和RPC实现

序列化和反序列化

• Maven:Pom.xml

    <dependencies>
        <dependency>
            <groupId>org.apache.avro</groupId>
            <artifactId>avro</artifactId>
            <version>1.8.2</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.avro</groupId>
                <artifactId>avro-maven-plugin</artifactId>
                <version>1.8.1</version>
                <executions>
                    <execution>
                        <phase>generate-sources</phase>
                        <goals>
                            <goal>schema</goal>
                        </goals>
                        <configuration>
                            <sourceDirectory>${project.basedir}/src/main/avro/</sourceDirectory>
                            <outputDirectory>${project.basedir}/src/main/java/</outputDirectory>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>
        </plugins>
    </build>

• Avro:MapAvro.avsc

{
  "type":"record",
  "namespace":"com.qidai",
  "name":"Employee",
  "doc":"Test Employee bean",
  "fields":[
    {"name":"id","type":["null","int"]},
    {"name":"name","type":["null","string"]},
    {"name":"age","type":["null","int"]},
    {"name":"gender","type":["null","string"]}
  ]
}

• 添加完依赖后直接点击maven插件install即可，就会产生对应的class
• 序列化和反序列化(使用javaclass)

@Test
public void ser() throws Exception {
    Employee employee = Employee.newBuilder().setAge(12).setGender("NAN").setId(1).setName("tom").build();
    DatumWriter<Employee> employeeDatumWriter = new SpecificDatumWriter<>(Employee.class);
    DataFileWriter<Employee> dataFileWriter = new DataFileWriter<>(employeeDatumWriter);

    dataFileWriter.create(employee.getSchema(), new File("emp.avro"));

    dataFileWriter.append(employee);
    dataFileWriter.close();
}
@Test
public void deSer() throws Exception {
    DatumReader<Employee> employeeDatumReader = new SpecificDatumReader<>(Employee.class);
    DataFileReader<Employee> dataFileReader = new DataFileReader<>(new File("emp.avro"), employeeDatumReader);

    Employee employee = null;

    while (dataFileReader.hasNext()) {
        employee = dataFileReader.next();
        System.out.println(employee);
    }
}

• 序列化反序列化(直接使用avro文件)

@Test
public void ser() throws Exception {
    Schema schema = new Schema.Parser().parse(new File("emp.avsc"));
    GenericRecord empRecord1 = new GenericData.Record(schema);
    empRecord1.put("id", 1);
    empRecord1.put("name", "Ben");
    empRecord1.put("age", 7);
    empRecord1.put("gender", "nv");
    File file = new File("empser.avro");
    DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<>(datumWriter);
    dataFileWriter.create(schema, file);
    dataFileWriter.append(empRecord1);
    dataFileWriter.close();
}
@Test
public void deSer() throws Exception {
    //指定avro格式文件
    Schema schema = new Schema.Parser().parse(new File("emp.avsc"));
    //指定序列化好的数据文件
    File file = new File("empser.avro");
    DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(schema);
    DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(file, datumReader);

    GenericRecord emp = null;

    while (dataFileReader.hasNext()) {
        emp = dataFileReader.next();
        System.out.println(emp);
    }
}

RPC实现

• 编写avsc文件：user.avsc，作为要发送的实体类

{
"namespace":"com.qidai.bean",
"name":"User",
"doc":"test rpc class",
"type":"record",
    "fields":[
        {"name":"name","type":["string","null"]},
        {"name":"age","type":["int","null"]},
        {"name":"date","type":["string","null"]}
    ]
}

• 编写协议文件:protomes.avdl

@namespace("com.qidai.proto")
protocol UserProtocol{
  import schema "user.avsc";           
  string sendMes(com.qidai.bean.User user);   //对应协议中的方法
}

• 编写完毕之后检查maven pom文件

<dependencies>
    <dependency>
        <groupId>org.apache.avro</groupId>
        <artifactId>avro</artifactId>
        <version>1.8.2</version>
    </dependency>
    <dependency>
        <groupId>org.apache.avro</groupId>
        <artifactId>avro-ipc</artifactId>
        <version>1.8.2</version>
    </dependency>
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.12</version>
    </dependency>
</dependencies>
<build>
    <plugins>
        <plugin>
            <groupId>org.apache.avro</groupId>
            <artifactId>avro-maven-plugin</artifactId>
            <version>1.8.1</version>
            <executions>
                <execution>
                    <phase>generate-sources</phase>
                    <goals>
                        <goal>schema</goal>
                        <goal>idl-protocol</goal>
                    </goals>
                    <configuration>
                        <sourceDirectory>${project.basedir}/src/main/avro/</sourceDirectory>
                        <outputDirectory>${project.basedir}/src/main/java/</outputDirectory>
                    </configuration>
                </execution>
            </executions>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
            <configuration>
                <source>1.8</source>
                <target>1.8</target>
            </configuration>
        </plugin>
    </plugins>
</build>

• 检查没错之后，直接插件install生成avro类
• 生成之后会看到一个User的传输类，还有一个proto包下的协议接口
• 实现协议接口，重写自己的逻辑：server

public class UserProtocolImpl implements UserProtocol {
    @Override
    public CharSequence sendMes(User user) throws AvroRemoteException {
        System.out.println("Server --> 接收" + user.toString());
        return "Client sendMes " + user.toString();
    }
}

• 实现协议接口，重写自己的逻辑：client

public class UserProtocolImpl implements UserProtocol {
    @Override
    public CharSequence sendMes(User user) throws AvroRemoteException {
        System.out.println("Client --> 发送" + user.toString());
        return "Client sendMes " + user.toString();
    }
}

• 编写serverApp

public class ServerApp {
    public static void main(String[] args) throws IOException, InterruptedException {
        Responder responder = new SpecificResponder(UserProtocol.class,new UserProtocolImpl());
        SaslSocketServer server = new SaslSocketServer(responder,new InetSocketAddress(9999));
        server.start();
        Thread.sleep(5000000);
    }
}

• 编写clientApp

public class ClientApp {
    public static void main(String[] args) throws IOException {
        SaslSocketTransceiver transceiver = new SaslSocketTransceiver(new InetSocketAddress(9999));
        UserProtocol client = SpecificRequestor.getClient(UserProtocol.class, transceiver);
        Scanner scanner = new Scanner(System.in);
        while (scanner.next() != null) {
            User user = new User();
            user.setName("xiaofen");
            System.out.println(client.sendMes(user));
        }
    }
}

• 测试：先启动serverApp，然后启动clientApp，这时候在client的console中输入任意值，会发现服务端和客户端都会输出消息

  • server

  Server --> {"name": "xiaofen", "age": null, "date": null}
  Server --> {"name": "xiaofen", "age": null, "date": null}
  Server --> {"name": "xiaofen", "age": null, "date": null}
  Server --> {"name": "xiaofen", "age": null, "date": null}
  Server --> {"name": "xiaofen", "age": null, "date": null}
  Server --> {"name": "xiaofen", "age": null, "date": null}
  Server --> {"name": "xiaofen", "age": null, "date": null}

  • client

  1
  Client sendMes {"name": "xiaofen", "age": null, "date": null}
  1
  Client sendMes {"name": "xiaofen", "age": null, "date": null}
  1
  Client sendMes {"name": "xiaofen", "age": null, "date": null}
  1
  Client sendMes {"name": "xiaofen", "age": null, "date": null}
  1
  Client sendMes {"name": "xiaofen", "age": null, "date": null}
  1
  Client sendMes {"name": "xiaofen", "age": null, "date": null}
  1
  Client sendMes {"name": "xiaofen", "age": null, "date": null}

• 对于server的client的实现有如下几种
  1.基于jetty的http实现:HttpServer 和HttpTransceiver

2.基于netty的实现:NettyServer和NettyTransceiver
3.基于TCP的实现:SocketServer和SocketTransceiver
4.基于UDP的实现:DatagramServer和DatagramTransceiver
5.基于加密的TCP实现:SaslSocketServer和SaslSocketTransceiver