Hadoop MapReduce自定义数据类型

一 自定义数据类型的实现

1.继承接口Writable,实现其方法write()和readFields(), 以便该数据能被序列化后完成网络传输或文件输入/输出；

2.如果该数据需要作为主键key使用，或需要比较数值大小时，则需要实现WritalbeComparable接口,实现其方法write(),readFields(),CompareTo() 。

3.重写toString()、hashCode()、equals()方法。 

二 自定义数据类型示例

OrderWritable — 作为key

UserWritable  — 作为value

 1 package com.ibeifeng.mapreduce.io;
 2 
 3 import java.io.DataInput;
 4 import java.io.DataOutput;
 5 import java.io.IOException;
 6 import org.apache.hadoop.io.WritableComparable;
 7 
 8 public class OrderWritable implements WritableComparable<OrderWritable> {
 9 
10     private String orderId;
11     private float price;
12 
13     public OrderWritable() {
14 
15     }
16 
17     public OrderWritable(String orderId, float price) {
18         this.set(orderId, price);
19     }
20 
21     public void set(String orderId, float price) {
22         this.orderId = orderId;
23         this.price = price;
24     }
25 
26     public String getOrderId() {
27         return orderId;
28     }
29 
30     public void setOrderId(String orderId) {
31         this.orderId = orderId;
32     }
33 
34     public float getPrice() {
35         return price;
36     }
37 
38     public void setPrice(float price) {
39         this.price = price;
40     }
41 
42     public void write(DataOutput out) throws IOException {
43         out.writeUTF(orderId);
44         out.writeFloat(price);
45 
46     }
47 
48     public void readFields(DataInput in) throws IOException {
49 
50         this.orderId = in.readUTF();
51         this.price = in.readFloat();
52     }
53 
54     public int compareTo(OrderWritable o) {
55 
56         int comp = this.getOrderId().compareTo(o.getOrderId());
57 
58         if (0 == comp) {
59             return Float.valueOf(this.getPrice()).compareTo(
60                     Float.valueOf(o.getPrice()));
61         }
62 
63         return comp;
64     }
65 
66     @Override
67     public int hashCode() {
68         final int prime = 31;
69         int result = 1;
70         result = prime * result + ((orderId == null) ? 0 : orderId.hashCode());
71         result = prime * result + Float.floatToIntBits(price);
72         return result;
73     }
74 
75     @Override
76     public boolean equals(Object obj) {
77         if (this == obj)
78             return true;
79         if (obj == null)
80             return false;
81         if (getClass() != obj.getClass())
82             return false;
83         OrderWritable other = (OrderWritable) obj;
84         if (orderId == null) {
85             if (other.orderId != null)
86                 return false;
87         } else if (!orderId.equals(other.orderId))
88             return false;
89         if (Float.floatToIntBits(price) != Float.floatToIntBits(other.price))
90             return false;
91         return true;
92     }
93 
94     @Override
95     public String toString() {
96         return orderId + "\t" + price;
97     }
98 
99 }

 1 package com.ibeifeng.mapreduce.io;
 2 
 3 import java.io.DataInput;
 4 import java.io.DataOutput;
 5 import java.io.IOException;
 6 import org.apache.hadoop.io.Writable;
 7 
 8 public class UserWritable implements Writable {
 9 
10     private int id;
11     private String name;
12 
13     public UserWritable() {
14 
15     }
16 
17     public UserWritable(int id, String name) {
18         this.set(id, name);
19     }
20 
21     public void set(int id, String name) {
22 
23         this.id = id;
24         this.name = name;
25     }
26 
27     public int getId() {
28         return id;
29     }
30 
31     public void setId(int id) {
32         this.id = id;
33     }
34 
35     public String getName() {
36         return name;
37     }
38 
39     public void setName(String name) {
40         this.name = name;
41     }
42 
43     public void write(DataOutput out) throws IOException {
44         out.writeInt(id);
45         out.writeUTF(name);
46 
47     }
48 
49     public void readFields(DataInput in) throws IOException {
50         this.id = in.readInt();
51         this.name = in.readUTF();
52     }
53 
54     @Override
55     public int hashCode() {
56         final int prime = 31;
57         int result = 1;
58         result = prime * result + id;
59         result = prime * result + ((name == null) ? 0 : name.hashCode());
60         return result;
61     }
62 
63     @Override
64     public boolean equals(Object obj) {
65         if (this == obj)
66             return true;
67         if (obj == null)
68             return false;
69         if (getClass() != obj.getClass())
70             return false;
71         UserWritable other = (UserWritable) obj;
72         if (id != other.id)
73             return false;
74         if (name == null) {
75             if (other.name != null)
76                 return false;
77         } else if (!name.equals(other.name))
78             return false;
79         return true;
80     }
81 
82     @Override
83     public String toString() {
84         return id + "\t" + name;
85     }
86 
87 }