package com.test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.PageFilter;
import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;
public class HBaseUtil {
@Value("${hbase.zookeeper.quorum}")
private String addr="node233,node232,node231";
@Value("${hbase.zookeeper.property.clientPort}")
private String port="2181";
Logger logger = Logger.getLogger(getClass());
private static Connection connection;
@SuppressWarnings("static-access")
public void getConnection(){
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum",addr);
conf.set("hbase.zookeeper.property.clientPort", port);
//conf.set("zookeeper.znode.parent", "/hbase-unsecure");
HBaseHelper hBaseHelper = new HBaseHelper(conf);
try {
this.connection = hBaseHelper.getConnection();
} catch (IOException e) {
e.printStackTrace();
}
}
public HBaseUtil() {
getConnection();
}
public static void insert() throws IOException {
Map<String, Map<String, String>> dataMap = new HashMap<>();
String basicTag1 = "14";
String basicTag2 = "15";
String basicTag3 = "18";
String basicTag4 = "21";
String mergeTag1 = "23";
Map<String, String> tagMap1 = new HashMap<>();
tagMap1.put("basictag-"+basicTag1, basicTag1);
tagMap1.put("basictag-"+basicTag2, basicTag2);
tagMap1.put("basictag-"+basicTag3, basicTag3);
tagMap1.put("basictag-"+basicTag4, basicTag4);
tagMap1.put("mergetag-"+mergeTag1, mergeTag1);
Map<String, String> userMap1 = new HashMap<>();
userMap1.put("name", "张三");
userMap1.put("idnum", "420922199008174270");
userMap1.put("phonenum", "15810469847");
userMap1.put("banknum", "6217002710000684874");
userMap1.put("gender", "1");
userMap1.put("qq", "847594757");
userMap1.put("email", "[email protected]");
userMap1.put("type", "0");
String rowKey1 = UUID.randomUUID().toString();
dataMap.put(rowKey1+"_personOrObjInfo", userMap1);
dataMap.put(rowKey1+"_tag", tagMap1);
Map<String, String> tagMap2 = new HashMap<>();
tagMap2.put("basictag-"+basicTag1, basicTag1);
tagMap2.put("basictag-"+basicTag2, basicTag2);
tagMap2.put("basictag-"+basicTag3, basicTag3);
tagMap2.put("basictag-"+basicTag4, basicTag4);
tagMap2.put("mergetag-"+mergeTag1, mergeTag1);
Map<String, String> userMap2 = new HashMap<>();
userMap2.put("name", "李四");
userMap2.put("idnum", "420922199008174271");
userMap2.put("phonenum", "15810469848");
userMap2.put("banknum", "6217002710000684875");
userMap2.put("gender", "1");
userMap2.put("qq", "847594758");
userMap2.put("email", "[email protected]");
userMap2.put("type", "0");
String rowKey2 = UUID.randomUUID().toString();
dataMap.put(rowKey2+"_personOrObjInfo", userMap2);
dataMap.put(rowKey2+"_tag", tagMap2);
Map<String, String> tagMap3 = new HashMap<>();
tagMap3.put("basictag-"+basicTag1, basicTag1);
tagMap3.put("basictag-"+basicTag2, basicTag2);
tagMap3.put("basictag-"+basicTag3, basicTag3);
tagMap3.put("basictag-"+basicTag4, basicTag4);
tagMap3.put("mergetag-"+mergeTag1, mergeTag1);
Map<String, String> userMap3 = new HashMap<>();
userMap3.put("name", "王五");
userMap3.put("idnum", "420922199008174272");
userMap3.put("phonenum", "15810469849");
userMap3.put("banknum", "6217002710000684876");
userMap3.put("gender", "1");
userMap3.put("qq", "847594759");
userMap3.put("email", "[email protected]");
userMap3.put("type", "0");
String rowKey3 = UUID.randomUUID().toString();
dataMap.put(rowKey3+"_personOrObjInfo", userMap3);
dataMap.put(rowKey3+"_tag", tagMap3);
addRows("portrait", "_", dataMap);
}
public static void delete() throws IOException {
Collection<Map<String, String>> a = queryForQuilfier("portrait","personOrObjInfo", "basictag-14", "14");
for (Map<String, String> map : a) {
System.out.println("====="+map.get("rowKey"));
deleteByRowKey("portrait",map.get("rowKey"));
}
}
public static void main(String[] args) throws IOException {
List<String> result = HBaseUtil.queryForQuilfierName("portrait","tag","basictag-14", 5);
System.out.println(result);
delete();
insert();
}
/*
* 判断表是否存在
*
*
* @tableName 表名
*/
public boolean isExist(String tableName) throws IOException {
TableName table_name = TableName.valueOf(tableName);
Admin admin = connection.getAdmin();
boolean exit = admin.tableExists(table_name);
admin.close();
return exit;
}
/**
* 单行添加
* @param tableName
* @param rowKey
* @param family
* @param keyValue
* @throws IOException
*/
@SuppressWarnings("unused")
private static void addRow(String tableName, String rowKey, String family, Map<String, String> keyValue) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Put put = new Put(Bytes.toBytes(rowKey));
for (Entry<String, String> entry : keyValue.entrySet()) {
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(entry.getKey()), Bytes.toBytes(entry.getValue()));
}
table.put(put);
table.close();
keyValue.clear();
}
public static void addRows(String tableName, String rowFamilySeparator, Map<String, Map<String, String>> keyValues) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
List<Put> puts = new ArrayList<Put>();
for (Entry<String, Map<String, String>> entry : keyValues.entrySet()) {
String key = entry.getKey();
if (null == rowFamilySeparator || rowFamilySeparator.isEmpty()) {
rowFamilySeparator = "_";
}
String rowKey = key.split(rowFamilySeparator)[0];
String family = key.split(rowFamilySeparator)[1];
Map<String, String> keyValue = entry.getValue();
Put put = new Put(Bytes.toBytes(rowKey), System.currentTimeMillis());
for (Entry<String, String> entry2 : keyValue.entrySet()) {
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(entry2.getKey()), Bytes.toBytes(entry2.getValue()));
}
puts.add(put);
}
table.put(puts);
table.close();
keyValues.clear();
}
/**
* 单行删除
* @param tableName
* @param rowKey
* @param family
* @throws IOException
*/
public static void deleteByRowKey(String tableName, String rowKey) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Delete delete = new Delete(Bytes.toBytes(rowKey));
table.delete(delete);
table.close();
}
/**
* 查询所有
* @param tableName
* @param family
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForScan(String tableName, String family) throws IOException {
List<Map<String, String>> rows = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(family));
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell)));
}
rows.add(row);
}
} finally {
rs.close();
}
return rows;
}
/**
* 查询所有rowKey
* @param tableName
* @return
* @throws IOException
*/
public static List<String> queryAllRowKeyForScan(String tableName) throws IOException {
List<String> result = new ArrayList<>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
if (null == cells || cells.length <= 0) {
continue;
}
Cell cell = cells[0];
String rowKey = new String(CellUtil.cloneRow(cell));
result.add(rowKey);
}
} finally {
rs.close();
}
return result;
}
/**
* 查询所有字段
* @param tableName
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForScan(String tableName) throws IOException {
List<Map<String, String>> rows = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put("rowKey", new String(CellUtil.cloneRow(cell)));
row.put("family", new String(CellUtil.cloneFamily(cell)));
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell)));
}
rows.add(row);
}
} finally {
rs.close();
}
return rows;
}
/**
* 根据时间范围
* @param tableName
* @param family
* @param minStamp
* @param maxStamp
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForTimeRange(String tableName, String family, long minStamp, long maxStamp) throws IOException {
List<Map<String, String>> rows = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(family));
scan.setTimeRange(minStamp, maxStamp);
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell)));
}
rows.add(row);
}
} finally {
rs.close();
}
return rows;
}
/**
* 根据RowKey查询
* @param tableName
* @param rowKey
* @param family
* @return
* @throws IOException
*/
public static Map<String, String> queryForRowKey(String tableName, String rowKey, String family) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Get get = new Get(Bytes.toBytes(rowKey));
get.addFamily(Bytes.toBytes(family));
Scan scan = new Scan(get);
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell),"UTF-8"));
}
}
} finally {
rs.close();
}
return row;
}
/**
* 根据多个RowKey查询
* @param tableName
* @param rowKeys
* @param family
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForRowKeys(String tableName, List<String> rowKeys, String family) throws IOException {
List<Map<String, String>> resultList = new ArrayList<>();
Table table = connection.getTable(TableName.valueOf(tableName));
List<Get> getList = new ArrayList();
for (String rowKey : rowKeys) {
Get get = new Get(Bytes.toBytes(rowKey));
get.addFamily(Bytes.toBytes(family));
getList.add(get);
}
Result[] results = table.get(getList);
for (Result result : results){//对返回的结果集进行操作
Map<String, String> row = new HashMap<String, String>();
for (Cell kv : result.rawCells()) {
row.put(new String(CellUtil.cloneQualifier(kv)), new String(CellUtil.cloneValue(kv),"UTF-8"));
}
resultList.add(row);
}
return resultList;
}
/**
* 根据RowKey范围查询
* @param tableName
* @param family
* @param startRow
* @param stopRow
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForRowKeyRange(String tableName, String family, String startRow, String stopRow) throws IOException {
List<Map<String, String>> rows = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(family));
scan.setStartRow(Bytes.toBytes(startRow));
scan.setStopRow(Bytes.toBytes(stopRow));
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put("timestamp", cell.getTimestamp() + "");
row.put("rowKey", new String(CellUtil.cloneRow(cell)));
row.put("family", new String(CellUtil.cloneFamily(cell)));
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell)));
}
rows.add(row);
}
} finally {
rs.close();
}
return rows;
}
/**
* 根据指定列名匹配列值
* @param tableName
* @param family
* @param qualifier
* @param value
* @return
* @throws IOException
*/
public static Collection<Map<String, String>> queryForQuilfier(String tableName, String family, String column, String value) throws IOException {
Map<String, Map<String, String>> rows = new HashMap<String, Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
SubstringComparator comp = new SubstringComparator(value);
SingleColumnValueFilter filter = new SingleColumnValueFilter(family.getBytes(), column.getBytes(), CompareOp.EQUAL, comp);
filter.setFilterIfMissing(true);
PageFilter p = new PageFilter(5);
scan.setFilter(filter);
scan.setFilter(p);
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
String rowKey = new String(CellUtil.cloneRow(cell));
if (null == row || !rows.containsKey(rowKey)) {
row = new HashMap<String, String>();
}
row.put("timestamp", cell.getTimestamp() + "");
row.put("rowKey", rowKey);
row.put("family", new String(CellUtil.cloneFamily(cell)));
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell),"UTF-8"));
rows.put(rowKey,row);
}
}
} finally {
rs.close();
}
return rows.values();
}
/**
* 根据指定列名完全匹配列值
* @param tableName
* @param family
* @param qualifier
* @param value
* @return
* @throws IOException
*/
public static Collection<Map<String, String>> queryForQuilfierExactly(String tableName, String family, String column, String value) throws IOException {
Map<String, Map<String, String>> rows = new HashMap<String, Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
SingleColumnValueFilter filter = new SingleColumnValueFilter(family.getBytes(), column.getBytes(), CompareOp.EQUAL,value.getBytes());
filter.setFilterIfMissing(true);
scan.setFilter(filter);
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
String rowKey = new String(CellUtil.cloneRow(cell));
if (null == row || !rows.containsKey(rowKey)) {
row = new HashMap<String, String>();
}
row.put("timestamp", cell.getTimestamp() + "");
row.put("rowKey", rowKey);
row.put("family", new String(CellUtil.cloneFamily(cell)));
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell),"UTF-8"));
rows.put(rowKey,row);
}
}
} finally {
rs.close();
}
return rows.values();
}
/**
* 获取列名匹配的rowkey
* @param tableName
* @param qualifier 列名
* @param pageSize 数量
* @return
* @throws IOException
*/
public static List<String> queryForQuilfierName(String tableName, String qualifier, long pageSize) throws IOException {
Set<String> rowKeys = new HashSet<>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
Filter filter = new QualifierFilter(CompareOp.EQUAL,new BinaryComparator(Bytes.toBytes(qualifier)));
PageFilter pageFilter = new PageFilter(pageSize);
FilterList filterList = new FilterList();
filterList.addFilter(filter);
filterList.addFilter(pageFilter);
scan.setFilter(filterList);
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
String rowKey = new String(CellUtil.cloneRow(cell));
rowKeys.add(rowKey);
}
}
} finally {
rs.close();
}
List<String> rows = new ArrayList<>(rowKeys);
return rows;
}
/**
* 获取多个列值匹配的rowkey
* @param tableName
* @param family
* @param qualifierMap 列名:列值
* @param pageSize 数量
* @return
* @throws IOException
*/
public static List<String> queryForMultiQuilfierName(String tableName, String family, Map<String, String> qualifierMap, long pageSize) throws IOException {
Set<String> rowKeys = new HashSet<>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
for (Entry<String, String> entry : qualifierMap.entrySet()) {
SingleColumnValueFilter filter = new SingleColumnValueFilter(family.getBytes(), entry.getKey().getBytes(), CompareOp.EQUAL,entry.getValue().getBytes());
filter.setFilterIfMissing(true);
filterList.addFilter(filter);
}
PageFilter pageFilter = new PageFilter(pageSize);
filterList.addFilter(pageFilter);
scan.setFilter(filterList);
scan.addFamily(Bytes.toBytes(family));
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
String rowKey = new String(CellUtil.cloneRow(cell));
rowKeys.add(rowKey);
}
}
} finally {
rs.close();
}
List<String> rows = new ArrayList<>(rowKeys);
return rows;
}
public static void qualifierFilter() throws IOException {
Table mTable = connection.getTable(TableName.valueOf("portrait"));
QualifierFilter columnsNameFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator("basictag-14".getBytes()));
Scan scan = new Scan();
scan.setFilter(columnsNameFilter);
ResultScanner rs = mTable.getScanner(scan);
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
System.out.println("==== "+new String(CellUtil.cloneRow(cell))+"\t"+
new String(CellUtil.cloneFamily(cell))+"\t"+
new String(CellUtil.cloneQualifier(cell))+"\t"+
new String(CellUtil.cloneValue(cell)));
}
}
}
/**
* 获取列名匹配的rowkey
* @param tableName
* @param qualifier 列名
* @param pageSize 数量
* @return
* @throws IOException
*/
public static List<String> queryForQuilfierName(String tableName, String family, String qualifier, long pageSize) throws IOException {
Set<String> rowKeys = new HashSet<>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
Filter qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL,new BinaryComparator(qualifier.getBytes()));
PageFilter pageFilter = new PageFilter(pageSize);
FilterList list = new FilterList();
list.addFilter(pageFilter);
list.addFilter(qualifierFilter);
scan.addFamily(Bytes.toBytes(family));
scan.setFilter(list);
System.out.println(scan.toJSON());
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
System.out.println("==== "+new String(CellUtil.cloneRow(cell))+"\t"+
new String(CellUtil.cloneFamily(cell))+"\t"+
new String(CellUtil.cloneQualifier(cell))+"\t"+
new String(CellUtil.cloneValue(cell)));
String rowKey = new String(CellUtil.cloneRow(cell));
rowKeys.add(rowKey);
}
}
} finally {
rs.close();
}
List<String> rows = new ArrayList<>(rowKeys);
return rows;
}
/**
* 根据指定列名匹配列值条数
* @param tableName
* @param family
* @param qualifier
* @param value
* @return
* @throws IOException
*/
public static long queryForQuilfierCount(String tableName, String family, String column, String value) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
SubstringComparator comp = new SubstringComparator(value);
SingleColumnValueFilter filter = new SingleColumnValueFilter(family.getBytes(), column.getBytes(), CompareOp.EQUAL, comp);
filter.setFilterIfMissing(true);
scan.setFilter(filter);
ResultScanner rs = table.getScanner(scan);
long count = 0;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
count++;
}
} finally {
rs.close();
}
return count;
}
/*
* 关闭连接
*
*/
public void close() {
/**
* close connection
**/
if (connection != null) {
try {
connection.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}