This article is hbase table operations, data manipulation, data query filtering, or if there is knowledge of the ADO JDBC, easily understood HBASE API.
hbase version is 2.0.
1, in order to facilitate the first part of the code of helper paste (the end the complete test code Git), mainly in order to reuse Connection.
public class HBaseHelper implements Closeable {
private Configuration configuration = null;
private Connection connection = null;
private Admin admin = null;
private HBaseHelper(Configuration configuration) throws IOException {
this.configuration = configuration;
this.connection = ConnectionFactory.createConnection(this.configuration);
admin = this.connection.getAdmin();
}
public static HBaseHelper getHBaseHelper(Configuration configuration) throws IOException {
return new HBaseHelper(configuration);
}
@Override
public void close() throws IOException {
admin.close();
connection.close();
}
public Connection getConnection() {
return connection;
}
public Configuration getConfiguration() {
return configuration;
}
... ...
｝
Initialization to initialize hbase arranged and connected hbase, obtaining hbase auxiliary operation based HbaseHelper herein.
//初始化
private void setUp() throws IOException{
conf = HBaseConfiguration.create();
conf.set("hbase.master","192.168.31.10");
//The port the HBase Master should bind to.
// conf.set("hbase.master.port","16000");
//The port for the HBase Master web UI. Set to -1 if you do not want a UI instance run.
// conf.set("hbase.master.info.port","16010");
//The port the HBase RegionServer binds to.
// conf.set("hbase.regionserver.port","16020");
//The port for the HBase RegionServer web UI Set to -1 if you do not want the RegionServer UI to run.
// conf.set("hbase.regionserver.info.port","16030");
conf.set("hbase.zookeeper.quorum","192.168.31.10");
//Property from ZooKeeper’s config zoo.cfg. The port at which the clients will connect.
// HBase数据库使用的端口
//conf.set("hbase.zookeeper.property.clientPort", "2181");
//单机
conf.set("hbase.rootdir","file:///opt/hbase_data");
conf.set("hbase.zookeeper.property.dataDir","/opt/hbase_data/zookeeper");
helper = HBaseHelper.getHBaseHelper(conf);
}
2, namespaces, table creation, deletion, exist, etc.
public void createNamespace(String namespace) {
try {
NamespaceDescriptor nd = NamespaceDescriptor.create(namespace).build();
admin.createNamespace(nd);
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
}
}
public void dropNamespace(String namespace, boolean force) {
try {
if (force) {
TableName[] tableNames = admin.listTableNamesByNamespace(namespace);
for (TableName name : tableNames) {
admin.disableTable(name);
admin.deleteTable(name);
}
}
} catch (Exception e) {
// ignore
}
try {
admin.deleteNamespace(namespace);
} catch (IOException e) {
System.err.println("Error: " + e.getMessage());
}
}
public boolean existsTable(String table)
throws IOException {
return existsTable(TableName.valueOf(table));
}
public boolean existsTable(TableName table)
throws IOException {
return admin.tableExists(table);
}
public void createTable(String table, String... colfams)
throws IOException {
createTable(TableName.valueOf(table), 1, null, colfams);
}
public void createTable(TableName table, String... colfams)
throws IOException {
createTable(table, 1, null, colfams);
}
public void createTable(String table, int maxVersions, String... colfams)
throws IOException {
createTable(TableName.valueOf(table), maxVersions, null, colfams);
}
public void createTable(TableName table, int maxVersions, String... colfams)
throws IOException {
createTable(table, maxVersions, null, colfams);
}
public void createTable(String table, byte[][] splitKeys, String... colfams)
throws IOException {
createTable(TableName.valueOf(table), 1, splitKeys, colfams);
}
void the createTable public (the TableName Table, int maxVersions, byte [] [] splitKeys,
String ... colfams)
throws IOException {
// constructor descriptor table
TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder (Table);
// Constructor column family described
cfDescBuilder ColumnFamilyDescriptorBuilder;
// column family descriptor
ColumnFamilyDescriptor cfDesc;
for (CF2 String: colfams) {
cfDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(cf));
cfDescBuilder.setMaxVersions (maxVersions);
cfDesc cfDescBuilder.build = ();
tableDescriptorBuilder.setColumnFamily (cfDesc);
}
// get descriptor table
TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
if (splitKeys != null) {
admin.createTable(tableDescriptor, splitKeys);
} else {
admin.createTable(tableDescriptor);
}
}
//禁用表
public void disableTable(String table) throws IOException {
disableTable(TableName.valueOf(table));
}
public void disableTable(TableName table) throws IOException {
admin.disableTable(table);
}
public void dropTable(String table) throws IOException {
dropTable(TableName.valueOf(table));
}
//删除前，先禁用表
public void dropTable(TableName table) throws IOException {
if (existsTable(table)) {
if (admin.isTableEnabled(table)) disableTable(table);
admin.deleteTable(table);
}
}
Example:
//插入testtable表数据
private void initTestTable() throws IOException{
String tableNameString = "testtable";
if(helper.existsTable(tableNameString)){
helper.disableTable(tableNameString);
helper.dropTable(tableNameString);
}
helper.createTable(tableNameString,"info","ex","memo");
helper.put(tableNameString,"row1","info","username","admin");
helper.put(tableNameString,"row1","ex","addr","北京大道");
helper.put(tableNameString,"row1","memo","detail","超级用户，地址：北京大道");
helper.PUT (tableNameString, "ROW2", "info", "username", "the Guest");
helper.put(tableNameString,"row2","ex","addr","全国各地");
helper.put (tableNameString, "row2", "memo", "detail", " tourists, Address: National everywhere");
helper.close ();
}
2, the insertion (or updated) data
public void put(String table, String row, String fam, String qual,
String val) throws IOException {
put(TableName.valueOf(table), row, fam, qual, val);
}
//插入或更新单行
public void put(TableName table, String row, String fam, String qual,
String val) throws IOException {
Table tbl = connection.getTable(table);
Put put = new Put(Bytes.toBytes(row));
put.addColumn(Bytes.toBytes(fam), Bytes.toBytes(qual), Bytes.toBytes(val));
tbl.put(put);
tbl.close();
}
public void put(String table, String row, String fam, String qual, long ts,
Val String) throws IOException {
PUT (TableName.valueOf (Table), Row, FAM, Qual, TS, Val);
}
// update or insert a single line with a time stamp
public void put (TableName table, String row, String fam, String Qual, TS Long,
String Val) throws IOException {
the Table TBL = Connection.GetTable (Table);
of Put PUT of Put new new = (Bytes.toBytes (Row));
put.addColumn (Bytes.toBytes (FAM), Bytes.toBytes ( Qual), TS,
Bytes.toBytes (Val));
tbl.put (PUT);
tbl.close ();
}
// a rowKey insert or update data, there is a rowKey a Put, there may be a plurality of columns and group column name
public void put (String tableNameString, Put put) throws IOException {
the tableName tableName = TableName.valueOf (tableNameString);
Table table = connection.getTable(tableName);
if (put != null && put.size() > 0) {
table.put(put);
}
table.close();
}
2.1, bulk insert, assembled according to the actual traffic data, is finally put into use API list
//批量插入数据,list里每个map就是一条数据，并且按照rowKey columnFamily columnName columnValue放入map的key和value
public void bulkInsert(String tableNameString, List<Map<String, Object>> list) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableNameString));
List<Put> puts = new ArrayList<Put>();
if (list != null && list.size() > 0) {
for (Map<String, Object> map : list) {
Put put = new Put(Bytes.toBytes(map.get("rowKey").toString()));
put.addColumn(Bytes.toBytes(map.get("columnFamily").toString()),
Bytes.toBytes(map.get("columnName").toString()),
Bytes.toBytes(map.get("columnValue").toString()));
puts.add(put);
}
}
table.put(puts);
table.close();
}
//批量插入，外部组装put放入list
public void bulkInsert2(String tableNameString, List<Put> puts) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableNameString));
if (puts != null && puts.size() > 0) {
table.put(puts);
}
table.close();
}
Example:
//批量插入
private void bulkInsertTestTable() throws IOException{
String tableNameString = "testtable";
if(!helper.existsTable(tableNameString)){
helper.createTable(tableNameString,"info","ex","memo");
}
System.out.println(".........批量插入数据start.........");
List<Map<String,Object>> mapList = new ArrayList<>();
for(int i=1;i<201;i++){
Map<String,Object> map = new HashMap<>();
map.put("rowKey","testKey"+i);
map.put("columnFamily","info");
map.put("columnName","username");
map.put("columnValue","guest"+i);
map.put("rowKey","testKey"+i);
map.put("columnFamily","ex");
map.put("columnName","addr");
map.put("columnValue","北京路"+i+"号");
map.put("rowKey","testKey"+i);
map.put("columnFamily","memo");
map.put("columnName","detail");
map.put("columnValue","联合国地球村北京路第"+i+"号");
mapList.add(map);
}
helper.bulkInsert(tableNameString,mapList);
System.out.println(".........批量插入数据end.........");
}
//批量插入2
private void insertByRowKey(String table,String rowKey) throws IOException{
Put put = new Put(Bytes.toBytes(rowKey));
String columnFamily ;
String columnName ;
String columnValue ;
for(int i=0;i<10;i++){
columnFamily = "info";
columnName = "username"+i;
columnValue = "user111";
put.addColumn(Bytes.toBytes(columnFamily),Bytes.toBytes(columnName),Bytes.toBytes(columnValue));
columnFamily = "ex";
columnName = "addr"+i;
columnValue = "street 111";
put.addColumn(Bytes.toBytes(columnFamily),Bytes.toBytes(columnName),Bytes.toBytes(columnValue));
columnFamily = "memo";
columnName = "detail"+i;
columnValue = "sssss zzz 111222 ";
put.addColumn(Bytes.toBytes(columnFamily),Bytes.toBytes(columnName),Bytes.toBytes(columnValue));
}
System.out.println("----> put size:"+put.size());
helper.put(table,put);
}
private void bulkInsertTestTable2(String tableNameString) throws IOException{
// String tableNameString = "testtable";
if(!helper.existsTable(tableNameString)){
helper.createTable(tableNameString,"info","ex","memo");
}
List<Put> puts = new ArrayList<>();
for(int i=0;i<10;i++){
String rowKey = "rowKey"+i;
Put put = new Put(Bytes.toBytes(rowKey));
String columnFamily = "info";
String columnName = "username2";
String columnValue = "user"+i;
put.addColumn(Bytes.toBytes(columnFamily),Bytes.toBytes(columnName),Bytes.toBytes(columnValue));
columnFamily = "ex";
columnName = "addr2";
columnValue = "street "+i;
put.addColumn(Bytes.toBytes(columnFamily),Bytes.toBytes(columnName),Bytes.toBytes(columnValue));
columnFamily = "memo";
columnName = "detail2";
columnValue = "aazzdd "+i;
put.addColumn(Bytes.toBytes(columnFamily),Bytes.toBytes(columnName),Bytes.toBytes(columnValue));
System.out.println("put size:"+put.size());
puts.add(put);
}
helper.bulkInsert2(tableNameString,puts);
}
3, delete data, since the data is hbase three dimensions, has a variety of operation data is deleted
//根据rowKey删除所有行数据
public void deleteByKey(String tableNameString,String rowKey) throws IOException{
Table table = connection.getTable(TableName.valueOf(tableNameString));
Delete delete = new Delete(Bytes.toBytes(rowKey));
table.delete(delete);
table.close();
}
//根据rowKey和列族删除所有行数据
public void deleteByKeyAndFamily(String tableNameString,String rowKey,String columnFamily) throws IOException{
Table table = connection.getTable(TableName.valueOf(tableNameString));
Delete delete = new Delete(Bytes.toBytes(rowKey));
delete.addFamily(Bytes.toBytes(columnFamily));
table.delete(delete);
table.close();
}
//根据rowKey、列族删除多个列的数据
public void deleteByKeyAndFC(String tableNameString,String rowKey,
String columnFamily,List<String> columnNames) throws IOException{
Table table = connection.getTable(TableName.valueOf(tableNameString));
Delete delete = new Delete(Bytes.toBytes(rowKey));
for(String columnName:columnNames){
delete.addColumns(Bytes.toBytes(columnFamily),Bytes.toBytes(columnName));
}
table.delete(delete);
table.close();
}
4, basic query, only to note that the cell where the value must be offset and length to take
//根据rowkey，获取所有列族和列数据
public List<Cell> getRowByKey(String tableNameString,String rowKey) throws IOException{
Table table = connection.getTable(TableName.valueOf(tableNameString));
Get get = new Get(Bytes.toBytes(rowKey));
Result result = table.get(get);
// Cell[] cells = result.rawCells();
List<Cell> list = result.listCells();
table.close();
return list;
}
// Array taken from Cell length and displacement to be added, or data is not correct
public void dumpResult (the Result Result) {
for (Cell Cell: result.rawCells ()) {
System.out.println ( "Cell:" + Cell +
", the Value:" + Bytes.toString (cell.getValueArray (),
cell.getValueOffset (), cell.getValueLength ()));
}
}
5, filter, this is an important part of the query HBASE
5.1 The filtered rowKey
// The rowKey filtering data, using regular expressions rowKey
// returns the value pair rowKey Cells and
public the Map <String, List <>> filterByRowKeyRegex the Cell (tableNameString String, String rowKey, CompareOperator operator) throws IOException {
the Table Table = Connection.GetTable (TableName.valueOf (tableNameString));
Scan Scan Scan new new = ();
// use regular
the RowFilter the RowFilter new new filter = (operator, new new RegexStringComparator (rowKey));
// substring matching comprises a case-insensitive .
= New new filter the RowFilter the RowFilter // (operator, new new SubstringComparator (rowKey));
scan.setFilter (filter);
ResultScanner = table.getScanner Scanner (Scan);
the Map <String, List <>> the Cell Map new new = the HashMap <> ( );
for(Result result:scanner){
map.put(Bytes.toString(result.getRow()),result.listCells());
}
table.close();
return map;
}
5.2, a column value, the value of the positive column filtration, etc.
// (support regular) to find family according to the column, the column name, column data value
// Return Value: If the query to the value of each column family will return in all matches rowKey, (when all the data even if the query column names of these group columns and column names do not match)
public the Map <String, List <>> filterByValueRegex the Cell (tableNameString String, String family, colName String,
String value, CompareOperator operator) throws IOException {
the Table Table Connection.GetTable = (TableName.valueOf ( tableNameString));
Scan Scan Scan new new = ();
// regular matching
SingleColumnValueFilter = new new filter SingleColumnValueFilter (Bytes.toBytes (Family),
Bytes.toBytes (colName), operator, new new RegexStringComparator (value));
// exact match
/ / SingleColumnValueFilter filter = new SingleColumnValueFilter (Bytes.toBytes (family),
Bytes.toBytes // (colName), operator, Bytes.toBytes (value));
// SingleColumnValueExcludeFilter exclusion column value
// column to filter must be present, if not, then these columns will be returned data does not exist. If you do not want to return the data set is setFilterIfMissing to true
filter.setFilterIfMissing (to true);
scan.setFilter (filter);
ResultScanner = table.getScanner Scanner (Scan);
the Map <String, List <the Cell >> the Map = new new HashMap <> ();
for (the Result Result: Scanner) {
map.put (Bytes.toString (result.getRow ()), result.listCells ());
}
return Map;
}
5.3, according to the column name prefix, a regular column names, a plurality of column names like filter
// The data column name prefix filtering
public the Map <String, List <>> filterByColumnPrefix the Cell (tableNameString String, String prefix) throws IOException {
the Table Table Connection.GetTable = (TableName.valueOf (tableNameString));
// column name prefix match
filter = new new ColumnPrefixFilter ColumnPrefixFilter (Bytes.toBytes (prefix));
// QualifierFilter diversity for column names that match the filter
// QualifierFilter = new new filter QualifierFilter (CompareOperator.EQUAL, new new SubstringComparator (prefix));
// a plurality of column names prefix matching
// new new MultipleColumnPrefixFilter MultipleColumnPrefixFilter Multifilter = (new new byte [] [] {});
Scan Scan Scan new new = ();
scan.setFilter (filter);
ResultScanner = table.getScanner Scanner (Scan);
Map<String,List<Cell>> map = new HashMap<>();
for(Result result:scanner){
map.put(Bytes.toString(result.getRow()),result.listCells());
}
return map;
}
5.4, a set of filters, a plurality of filters to filter press by policy
// The column name and column name prefix filtering range data
public the Map <String, List <>> filterByPrefixAndRange the Cell (String tableNameString, colPrefix String,
String MinCol, String maxCol) throws IOException {
the Table Table Connection.GetTable = (TableName.valueOf ( tableNameString));
// column name prefix match
ColumnPrefixFilter = new new filter ColumnPrefixFilter (Bytes.toBytes (colPrefix));
// the name of the column scanning range, the lower range comprises
ColumnRangeFilter rangeFilter = new ColumnRangeFilter (Bytes.toBytes ( minCol), true,
Bytes.toBytes (maxCol), to true);
FilterList FilterList = new new FilterList (FilterList.Operator.MUST_PASS_ALL);
filterList.addFilter (filter);
filterList.addFilter(rangeFilter);
Scan scan = new Scan();
scan.setFilter(filterList);
ResultScanner scanner = table.getScanner(scan);
Map<String,List<Cell>> map = new HashMap<>();
for(Result result:scanner){
map.put(Bytes.toString(result.getRow()),result.listCells());
}
return map;
}
6, the filter Introduction
6.1, comparison operations, such as equal to, greater than, less than
public enum CompareOperator {
// Keeps same names as the enums over in filter's CompareOp intentionally.
// The convertion of operator to protobuf representation is via a name comparison.
/** less than */
LESS,
/** less than or equal to */
LESS_OR_EQUAL,
/** equals */
EQUAL,
/** not equal */
NOT_EQUAL,
/** greater than or equal to */
GREATER_OR_EQUAL,
/** greater than */
GREATER,
/** no operation */
NO_OP,
}
6.2, comparators, mainly inherited ByteArrayComparable class
RegexStringComparator support the value of the expression being compared Scan Scan = new new Scan (); RegexStringComparator CoMP = new new RegexStringComparator ( "you."); // at the beginning of the string you SingleColumnValueFilter filter = new SingleColumnValueFilter (Bytes.toBytes ( "family") , Bytes.toBytes ( "qualifier on"), CompareOp.EQUAL, CoMP); scan.setFilter (filter);
SubStringComparator for determining whether there is a substring in value, and is not case sensitive. = New new Scan Scan Scan (); SubstringComparator new new SubstringComparator CoMP = ( "substr"); // Find the string contains SingleColumnValueFilter filter = new SingleColumnValueFilter (Bytes.toBytes ( "family"), Bytes.toBytes ( "qualifier"), CompareOp.EQUAL, CoMP); scan.setFilter (filter);
BinaryComparator binary comparator, not byte deserialization direct comparison, more efficient. = New new Scan Scan Scan (); BinaryComparator CoMP = new new BinaryComparator (Bytes.toBytes ( "My HBase")); ValueFilter = new new filter ValueFilter (CompareOp.EQUAL, CoMP); scan.setFilter (filter);
BinaryPrefixComparator prefix binary comparator. Only relatively prefixes are the same. = New new Scan Scan Scan (); BinaryPrefixComparator CoMP = new new BinaryPrefixComparator (Bytes.toBytes ( "Test")); // SingleColumnValueFilter = new new filter SingleColumnValueFilter (Bytes.toBytes ( "Family"), Bytes.toBytes ( "qualifier on"), CompareOp.EQUAL, CoMP); scan.setFilter (filter);
Note: BitComparator, RegexStringComparator, SubStringComparator only with EQUAL and NOT_EQUAL with the use, because the return of these comparators when compareTo () method to match 0, 1 time does not match the return, and if an error occurs with the GREATER or LESS.
String-based comparator is slower than byte-based comparator consumes more resources.
6.3, filter, section describes
OK key filter RowFilter filter a row. = New new Scan Scan Scan (); the RowFilter the RowFilter new new filter = (CompareOp.EQUAL, new new BinaryComparator (Bytes.toBytes ( "ROW1"))); scan.setFilter (filter);
Column family filter FamilyFilter for filtering column family (this function may be realized by setting some of the columns in the group during Scan) Scan Scan Scan new new = (); FamilyFilter = new new FamilyFilter filter (CompareOp.EQUAL, new new BinaryComparator ( Bytes.toBytes ( "info"))) ; // column family is info scan.setFilter (filter);
Column name filter
QualifierFilter column names match-
Scan Scan Scan new new = ();
QualifierFilter = new new filter QualifierFilter (CompareOp.EQUAL, new new BinaryComparator (Bytes.toBytes ( "username"))); // column called username
scan.setFilter (filter);
ColumnPrefixFilter for the column name (Qualifier) prefix filter that contains all of the column names a prefix.
= New new Scan Scan Scan ();
ColumnPrefixFilter = new new filter ColumnPrefixFilter (Bytes.toBytes ( "addr")); // prefix addr
scan.setFilter (filter);
MultipleColumnPrefixFilter
similar to MultipleColumnPrefixFilter ColumnPrefixFilter behavior, but can specify a plurality of columns name (Qualifier) prefix.
= New new Scan Scan Scan ();
byte [] [] = new new prefixes byte [] [] {Bytes.toBytes ( "My--prefix. 1"), Bytes.toBytes ( "My-prefix-2")};
Filter = new new MultipleColumnPrefixFilter MultipleColumnPrefixFilter (prefixes);,
scan.setFilter (filter);
ColumnRangeFilter column names range filter for efficient internal scan column names. Keywords: sorted
Scan Scan = new new Scan ();
ColumnRangeFilter filter = new new ColumnRangeFilter (Bytes.toBytes ( "minColumn"), to true, Bytes.toBytes ( "maxColumn"), false);
scan.setFilter (filter);
DependentColumnFilter try to find where each row of the column, and returns the rows with the same time stamp of all the key pair.
= New new Scan Scan Scan ();
DependentColumnFilter = new new filter DependentColumnFilter (Bytes.toBytes ( "Family"), Bytes.toBytes ( "qualifier on"));
scan.setFilter (filter);
Filter column value comparison SingleColumnValueFilter column value column values and character strings in the column username column family info "admin" equal data: Scan Scan Scan new new = (); SingleColumnValueFilter = new new filter SingleColumnValueFilter (Bytes.toBytes ( "info") , Bytes.toBytes ( "username"), CompareOp.EQUAL, Bytes.toBytes ( "ADMIN")); scan.setFilter (filter);
6.4, the code:
