HBase常用API操作

package test;

import java.io.IOException;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.List;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Operation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.FamilyFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.filter.RegexStringComparator;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator;
import org.apache.hadoop.hbase.filter.ValueFilter;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.RawComparator;
import org.junit.Before;
import org.junit.Test;


public class HBaseClient {
	
	private Configuration hbaseConf;
	private Connection hbaseConn;
	private Admin hbaseAdmin;
	
	@Before
	public void initHBase() throws Exception{
		hbaseConf = HBaseConfiguration.create();
		hbaseConn = ConnectionFactory.createConnection(hbaseConf);
		System.out.println("连接上了？" + !hbaseConn.isClosed());
		hbaseAdmin = hbaseConn.getAdmin();
	}
	
	/**
	 * 表是否存在
	 * @throws IOException
	 */
	@Test
	public void testExists() throws IOException {
		System.out.println(hbaseAdmin.tableExists(TableName.valueOf("mytest")));
	}
	
	/**
	 * hbase表的创建
	 * @throws IOException
	 */
	@Test
	public void testCreate() throws IOException {
		TableName tableName = TableName.valueOf("mytest");
		if(!hbaseAdmin.tableExists(tableName)) {
			HTableDescriptor tableDesc = new HTableDescriptor(tableName);
			HColumnDescriptor familyDesc1 = new HColumnDescriptor(Bytes.toBytes("info1"));
			familyDesc1.setMaxVersions(3);
			HColumnDescriptor familyDesc2 = new HColumnDescriptor(Bytes.toBytes("info2"));
			tableDesc.addFamily(familyDesc1);
			tableDesc.addFamily(familyDesc2);
			hbaseAdmin.createTable(tableDesc);
		}else {
			System.out.println("表已存在");
		}
	}
	
	/**
	 * 禁用
	 * @throws IOException
	 */
	@Test
	public void testDisable() throws IOException {
		TableName tableName = TableName.valueOf("mytest");
		hbaseAdmin.disableTable(tableName);
	}
	
	@Test
	public void testIsDisable() throws IOException {
		TableName tableName = TableName.valueOf("mytest");
		System.out.println(hbaseAdmin.isTableDisabled(tableName));
	}
	
	@Test
	public void testDrop() throws IOException {
		TableName tableName = TableName.valueOf("mytest");
		hbaseAdmin.deleteTable(tableName);
	}
	
	/**
	 * 一次增加一个单元格
	 * @throws IOException
	 */
	@Test
	public void testPut() throws IOException {
		TableName tableName = TableName.valueOf("mytest");
		Table table = hbaseConn.getTable(tableName);
		Put put = new Put(Bytes.toBytes("aa"));
		put.addColumn(Bytes.toBytes("info1"), Bytes.toBytes("name"), Bytes.toBytes("zhangfei"));
		table.put(put);
	}
	
	/**
	 * 一次增加多个单元格
	 * @throws IOException 
	 */
	@Test
	public void testPutBatch() throws IOException {
		TableName tableName = TableName.valueOf("mytest");
		Table table = hbaseConn.getTable(tableName);
		List<Put> list = new ArrayList<>();
		Put put1 = new Put(Bytes.toBytes("aa"));
		put1.addColumn(Bytes.toBytes("info1"), Bytes.toBytes("name"), Bytes.toBytes("guanYu"));
		Put put2 = new Put(Bytes.toBytes("aa"));
		put2.addColumn(Bytes.toBytes("info1"), Bytes.toBytes("age"), Bytes.toBytes("20"));
		Put put3 = new Put(Bytes.toBytes("aa"));
		put3.addColumn(Bytes.toBytes("info2"), Bytes.toBytes("name"), Bytes.toBytes("liuBei"));
		Put put4 = new Put(Bytes.toBytes("aa"));
		put4.addColumn(Bytes.toBytes("info1"), Bytes.toBytes("otherName"), Bytes.toBytes("guan2"));
		
		list.add(put1);
		list.add(put2);
		list.add(put3);
		list.add(put4);
		
		table.put(list);
	}
	
	/**
	 * 删除一个单元格
	 * @throws IOException 
	 */
	@Test
	public void testDelete() throws IOException {
		TableName tableName = TableName.valueOf("mytest");
		Table table = hbaseConn.getTable(tableName);
		Delete delete = new Delete(Bytes.toBytes("aa"));
		delete.addColumn(Bytes.toBytes("info1"), Bytes.toBytes("otherName"));
		table.delete(delete);
	}
	
	/**
	 * 查询一行数据(一个rowKey对应的数据),hbase shell中get操作
	 * @throws IOException
	 */
	@Test
	public void testGet() throws IOException {
		TableName tableName = TableName.valueOf("mytest");
		Table table = hbaseConn.getTable(tableName);
		Get get = new Get(Bytes.toBytes("aa"));
//		get.addColumn(Bytes.toBytes("info1"), Bytes.toBytes("name"));
		get.addFamily(Bytes.toBytes("info1"));
		Result result = table.get(get);
		
		for(Cell cell : result.rawCells()) {
			System.out.print("行键:"+ Bytes.toString(CellUtil.cloneRow(cell))+"\t");
			System.out.print("列族:"+ Bytes.toString(CellUtil.cloneFamily(cell))+"\t");
			System.out.print("列标识:"+ Bytes.toString(CellUtil.cloneQualifier(cell))+"\t");
			System.out.println("值:"+ Bytes.toString(CellUtil.cloneValue(cell)));
		}
	}
	
	/**
	 * 生成测试数据,生成表test001
	 * @throws IOException 
	 */
	@Test
	public void testGeneData() throws IOException {
		List<Put> putList = new ArrayList<>();
		DecimalFormat formatter = new DecimalFormat("000");
		//创建表
		TableName tableName = TableName.valueOf("test001");
		if(hbaseAdmin.tableExists(tableName)) {
			hbaseAdmin.disableTable(tableName);
			hbaseAdmin.deleteTable(tableName);
			HTableDescriptor tableDesc = new HTableDescriptor(tableName);
			HColumnDescriptor familyDesc1 = new HColumnDescriptor(Bytes.toBytes("info1"));
			HColumnDescriptor familyDesc2 = new HColumnDescriptor(Bytes.toBytes("info2"));
			tableDesc.addFamily(familyDesc1);
			tableDesc.addFamily(familyDesc2);
			hbaseAdmin.createTable(tableDesc);
		}
		//生成数据
		HTable table = (HTable)hbaseConn.getTable(tableName);
		//设置手动提交,设置缓冲区大小为100MB
		table.setAutoFlushTo(false);//put完成后得数据在缓冲区中，默认情况下自动刷新
		table.setWriteBufferSize(1024*1024*100);
		
		for(int i = 1;i<1000;i++) {
			String formatResult = formatter.format(i);
			Put put1 = new Put(Bytes.toBytes("row"+formatResult));
			put1.addColumn(Bytes.toBytes("info1"), Bytes.toBytes("name"), Bytes.toBytes("ming"+formatResult));
			Put put2 = new Put(Bytes.toBytes("row"+formatResult));
			put2.addColumn(Bytes.toBytes("info1"), Bytes.toBytes("sno"), Bytes.toBytes(formatResult));
			Put put3 = new Put(Bytes.toBytes("row"+formatResult));
			put3.addColumn(Bytes.toBytes("info1"), Bytes.toBytes("othername"), Bytes.toBytes("wang"+formatResult));
			Put put4 = new Put(Bytes.toBytes("row"+formatResult));
			put4.addColumn(Bytes.toBytes("info2"), Bytes.toBytes("name"), Bytes.toBytes("gao"+formatResult));
			
			putList.add(put1);
			putList.add(put2);
			putList.add(put3);
			putList.add(put4);
			
			if(putList.size()>=2000) {
				table.put(putList);
				table.flushCommits();
				putList.clear();
			}
		}
		table.put(putList);
		table.flushCommits();
		putList.clear();
	}
	
	/**
	 * 简单查询全表所有数据，hbase shell中scan操作
	 * @throws IOException 
	 */
	@Test
	public void testFullScan() throws IOException {
		TableName tableName = TableName.valueOf("test001");
		Table table = hbaseConn.getTable(tableName);
		Scan scan = new Scan();
		ResultScanner rs = table.getScanner(scan);
		
		this.showResult(rs);	
	} 
	
	/**
	 * 简单指定扫描条件，如扫描什么列族(什么列)，指定从哪一条记录开始，那一条记录结束
	 */
	@Test
	public void testFilterScan() throws Exception{
		TableName tableName = TableName.valueOf("test001");
		Table table = hbaseConn.getTable(tableName);
		Scan scan = new Scan();
		scan.withStartRow(Bytes.toBytes("row950"));//开始行包含
		scan.withStopRow(Bytes.toBytes("row969"));//结束行不包含
		
//		scan.addColumn(Bytes.toBytes("info1"), Bytes.toBytes("name"));
		scan.addFamily(Bytes.toBytes("info1"));
		
		ResultScanner rs = table.getScanner(scan);
		this.showResult(rs);
	}
	
	
	/**
	 * 行键过滤器扫描
	 * @throws IOException 
	 */
	@Test
	public void testRowFilterScan() throws IOException {
		TableName tableName = TableName.valueOf("test001");
		Table table = hbaseConn.getTable(tableName);
		Scan scan = new Scan();
		
		Filter filter = new RowFilter(
				CompareOp.EQUAL, 
				new RegexStringComparator("^\\w*99$") 
				);
		scan.setFilter(filter);
		
		this.showResult(table.getScanner(scan));
	}
	
	/**
	 * 列族过滤器
	 * @throws IOException 
	 */
	@Test
	public void testFamilyFilterScan() throws IOException {
		TableName tableName = TableName.valueOf("test001");
		Table table = hbaseConn.getTable(tableName);
		Scan scan = new Scan();
		scan.addFamily(Bytes.toBytes("info1"));
	
		Filter filter = new FamilyFilter(
				CompareOp.EQUAL, 
				new BinaryComparator(Bytes.toBytes("info2"))
				);
		scan.setFilter(filter);
		
		this.showResult(table.getScanner(scan));
	}
	
	/**
	 * 列标识名过滤器
	 * @throws IOException
	 */
	@Test
	public void testQualifierFilterScan() throws IOException {
		TableName tableName = TableName.valueOf("test001");
		Table table = hbaseConn.getTable(tableName);
		Scan scan = new Scan();
		
		Filter filter = new QualifierFilter(
				CompareOp.EQUAL, 
				new RegexStringComparator("^\\w*m\\w*$")
				);
		scan.setFilter(filter);
		
		this.showResult(table.getScanner(scan));		
	}
	
	/**
	 * 值过滤器，对所有列族下所有列的值进行过滤
	 * @throws IOException
	 */
	@Test
	public void testValueFilterScan() throws IOException {
		TableName tableName = TableName.valueOf("test001");
		Table table = hbaseConn.getTable(tableName);
		Scan scan = new Scan();
		scan.withStartRow(Bytes.toBytes("row900"));
		
//		Filter filter = new ValueFilter(
//				CompareOp.EQUAL, 
//				new RegexStringComparator("^\\d+")
//				);
		Filter filter = new ValueFilter(
				CompareOp.EQUAL, 
				new SubstringComparator("100")
				);
		scan.setFilter(filter);
		
		this.showResult(table.getScanner(scan));		
	}
	
	/**
	 * 单列值过滤器,按指定列的值进行过滤
	 * @throws IOException 
	 */
	@Test
	public void testSingleColumnValueFilteScan() throws IOException {
		TableName tableName = TableName.valueOf("test001");
		Table table = hbaseConn.getTable(tableName);
		Scan scan = new Scan();
		
		Filter filter = new SingleColumnValueFilter(
				Bytes.toBytes("info1"), 
				Bytes.toBytes("name"), 
				CompareOp.EQUAL, 
				new SubstringComparator("88"));
		scan.setFilter(filter);
		
		this.showResult(table.getScanner(scan));
	}
	
	/**
	 * 过滤器列表,过滤组合条件
	 * @throws IOException 
	 */
	@Test
	public void testFilterList() throws IOException {
		TableName tableName = TableName.valueOf("test001");
		Table table = hbaseConn.getTable(tableName);
		Scan scan = new Scan();
		
		//过滤出info1.name中含有8并且info.sno含有6，或者info1.name中含有6并且info2.name含有8
		Filter filter1 =new SingleColumnValueFilter(
				Bytes.toBytes("info1"), 
				Bytes.toBytes("name"), 
				CompareOp.EQUAL, 
				new SubstringComparator("8"));
		Filter filter2 =new SingleColumnValueFilter(
				Bytes.toBytes("info1"), 
				Bytes.toBytes("sno"), 
				CompareOp.EQUAL, 
				new SubstringComparator("6"));
		FilterList list1 = new FilterList(Operator.MUST_PASS_ALL);
		list1.addFilter(filter1);
		list1.addFilter(filter2);
		
		Filter filter3 =new SingleColumnValueFilter(
				Bytes.toBytes("info1"), 
				Bytes.toBytes("name"), 
				CompareOp.EQUAL, 
				new SubstringComparator("6"));
		Filter filter4 =new SingleColumnValueFilter(
				Bytes.toBytes("info2"), 
				Bytes.toBytes("name"), 
				CompareOp.EQUAL, 
				new SubstringComparator("8"));
		FilterList list2 = new FilterList(Operator.MUST_PASS_ALL);
		list2.addFilter(filter3);
		list2.addFilter(filter4);
		
		FilterList list = new FilterList(Operator.MUST_PASS_ONE);
		list.addFilter(list1);
		list.addFilter(list2);
		
		scan.setFilter(list);
		
		this.showResult(table.getScanner(scan));
		
	}
	
	
	/**
	 * 显示查询结果
	 * @param rs
	 */
	private void showResult(ResultScanner rs) {
		for(Result result : rs) {
			Cell[] cells = result.rawCells();
			
			for(Cell cell : cells) {
				System.out.print("rowkey:" + Bytes.toString(CellUtil.cloneRow(cell))+"\t");
				System.out.print("family:" + Bytes.toString(CellUtil.cloneFamily(cell))+"\t");
				System.out.print("qualifier:" + Bytes.toString(CellUtil.cloneQualifier(cell))+"\t");
				System.out.println("value:" + Bytes.toString(CellUtil.cloneValue(cell)));
			}
		}
	}
}