Big Data Performance Tuning
http://www.open-open.com/lib/view/open1417612091323.html
Several filters used in HBASE SingleColumnValueFilter, RowFilter, ValueFilter
http://www.tuicool .com/articles/2eiqMfz
SingleColumnValueFilter
filters how much a column value is greater than how much less than how much
List<Filter> filters = new ArrayList<Filter>();
filters.add( new SingleColumnValueFilter(Bytes.toBytes("pinfo"), //列族
Bytes.toBytes("t"), //列名
CompareOp.GREATER,Bytes.toBytes("1359901"]) ) //值
filters.add( new SingleColumnValueFilter(Bytes.toBytes("pinfo"),
Bytes.toBytes("t"),
CompareOp.LESS,Bytes.toBytes("1389901"]) )
FilterList filterList1 = new FilterList(Operator.MUST_PASS_ALL,filters);
sn.setFilter(filterList1);
1. Initialize Hbase
private Configuration conf = null;
HTable table = null;
private String tableName;
private static final Logger LOG = LoggerFactory.getLogger(HbaseUtils.class);
public HbaseUtils(PropertiesType propertiesType, String tableName) {
conf = HBaseConfiguration.create();
PropertiesUtil properties = new PropertiesUtil(propertiesType.getValue());
conf.set("hbase.zookeeper.quorum", properties.getValue("hbase.zookeeper.quorum"));
conf.set("hbase.zookeeper.property.clientPort", properties.getValue("hbase.zookeeper.property.clientPort"));
try {
table = new HTable(conf, Bytes.toBytes(tableName));
} catch (IOException e) {
LOG.error(e.getMessage());
}
this.tableName = tableName;
}
2. Under the specified conditions, press all the results of a field
/**
* Aggregate by a field under specified conditions
* @param paramMap parameter condition
* @param dimensionColumns dimension
* @param aggregateColumn aggregate field
* @return returns the map, the key is the data corresponding to the dimensionColumns dimension, and the value is the value corresponding to the aggregateColumn field
*/
public Map<String, Long> aggregateBySingleColumn(Map<String, String> paramMap, String[] dimensionColumns, String aggregateColumn) {
if (dimensionColumns == null || dimensionColumns.length == 0 || paramMap == null || aggregateColumn == null || aggregateColumn.equals("")) {
return null;
}
Map<String, Long> map = null;
try {
FilterList filterList = new FilterList();
Scan scan = new Scan();
//add filter condition
for (String paramKey : paramMap.keySet()) {
SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes(paramKey), Bytes.toBytes(paramKey), CompareOp.EQUAL, new SubstringComparator(paramMap.get(paramKey)));
filterList.addFilter(filter);
}
scan.setFilter(filterList);
// column to display
for (String column : dimensionColumns) {
scan.addColumn(Bytes.toBytes(column), Bytes.toBytes(column));
}
scan.addColumn(Bytes.toBytes(aggregateColumn), Bytes.toBytes(aggregateColumn));
ResultScanner results = table.getScanner(scan);
// put the query result into the map
map = new ConcurrentHashMap<String, Long>();
for (Result result : results) {
// String dimensionKey = "";
StringBuilder dimensionKey = new StringBuilder();
// get value
String value = new String(result.getValue(Bytes.toBytes(aggregateColumn), Bytes.toBytes(aggregateColumn)));
Long aggregateValue = value == null? 0 : Long.parseLong(value);
//splicing Key
for (String column : dimensionColumns) {
dimensionKey.append("\t" + new String(result.getValue(Bytes.toBytes(column), Bytes.toBytes(column))));
}
dimensionKey = dimensionKey.deleteCharAt(0);
if(map.containsKey(dimensionKey)) {
map.put(dimensionKey.toString(), map.get(dimensionKey.toString()) + aggregateValue);
} else {
map.put(dimensionKey.toString(), aggregateValue);
}
}
} catch (Exception e) {
LOG.error(e.getMessage());
}
return map;
}
2. Call the example to further calculate the sum of the obtained results according to their own dimensions
Map<String, String> paramMap = new HashMap<String, String>();
paramMap.put("stat_date", statDate);
//Get the summary data for the day
Map<String, Long> resultMap = new HashMap<String, Long>();
synchronized (this.db) {
resultMap = this.db.aggregateBySingleColumn(paramMap, columns, "pv");
}
this.finalMap.clear();
if(null != resultMap && resultMap.size() > 0){
for(String key : resultMap.keySet()) {
String[] arr = key.split("\t");
if(null != arr && null != columns && arr.length == columns.length) {
String appId = arr[1];
String appVersion = arr[2];
String eventKey = arr[3];
Long pv = resultMap.get(key);
pv = pv == null? 0 : pv;
StringBuilder buf = new StringBuilder();
buf.append(appId).append("\t").append(appVersion).append("\t").append(eventKey);
if(this.finalMap.containsKey(buf.toString())) {
this.finalMap.put(buf.toString(), this.finalMap.get(buf.toString()) + pv);
} else {
this.finalMap.put(buf.toString(), pv);
}
}
}
}
if(null!=this.finalMap && this.finalMap.size() > 0){
for(String key : this.finalMap.keySet()) {
String[] arr = key.split("\t");
if(null!=arr && arr.length >=3){
String appId = arr[0];
String appVersion = arr[1];
String eventKey = arr[2];
Long pv = this.finalMap.get(key);
if(pv > 0) {
String updateSql = String.format(UPDATE_QUERY_SQL, pv, statDate, appId, appVersion, eventKey);
this.sqlList.add(updateSql);
LOG.info("execute:execute mysql updateSql sql: " + updateSql);
}
if(null!=this.sqlList&&this.sqlList.size() > 100){
for(String sql : sqlList){
if(StringUtils.isNotBlank(sql)){
synchronized (this.stat_db) {
LOG.info("execute:execute mysql updateSql sql: " + sql);
this.stat_db.execute(sql);
}
}
}
this.sqlList.clear();
}
}
}
if(null!=this.sqlList&&this.sqlList.size() > 0){
for(String sql : sqlList){
if(StringUtils.isNotBlank(sql)){
synchronized (this.stat_db) {
LOG.info("execute:execute mysql updateSql sql: " + sql);
this.stat_db.execute(sql);
}
}
}
this.sqlList.clear();
}
}
this.finalMap.clear();