从一个集群的hbase 读出数据 放到另一个集群的Hbase

版权声明: https://blog.csdn.net/u014384314/article/details/80923743

object getData {

//获取前一个集群的访问权限

System.setProperty( “HADOOP_USER_NAME”, “AirConReader”)

val conf = HBaseConfiguration.create()

conf.set( “HBase的,zookeeper.quorum”, “10.2.5.203”)

//设置HBase的客户端发起一次数据操作(GET,PUT追加,增加,删除),直到得到响应的超时时间,不包括扫描操作

conf.setInt( “hbase.client.operation.timeout”,1200000)

//设置Hbase一次远程调用请求(RPC)的超时时间

conf.setInt(“hbase.client.rpc.timeout”,1200000)

//设置每次扫描操作的超时时间

conf.setInt(“hbase.client.scanner.timeout.period”,600000)

// conn的创建是个重量级的工作,县城安全,是操作hbase的入口,最好只创建一个

val conn = ConnectionFactory.createConnection(conf)

def main(args:Array [String]):Unit = {

//获取数据

val rs = testFilterScan(“nameSpace:tableName”)

}

def testFilterScan(tableName:String):ResultScanner = {

val userTable = TableName.valueOf(tableName)

val table = conn.getTable(userTable)

val filterList = new FilterList(FilterList.OPerator.MUST_PASS_ALL)

/ *

val f1 = new SingleColumnValueFilter(

Bytes.toBytes( “F”),Bytes.toBytes( “svrCtime”),

compareOp.GREATE_OR_EQUAL,

Bytes.toBytes(“2018-06-18 00:00:00”)

val f2 = new SingleColumnValueFilter(

Bytes.toBytes( “F”),Bytes.toBytes( “svrCtime”),

compareOp.LESS_OR_EQUAL,

Bytes.toBytes(“2018-06-18 00:10:00”)

filterList.addFilter(F1)

filterList.addFilter(F2)

* /

val a = 5

val b = 7

val regxKey =“.* 201806151 [”+ a +“ - ”+ b +“].*”

val rf = new RowFilter(CompareOp.EQUAL,new RegexStringComparator(regxKey))

filterList.addFilter(RF)

val scan = new Scan()

scan.setFilter(filterList)

val rs = table.getScanner(scan)

/ * 

val i = 0

val j = 0

try{

val it = rs.iterator()

while(it.hasNext){

val next = it.next()

j= Jj+ 1

for(cell<-next.rawCells()){

print(“rowKey:” + Bytes.toString(next.getRow))

print(“familiy:Quilifer”+ Bytes.toString(CellUtil.cloneQualifer(cell)))

print(“value:” + Bytes.toString(CellUtil.cloneValue(value)))

print(“time:”+ cell.getTimestamp)

i= i + 1

print(“j:”+ j)

println(“,i:”+ i)

}

i= 0

}

}

抓住{

case e:Exception => println(“读取数据失败”)}

finally{

table.close()

conn.close()

}

* /

return rs

}

}

object saveData {

val conf = HBaseConfiguration.create()

conf.set(“HBase的,zookeeper.quorum”,“10.2.5.203”)

//设置HBase的的客户端发起一次数据操作(GET,PUT追加,增加,删除),直到得到响应的超时时间,不包括扫描操作

conf.setInt(“hbase.client.operation.timeout”,1200000)

//设置HBASE一次远程调用请求(RPC)的超时时间

conf.setInt(“hbase.client.rpc.timeout”,1200000)

//设置每次扫描操作的超时时间

conf.setInt(“hbase.client.scanner.timeout.period”,600000)

// conn的创建是个重量级的工作,县城安全,是操作hbase的入口,最好只创建一个

val conn = ConnectionFactory.createConnection(conf)

def createTable(tableName:String,columnFamilys:Array [String]):Unit = {

val userTable = TableName.valueOf(tableName)

val admin = conn.getAdmin

if(admin.tableExists( userTable )){

的println(“表”+ userTable +“已经存在”)

return}

else{

val tableDesc = new HTableDescriptor(userTable)

for(ColumnFamily <-columnFamilys){

 tableDesc.addFamily(new HColumnDescriptor(columnFamily.getBytes))

}

admin.createTable(tableDesc)

打印( “创建表成功”)

}

}

def saveData(tableName:String,columnFamily:String,rs:ResultScanner):Boolean = {

val userTable = TableName.valueOf(tableName)

val table = conn.getTable(userTable)

val lp = new util.ArrayList [Put]()

var j = 0

打印( “开始写入”)

try{

val it = rs.iterator()

while(it.hasNext){

val next = it.next()

j = j+ 1

for(cell<-next.rawCells){

val put = new Put(Bytes.toBytes(BYtes.toString(next.getRow)))

put.addColumn(columnFamily.getBytes,Bytes.toBytes(Bytes.toString(CellUtil.cloneQualifier(cell))),CellUtil.cloneValue(cell))

lp.add(put)

//每次放1万行

if(lp%10000 == 0){

table.put(lp)

lp.clear()

}

}

}

//如果总数大于0行且小于1万行或者剩下的行数不足1万行,直接放

if(lp.size> 0)

table.put(lp)

table.close()

conn.close()

println( “数据写入成功”)

}catch{

case e:Exception => println(“写入hbase失败”)}

}

return true

}

}

猜你喜欢

转载自blog.csdn.net/u014384314/article/details/80923743