import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.client.{HBaseAdmin, Result}
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapreduce.TableInputFormat
import org.apache.hadoop.hbase.util.Bytes
import org.apache.spark.{SparkConf, SparkContext}
object HbaseUtils {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("HbaseTest").setMaster("local")
val sc = new SparkContext(conf)
val tableName = "mkd_sdata:u_user_info"
val hbaseConfig = HBaseConfiguration.create()
hbaseConfig.set(TableInputFormat.INPUT_TABLE, tableName)
val admin = new HBaseAdmin(hbaseConfig)
val hbaseRDD = sc.newAPIHadoopRDD(hbaseConfig,
classOf[TableInputFormat],
classOf[ImmutableBytesWritable],
classOf[Result])
val count = hbaseRDD.count()
println("------------------:"+count)
val resultRDD = hbaseRDD.map { result => {
val key = Bytes.toString(result._2.getRow)
val userId = Bytes.toString(result._2.getValue("base".getBytes, "user_id".getBytes))
val mobile = Bytes.toString(result._2.getValue("base".getBytes, "mobile".getBytes))
println("key:" + key + "\t userId:" + userId + "\t mobile:" + mobile)
(userId, mobile)
}
}
resultRDD.saveAsTextFile("file:///e:/outs")
sc.stop()
admin.close()
}
}
Spark之读取Hbase数据库表并写入文件
猜你喜欢
转载自blog.csdn.net/niuchenliang524/article/details/80737585
今日推荐
周排行