下方有数据可免费下载
目录
数据下载请点击我，提取码：v3rn，觉得有用希望您能点一个赞哦。
原始数据
文件夹中含有多个json文件，每个文件中的信息如上图所示。
总共19个字段
beCommentWeiboId 是否评论
beForwardWeiboId 是否是转发微博
catchTime 抓取时间
commentCount 评论次数
content 内容
createTime 创建时间
info1 信息字段1
info2信息字段2
info3信息字段3
mlevel no sure
musicurl 音乐链接
pic_list 照片列表（可以有多个）
praiseCount 点赞人数
reportCount 转发人数
source 数据来源
userId 用户id
videourl 视频链接
weiboId 微博id
weiboUrl 微博网址
数据仓库构建
create database if not exists weibo;
use weibo;
create external table weibo(json string)
location '/weibo';
create external table weibo_aaa(json string)
location '/aaa';
*加载数据*
hadoop fs -put ./weibo/* /weibo
hadoop fs -ls /weibo/
select * from weibo limit 3;
统计需求
1.微博总量和独立用户数
select count(*) from weibo;
Total MapReduce CPU Time Spent: 16 seconds 990 msec
OK
1451868
Time taken: 41.665 seconds, Fetched: 1 row(s)
select count(distinct(get_json_object(a.j,'$.userId')))
from
(select substring(json,2,length(json)-2) as j from weibo) a;
Total MapReduce CPU Time Spent: 47 seconds 400 msec
OK
78540
Time taken: 47.451 seconds, Fetched: 1 row(s)
2.用户所有微博被转发的总数，输出前3个用户
select b.id,sum(b.cnt) as bsum
from
(select get_json_object(a.j,'$.userId') as id,get_json_object(a.j,'$.reportCount') as cnt
from (select substring(json,2,length(json)-2) as j from weibo) a) b
group by b.id
order by bsum desc
limit 3;
Total MapReduce CPU Time Spent: 56 seconds 720 msec
OK
1793285524 7.6454805E7
1629810574 7.3656898E7
2803301701 6.8176008E7
Time taken: 72.604 seconds, Fetched: 3 row(s)
3.被转发次数最多的前3条微博，输出用户id
select get_json_object(a.j,'$.userId') as id,cast(get_json_object(a.j,'$.reportCount') as INT) as cnt
from
(select substring(json,2,length(json)-2) as j from weibo) a
order by cnt desc limit 3;
Total MapReduce CPU Time Spent: 52 seconds 560 msec
OK
2202387347 2692012
2202387347 2692012
2202387347 2692012
Time taken: 51.86 seconds, Fetched: 3 row(s)
4.每个用户发布的微博总数，存储到临时表
create table weibo_uid_wbcnt(
userid string,
wbcnt int
)
row format delimited
fields terminated by '\t';
insert overwrite table weibo_uid_wbcnt select get_json_object(a.j,'$.userId'),count(1)
from
(select substring(json,2,length(json)-2) as j from weibo) a
group by get_json_object(a.j,'$.userId');
select * from weibo_uid_wbcnt limit 10;
1000432103 10
1001869117 1
1002133091 2
1002282395 2
1002737912 2
1002861732 1
1002906354 2
1002909672 3
1003106904 1
1003669351 1
Time taken: 0.062 seconds, Fetched: 10 row(s)
5.统计带图片的微博数
select count(1)
from
(select substring(json,2,length(json)-2) as j from weibo) a
where get_json_object(a.j,'$.pic_list') like '%http%';
Total MapReduce CPU Time Spent: 41 seconds 720 msec
OK
750512
Time taken: 45.552 seconds, Fetched: 1 row(s)
6.统计使用iphone发微博的独立用户数
select count(distinct get_json_object(a.j,'$.userId'))
from
(select substring(json,2,length(json)-2) as j from weibo) a
where lower(get_json_object(a.j,'$.source')) like '%iphone%';
Total MapReduce CPU Time Spent: 42 seconds 690 msec
OK
936
Time taken: 45.606 seconds, Fetched: 1 row(s)
7.微博中评论次数小于1000的用户id和数据来源，放入视图
create view weibo_view as
select get_json_object(a.j,'$.userId') as id,get_json_object(a.j,'$.source') as source
from
(select substring(json,2,length(json)-2) as j from weibo) a
where get_json_object(a.j,'$.commentCount')<1000;
select * from weibo_view limit 10;
2989711735
1087770692 iPad客户端
1390470392
1390470392
1498502972
1087770692 iPad客户端
1589706710
1087770692 iPad客户端
1087770692 iPad客户端
1589706710
Time taken: 0.414 seconds, Fetched: 10 row(s)
8.统计上条视图中数据来源“ipad客户端”的用户数目
select count(distinct id) as cnt from weibo_view where source='iPad客户端';
Total MapReduce CPU Time Spent: 56 seconds 750 msec
OK
537
Time taken: 57.111 seconds, Fetched: 1 row(s)
特殊需求
1.Hive的UDF应用一
将微博的点赞数和转发数求和，降序，取前10条。
public class DemoTest1 extends UDF {
public Integer evaluate(Integer num1,Integer num2){
try{
return num1+num2;
}catch (Exception e){
return null;
}
}
}
create temporary function wb as 'DemoTest1';
select wb(cast(get_json_object(a.j,'$.praiseCount') as int),cast(get_json_object(a.j,'$.reportCount') as int)) as cnt
from
(select substring(json,2,length(json)-2) as j from weibo) a
order by cnt desc limit 10;
Total MapReduce CPU Time Spent: 59 seconds 10 msec
OK
2692013
2692013
2692013
2692013
2692013
2692013
2692013
2692010
2692010
2692009
Time taken: 62.921 seconds, Fetched: 10 row(s)
2.Hive的UDF应用二
用户微博内容中出现iphone关键词的最大次数
public class DemoTest2 extends UDF {
public int evaluate(String content,String word){
int count = 0;
if(content != null&&content.length()>0){
String[] array = content.split(word);
count = array.length-1;
}
return count;
}
}
create temporary function wcount as 'DemoTest2';
select b.id,max(b.cnt) as cn
from
(select get_json_object(a.j,'$.userId') as id,wcount(get_json_object(a.j,'$.content'),'iphone') as cnt
from
(select substring(json,2,length(json)-2) as j from weibo) a) b
group by b.id
order by cn desc limit 10;
Total MapReduce CPU Time Spent: 1 minutes 4 seconds 570 msec
OK
2003347594 11
1735618041 11
1220291284 7
2096991555 5
1144393351 5
2438243845 5
1136793703 4
1687914673 4
1346470754 4
2668575357 3
Time taken: 85.444 seconds, Fetched: 10 row(s)
数据ETL
在mysql中操作，将hive中的数据导入到mysql
每个用户发布的微博总数，存储到临时表
use sqoop_test;
create table weibo(
userid varchar(255),
wbcnt int
);
bin/sqoop export \
--connect jdbc:mysql://hadoop000:3306/sqoop_test \
--username root \
--password 123456 \
--table weibo \
--export-dir 'hdfs://hadoop000:8020/user/hive/warehouse/weibo.db/weibo_uid_wbcnt' \
--fields-terminated-by '\t';
select * from weibo limit 10;
+------------+-------+
| userid | wbcnt |
+------------+-------+
| 1837902673 | 1 |
| 1837921322 | 12 |
| 1837934232 | 1 |
| 1837937837 | 1 |
| 1837956335 | 1 |
| 1837963841 | 2 |
| 1837975031 | 1 |
| 1837999323 | 2 |
| 1838068594 | 1 |
| 1838101235 | 1 |
+------------+-------+
10 rows in set (0.00 sec)