動画アドレス: Shang シリコンバレービッグデータプロジェクト「オンライン教育のためのオフラインデータウェアハウス」_bilibili_bilibili
目次
第 9 章 データ ウェアハウス開発の DWD 層
P049
第 9 章 データ ウェアハウス開発の DWD 層
DWD レイヤーの設計ポイント:
(1) DWD 層の設計は次元モデリング理論に基づいており、この層には次元モデルのファクト テーブルが格納されます。
(2) DWD レイヤーのデータ格納形式は、orc 列ストレージ + Snappy 圧縮です。
(3) DWD 層テーブル名の命名仕様は、dwd_data ドメイン_テーブル名_単一パーティション インクリメンタル フル識別子 (inc/full) です。
-- 9.1 交易域加购事务事实表
DROP TABLE IF EXISTS dwd_trade_cart_add_inc;
CREATE EXTERNAL TABLE dwd_trade_cart_add_inc
(
`id` STRING COMMENT '编号',
`user_id` STRING COMMENT '用户id',
`course_id` STRING COMMENT '课程id',
`date_id` STRING COMMENT '时间id',
`session_id` STRING COMMENT '会话id',
`create_time` STRING COMMENT '加购时间',
`cart_price` DECIMAL(16, 2) COMMENT '加购时价格'
) COMMENT '交易域加购事务事实表'
PARTITIONED BY (`dt` STRING)
STORED AS ORC
LOCATION '/warehouse/edu/dwd/dwd_trade_cart_add_inc/'
TBLPROPERTIES ('orc.compress' = 'snappy');
select *
from edu2077.ods_cart_info_full;
select *
from edu2077.ods_cart_info_inc;
select data.id,
data.user_id,
data.course_id,
data.date_id,
data.session_id,
data.create_time,
data.cart_price
from edu2077.ods_cart_info_inc
where dt = '2022-02-21'
and type = 'bootstrap-insert';
set hive.exec.dynamic.partition.mode=nonstrict;--关闭严格模式
insert overwrite table edu2077.dwd_trade_cart_add_inc
select data.id,
data.user_id,
data.course_id,
date_format(data.create_time, 'yyyy-MM-dd') date_id,
data.session_id,
data.create_time,
data.cart_price,
date_format(data.create_time, 'yyyy-MM-dd') dt
from edu2077.ods_cart_info_inc
where dt = '2022-02-21'
and type = 'bootstrap-insert';
insert overwrite table edu2077.dwd_trade_cart_add_inc partition (dt = '2022-02-21')
select data.id,
data.user_id,
data.course_id,
date_format(data.create_time, 'yyyy-MM-dd') date_id,
data.session_id,
data.create_time,
data.cart_price
from edu2077.ods_cart_info_inc
where dt = '2022-02-21'
and type = 'bootstrap-insert';
select *
from dwd_trade_cart_add_inc;
--每日装载
insert overwrite table edu2077.dwd_trade_cart_add_inc partition (dt = '2022-02-22')
select data.id,
data.user_id,
data.course_id,
date_format(data.create_time, 'yyyy-MM-dd') date_id,
data.session_id,
data.create_time,
data.cart_price
from edu2077.ods_cart_info_inc
where dt = '2022-02-22'
and type = 'insert';P050
-- 9.2 交易域加购周期快照事实表
DROP TABLE IF EXISTS dwd_trade_cart_full;
CREATE EXTERNAL TABLE dwd_trade_cart_full
(
`id` STRING COMMENT '编号',
`user_id` STRING COMMENT '用户id',
`course_id` STRING COMMENT '课程id',
`date_id` STRING COMMENT '时间id',
`session_id` STRING COMMENT '会话id',
`course_name` STRING COMMENT '课程名称',
`create_time` STRING COMMENT '加购时间',
`cart_price` DECIMAL(16, 2) COMMENT '加购时价格'
) COMMENT '交易域加购周期快照事实表'
PARTITIONED BY (`dt` STRING)
STORED AS ORC
LOCATION '/warehouse/edu/dwd/dwd_trade_cart_full/'
TBLPROPERTIES ('orc.compress' = 'snappy');
select * from ods_cart_info_full
where dt = '2022-02-21';
--数据装载
insert overwrite table edu2077.dwd_trade_cart_full partition (dt = '2022-02-21')
select id,
user_id,
course_id,
date_format(create_time, 'yyyy-MM-dd'),
session_id,
course_name,
create_time,
cart_price
from edu2077.ods_cart_info_full
where dt = '2022-02-21'
and deleted = '0'
and sold = '0';
select * from dwd_trade_cart_full;P051
P052
P053
--9.3 交易域试听下单累积快照事实表
DROP TABLE IF EXISTS dwd_trade_course_order_inc;
CREATE EXTERNAL TABLE dwd_trade_course_order_inc
(
`id` STRING COMMENT '编号',
`user_id` STRING COMMENT '用户id',
`course_id` STRING COMMENT '课程id',
`course_name` STRING COMMENT '课程名称',
`category_id` STRING COMMENT '分类id',
`category_name` STRING COMMENT '分类名称',
`subject_id` STRING COMMENT '科目id',
`subject_name` STRING COMMENT '科目名称',
`order_id` STRING COMMENT '订单id',
`province_id` STRING COMMENT '省份id',
`play_time` STRING COMMENT '首次播放时间',
`play_date` STRING COMMENT '首次播放日期',
`order_time` STRING COMMENT '首次下单时间',
`order_date` STRING COMMENT '首次下单日期',
`end_date` STRING COMMENT '结束日期,试听后七天内未下单即为结束,试听日期+7为结束日期',
`session_id` STRING COMMENT '会话id',
`original_amount` DECIMAL(16, 2) COMMENT '原始金额分摊',
`coupon_reduce_amount` DECIMAL(16, 2) COMMENT '优惠金额分摊',
`final_amount` DECIMAL(16, 2) COMMENT '最终价格分摊'
) COMMENT '交易域试听下单累积快照事实表'
PARTITIONED BY (`dt` STRING)
STORED AS ORC
LOCATION '/warehouse/edu/dwd/dwd_trade_course_order_inc/'
TBLPROPERTIES ('orc.compress' = 'snappy');
set hive.exec.dynamic.partition.mode=nonstrict;--关闭严格模式
select * from ods_user_chapter_process_full;
select * from edu2077.ods_order_info_inc;
--（1）首日装载
with play as
(
select min(id) id,
user_id,
course_id,
min(create_time) play_time,
date_format(min(create_time), 'yyyy-MM-dd') play_date
from edu2077.ods_user_chapter_process_full
where dt = '2022-02-21'
group by user_id, course_id
),
oi as
(
select data.id,
data.province_id,
data.session_id
from edu2077.ods_order_info_inc
where dt = '2022-02-21'
and type = 'bootstrap-insert'
),
od as
(
select data.id,
data.course_id,
data.order_id,
data.user_id,
data.origin_amount,
data.coupon_reduce,
data.final_amount,
data.create_time order_time,
date_format(data.create_time, 'yyyy-MM-dd') order_date
from edu2077.ods_order_detail_inc
where dt = '2022-02-21'
and type = 'bootstrap-insert'
),
dim_course as (
select id,
course_name,
category_id,
category_name,
subject_id,
subject_name
from edu2077.dim_course_full
where dt = '2022-02-21'
)
insert
overwrite
table
edu2077.dwd_trade_course_order_inc
partition
(
dt
)
select final.id,
user_id,
course_id,
course_name,
category_id,
category_name,
subject_id,
subject_name,
order_id,
province_id,
play_time,
play_date,
order_time,
order_date,
end_date,
session_id,
origin_amount,
coupon_reduce,
final_amount,
case
when end_date is not null then end_date
when order_date is not null then order_date
else '9999-12-31' end dt
from (select play.id,
play.user_id,
play.course_id,
od.order_id,
oi.province_id,
play.play_time,
play.play_date,
od.order_time,
od.order_date,
if(od.order_date is null and
date_add(play.play_date, 7) <= '2022-02-21',
date_add(play.play_date, 7), null) end_date,
oi.session_id,
od.origin_amount,
od.coupon_reduce,
od.final_amount
from play
left join od on play.user_id = od.user_id and play.course_id = od.course_id
left join oi on od.order_id = oi.id
where od.order_time is null
or od.order_time > play.play_time
) final
left join dim_course on course_id = dim_course.id;
select * from dwd_trade_course_order_inc;P054
--9.3 交易域试听下单累积快照事实表
--（2）每日装载
set hive.exec.dynamic.partition.mode=nonstrict;
with play as
(select id,
user_id,
course_id,
play_time,
play_date
from edu2077.dwd_trade_course_order_inc
where dt = '9999-12-31'
union
select min(id) id,
user_id,
course_id,
min(create_time),
date_format(min(create_time), 'yyyy-MM-dd') play_date
from edu2077.ods_user_chapter_process_full
where dt = '2022-02-22'
group by user_id, course_id
having date_format(min(create_time), 'yyyy-MM-dd') = '2022-02-22'),
oi as
(
select data.id,
data.province_id,
data.session_id,
data.create_time order_time
from edu2077.ods_order_info_inc
where dt = '2022-02-22'
and type = 'insert'
),
od as
(
select data.id,
data.course_id,
data.order_id,
data.user_id,
data.origin_amount,
data.coupon_reduce,
data.final_amount,
date_format(data.create_time, 'yyyy-MM-dd') order_date
from edu2077.ods_order_detail_inc
where dt = '2022-02-22'
and type = 'insert'
),
dim_course as
(
select id,
course_name,
category_id,
category_name,
subject_id,
subject_name
from edu2077.dim_course_full
where dt = '2022-02-22'
)
insert
overwrite
table
edu2077.dwd_trade_course_order_inc
partition
(
dt
)
select final.id,
user_id,
course_id,
course_name,
category_id,
category_name,
subject_id,
subject_name,
order_id,
province_id,
play_time,
play_date,
order_time,
order_date,
end_date,
session_id,
origin_amount,
coupon_reduce,
final_amount,
case
when end_date is not null then end_date
when order_date is not null then order_date
else '9999-12-31' end dt
from (select play.id,
play.user_id,
play.course_id,
od.order_id,
oi.province_id,
play.play_time,
play.play_date,
oi.order_time,
od.order_date,
if(order_date is null and date_add(play_date, 7) = '2022-02-22', '2022-02-22', null) end_date,
oi.session_id,
od.origin_amount,
od.coupon_reduce,
od.final_amount
from play
left join od on play.user_id = od.user_id and play.course_id = od.course_id
left join oi on od.order_id = oi.id
where order_time is null
or order_time > play_time
) final
left join dim_course on course_id = dim_course.id;P055
--9.4 交易域下单事务事实表
DROP TABLE IF EXISTS dwd_trade_order_detail_inc;
CREATE EXTERNAL TABLE dwd_trade_order_detail_inc
(
`id` STRING COMMENT '编号',
`order_id` STRING COMMENT '订单id',
`user_id` STRING COMMENT '用户id',
`course_id` STRING COMMENT '课程id',
`course_name` STRING COMMENT '课程名称',
`category_id` STRING COMMENT '分类id',
`category_name` STRING COMMENT '分类名称',
`subject_id` STRING COMMENT '科目id',
`subject_name` STRING COMMENT '科目名称',
`province_id` STRING COMMENT '省份id',
`date_id` STRING COMMENT '下单日期id',
`session_id` STRING COMMENT '会话id',
`source_id` STRING COMMENT '来源id',
`create_time` STRING COMMENT '下单时间',
`original_amount` DECIMAL(16, 2) COMMENT '原始金额分摊',
`coupon_reduce_amount` DECIMAL(16, 2) COMMENT '优惠金额分摊',
`final_amount` DECIMAL(16, 2) COMMENT '最终价格分摊',
`out_trade_no` STRING COMMENT '订单交易编号',
`trade_body` STRING COMMENT '订单描述'
) COMMENT '交易域下单事务事实表'
PARTITIONED BY (`dt` STRING)
STORED AS ORC
LOCATION '/warehouse/edu/dwd/dwd_trade_order_detail_inc/'
TBLPROPERTIES ('orc.compress' = 'snappy');
select * from edu2077.ods_order_detail_inc where dt = '2022-02-21';
--（1）首日装载
set hive.exec.dynamic.partition.mode=nonstrict;
insert overwrite table edu2077.dwd_trade_order_detail_inc
partition (dt)
select odt.id,
order_id,
user_id,
course_id,
course_name,
category_id,
category_name,
subject_id,
subject_name,
province_id,
date_id,
session_id,
source_id,
create_time,
origin_amount,
coupon_reduce,
final_amount,
out_trade_no,
trade_body,
date_id
from (
select data.id,
data.order_id,
data.user_id,
data.course_id,
date_format(data.create_time, 'yyyy-MM-dd') date_id,
data.create_time,
data.origin_amount,
data.coupon_reduce,
data.final_amount
from edu2077.ods_order_detail_inc
where dt = '2022-02-21'
and type = 'bootstrap-insert'
) odt
left join
(
select data.id,
data.province_id,
data.out_trade_no,
data.session_id,
data.trade_body
from edu2077.ods_order_info_inc
where dt = '2022-02-21'
and type = 'bootstrap-insert'
) od
on odt.order_id = od.id
left join
(
select distinct common.sid,
common.sc source_id
from edu2077.ods_log_inc oli
where dt = '2022-02-21'
) log
on od.session_id = log.sid
left join
(
select id,
course_name,
category_id,
category_name,
subject_id,
subject_name
from edu2077.dim_course_full
where dt = '2022-02-21'
) dim_course
on course_id = dim_course.id;P056
--9.4 交易域下单事务事实表
--（2）每日装载
insert overwrite table edu2077.dwd_trade_order_detail_inc
partition (dt = '2022-02-22')
select odt.id,
order_id,
user_id,
course_id,
course_name,
category_id,
category_name,
subject_id,
subject_name,
province_id,
date_id,
session_id,
source_id,
create_time,
origin_amount,
coupon_reduce,
final_amount,
out_trade_no,
trade_body
from (
select data.id,
data.order_id,
data.user_id,
data.course_id,
date_format(data.create_time, 'yyyy-MM-dd') date_id,
data.create_time,
data.origin_amount,
data.coupon_reduce,
data.final_amount
from edu2077.ods_order_detail_inc
where dt = '2022-02-22'
and type = 'insert'
) odt
left join
(
select data.id,
data.province_id,
data.session_id,
data.out_trade_no,
data.trade_body
from edu2077.ods_order_info_inc
where dt = '2022-02-22'
and type = 'insert'
) od
on odt.order_id = od.id
left join
(
select distinct common.sid,
common.sc source_id
from edu2077.ods_log_inc oli
where dt = '2022-02-22'
) log
on od.session_id = log.sid
left join
(
select id,
course_name,
category_id,
category_name,
subject_id,
subject_name
from edu2077.dim_course_full
where dt = '2022-02-22'
) dim_course
on course_id = dim_course.id;P057
--9.5 交易域支付成功事务事实表
DROP TABLE IF EXISTS dwd_trade_pay_detail_suc_inc;
CREATE EXTERNAL TABLE dwd_trade_pay_detail_suc_inc
(
`id` STRING COMMENT '编号',
`order_id` STRING COMMENT '订单id',
`user_id` STRING COMMENT '用户id',
`course_id` STRING COMMENT '课程id',
`province_id` STRING COMMENT '省份id',
`date_id` STRING COMMENT '支付日期id',
`alipay_trade_no` STRING COMMENT '支付宝交易编号',
`trade_body` STRING COMMENT '交易内容',
`payment_type` STRING COMMENT '支付类型名称',
`payment_status` STRING COMMENT '支付状态',
`callback_time` STRING COMMENT '支付成功时间',
`callback_content` STRING COMMENT '回调信息',
`original_amount` DECIMAL(16, 2) COMMENT '原始支付金额分摊',
`coupon_reduce_amount` DECIMAL(16, 2) COMMENT '优惠支付金额分摊',
`final_amount` DECIMAL(16, 2) COMMENT '最终支付金额分摊'
) COMMENT '交易域支付成功事务事实表'
PARTITIONED BY (`dt` STRING)
STORED AS ORC
LOCATION '/warehouse/edu/dwd/dwd_trade_pay_detail_suc_inc/'
TBLPROPERTIES ('orc.compress' = 'snappy');
--（1）首日装载
set hive.exec.dynamic.partition.mode=nonstrict;
insert overwrite table edu2077.dwd_trade_pay_detail_suc_inc
partition (dt)
select odt.id,
od.id,
user_id,
course_id,
province_id,
date_format(create_time, 'yyyy-MM-dd') date_id,
alipay_trade_no,
trade_body,
payment_type,
payment_status,
callback_time,
callback_content,
origin_amount,
coupon_reduce,
final_amount,
date_format(create_time, 'yyyy-MM-dd') date_id
from (
select data.id,
data.order_id,
data.user_id,
data.course_id,
data.origin_amount,
data.coupon_reduce,
data.final_amount,
data.create_time
from edu2077.ods_order_detail_inc
where dt = '2022-02-21'
and type = 'bootstrap-insert'
) odt
left join
(
select data.id,
data.province_id
from edu2077.ods_order_info_inc
where dt = '2022-02-21'
and type = 'bootstrap-insert'
) od
on odt.order_id = od.id
join
(
select data.alipay_trade_no,
data.trade_body,
data.order_id,
data.payment_type,
data.payment_status,
data.callback_time,
data.callback_content
from edu2077.ods_payment_info_inc
where dt = '2022-02-21'
and type = 'bootstrap-insert'
and data.callback_time is not null
) pi
on od.id = pi.order_id;
select * from dwd_trade_pay_detail_suc_inc;P058
--9.5 交易域支付成功事务事实表
-- （2）每日装载
insert overwrite table edu2077.dwd_trade_pay_detail_suc_inc
partition (dt = '2022-02-22')
select odt.id,
od.id,
user_id,
course_id,
province_id,
date_format(create_time, 'yyyy-MM-dd') date_id,
alipay_trade_no,
trade_body,
payment_type,
payment_status,
callback_time,
callback_content,
origin_amount,
coupon_reduce,
final_amount
from (
select data.id,
data.order_id,
data.user_id,
data.course_id,
data.origin_amount,
data.coupon_reduce,
data.final_amount,
data.create_time
from edu2077.ods_order_detail_inc
where (dt = '2022-02-22' or dt = date_add('2022-02-22', -1))
and (type = 'insert' or type = 'bootstrap-insert')
) odt
left join
(
select data.id,
data.province_id
from edu2077.ods_order_info_inc
where (dt = '2022-02-22' or dt = date_add('2022-02-22', -1))
and (type = 'insert' or type = 'bootstrap-insert')
) od
on odt.order_id = od.id
join
(
select data.alipay_trade_no,
data.trade_body,
data.order_id,
data.payment_type,
data.payment_status,
data.callback_time,
data.callback_content
from edu2077.ods_payment_info_inc
where dt = '2022-02-22'
and type = 'update'
and array_contains(map_keys(old), 'callback_time')
) pi
on od.id = pi.order_id;P059
9.6 トラフィックドメインページの閲覧トランザクションファクトテーブル
P060
9.7 トラフィックドメイン起動トランザクションファクトテーブル
P061
9.8 トラフィックドメインアクショントランザクションファクトテーブル
9.9 トラフィックドメインエクスポージャトランザクションファクトテーブル
9.10 トラフィックドメインエラートランザクションファクトテーブル
P062
9.11 対話型ドメイン収集トランザクションファクトテーブル
P063
9.12 インタラクティブドメインチャプター評価トランザクションファクトテーブル
9.13 インタラクティブドメインコース評価トランザクションファクトシート
P064
9.14 試験ドメイン解答用紙 取引ファクトシート
9.15 試験分野解答事務ファクトシート
P065
9.16 学習ドメインの再生サイクルのスナップショット ファクト テーブル
(1) 初日搬入
P066
9.16 学習ドメインの再生サイクルのスナップショット ファクト テーブル
(2) 日々の積載量
P067
9.17 Domain Play トランザクション ファクト テーブルの学習
P068
9.18 ユーザードメインユーザー登録トランザクションファクトテーブル
P069
9.19 ユーザードメインユーザーログイントランザクションファクトテーブル
P070
9.20 データロードスクリプト
9.20.1 初日のロードスクリプト
9.20.2 毎日のロードスクリプト