Spark 项目实战之数据服务分析（八）

附录

在 Hive 创建 allData 表

CREATE EXTERNAL TABLE t_people (ID string,
RSFZ string,
GRXB string,
PSQK string,
SSYZ string,
SSYS string,
XSYZ string,
XSYS string,
TGSJ string,
SBBH string,
JWZB string)
row format delimited fields terminated by ',' lines terminated by '\n' location '/people' tblproperties ("skip.header.line.count"="1", "skip.footer.line.count"="1");

在mysql 上创建 t_people_result

CREATE TABLE t_people_result (
CJCJ text,
RSFZ text,
TGSJ text,
SBBH text,
CREATETIME text
)charset utf8 collate utf8_general_ci;

spark-submit

./spark-submit --master spark://bigdata01:7077 --class com.monitor.firstrecord.FirstPhoto --deploy-mode client /root/monitoranalysis.jar

启动 kafka

nohup bin/kafka-server-start.sh  config/server.properties &

在 mysql 上创建 t_people_result2

CREATE TABLE t_people_result2 (
CJCJ text,
RSFZ text,
TGSJ text,
SBBH text
)charset utf8 collate utf8_general_ci;

实时数据模拟处理

java -cp /root/monitoranalysis.jar com.monitor.produce.PeopleProducer

./spark-submit --class com.monitor.firstrecord.streaming.FirstPhotoStreaming --deploy-mode client /root/monitoranalysis.jar

在hive 上创建 people_together 表

-- hdfs 导入数据
bin/hadoop dfs -mkdir /people_together
bin/hdfs dfs -put /root/people01.csv /people_together

-- hive 创建表
CREATE EXTERNAL TABLE t_people_together (ID string,
RSFZ string,
GRXB string,
PSQK string,
SSYZ string,
SSYS string,
XSYZ string,
XSYS string,
TGSJ string,
SBBH string,
JWZB string)
row format delimited fields terminated by ',' lines terminated by '\n' location '/people_together' tblproperties ("skip.header.line.count"="1", "skip.footer.line.count"="1");

-- spark submit
./spark-submit --master spark://bigdata01:7077 --class com.monitor.together.TogetherCompute --deploy-mode client /root/monitoranalysis-1.0-SNAPSHOT.jar

在 mysql 上创建数据表

-- t_verify_result
CREATE TABLE t_verify_result(
RSFZ text,
GRXB text,
PSQK text,
TGSJ text,
SBBH text,
JWZB text,
CJSJ text
)charset utf8 collate utf8_general_ci;

-- t_verify_result2
CREATE TABLE t_verify_result2(
RSFZ text,
GRXB text,
PSQK text,
TGSJ text,
SBBH text,
JWZB text,
CJSJ text
)charset utf8 collate utf8_general_ci;

--  t_streaming_result
CREATE TABLE t_streaming_result(
JSBH text,
RSFZ text,
GRXB text,
PSQK text,
TGSJ text,
SBBH text,
CJSJ text
)charset utf8 collate utf8_general_ci;