将hive采集的数据结构导入MySQL

#!/bin/sh

###############

collect mysql data import into hdfs

约定：变量都用大写，多个单词之间使用下划线分割

mysql: test/t_user

hdfs : /input/data-clean/t_user

在""中会执行``命令，但是在’'中，无法解析${}

###############

##source配置文件

function run() {
source /home/centos/shells/db-conf.conf
HIVE_BIN=/home/centos/hive/bin/hive
SQOOP_BIN=/home/centos/sqoop/bin/sqoop
##创建hive中外部表关联hbase
##查询对应的数据，生成临时表
##统计数据，生成临时表1
sql="
set hive.exec.mode.local.auto=true;
USE event_log;
CREATE EXTERNAL TABLE event_log_hourly ( rowkey string, u_ud string, pl string, s_time bigint, bt string, bv string, province string ) STORED BY ‘org.apache.hadoop.hive.hbase.HBaseStorageHandler’ WITH SERDEPROPERTIES (“hbase.columns.mapping” = “:key,f:u_ud, f:pl,f:s_time,f:bt, f:bv, f:province”) TBLPROPERTIES (“hbase.table.name” = “nginx_js”);
CREATE TABLE event_log_hourly_tmp(date_ string, hour string, pl string, bt string, bv string, u_ud string);
insert overwrite table event_log_hourly_tmp select from_unixtime(cast((s_time/1000) as int), ‘yyyy-MM-dd’), from_unixtime(cast((s_time/1000) as int), ‘yyyy-MM-dd HH:mm:ss’), pl, bt, bv, u_ud from event_log_hourly;
CREATE TABLE event_log_hourly_tmp1 (date_ string, hour_ string, pl string, bt string , bv string, u_count int);
insert overwrite table event_log_hourly_tmp1 select date_, hour, pl, bt, bv, count(distinct u_ud) as u_count from event_log_hourly_tmp t group by date_, hour, pl, bt, bv;
"
${HIVE_BIN} -e "${sql}"
##将结果通过sqoop迁移到mysql
KaTeX parse error: Expected 'EOF', got '\ ' at position 20: …OP_BIN} export \̲ ̲ --connect j…{DB_PRODUCT_HOST}?{DB_PRODUCT_PORT}/test
–username ${DB_PRODUCT_USER}
–password ${DB_PRODUCT_PASSWORD}
–export-dir hdfs://ns1/user/hive/warehouse/event_log.db/event_log_hourly_tmp1
–table event_log_hourly
–fields-terminated-by ‘\001’;
}
run $*
wc命令查看文件行数
cat *.txt > tex.txt
wc -l tex.txt
wc -c tex.txt 只显示字节数
wc -w tex.txt 只显示字数