vertica version 8.1
The official document says: Use the vtconfig script to complete the following tasks
Create cluster-create and configure schedule-create data table-create data source-create target-create load space-create Microbatch-load schedule
1. Create a cluster
You must associate at least one Kafka cluster with your schedule.
Use the following template:
# Vertica 8.1.0
kafka_config=" --config-schema kafka_date_dimension --dbhost v001 --username dbadmin --password vertica"
# shutdown instance
/opt/vertica/packages/kafka/bin/vkconfig shutdown --instance-name load_date_dimension_scheduler ${kafka_config}
echo "Shutdown Instance Complete!"
# truncate table
$VSQL <<- EOF
drop schema kafka_date_dimension cascade;
truncate table public.date_dimension;
EOF
# Create and Configure Scheduler
/opt/vertica/packages/kafka/bin/vkconfig scheduler --create --add ${kafka_config} --frame-duration '00:00:10' --eof-timeout-ms 3000 --operator dbadmin
echo "Create and Configure Scheduler Complete!"
# Create a Cluster
/opt/vertica/packages/kafka/bin/vkconfig cluster --create --cluster kafka_cluster --hosts v001:9092 ${kafka_config}
echo "Create Cluster Complete!"
# Create a Data Table
# Create a Source
/opt/vertica/packages/kafka/bin/vkconfig source --create --source date_dimension --cluster kafka_cluster --partitions 1 ${kafka_config}
echo "Create Kafka Source Complete!"
# Create a Target
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema public --target-table date_dimension ${kafka_config}
echo "Create Target Complete!"
# Create a Load-Spec
/opt/vertica/packages/kafka/bin/vkconfig load-spec --create --load-spec load_date_dimension_spec --parser KafkaJSONParser --parser-parameters flatten_arrays=False,flatten_maps=False ${kafka_config}
#/opt/vertica/packages/kafka/bin/vkconfig load-spec --create --load-spec load_date_dimension_spec --parser KafkaJSONParser --filters "FILTER KafkaInsertDelimiters(delimiter=E'\n')" ${kafka_config}
echo "Create Load-Spec Complete!"
# Create a Microbatch
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch date_dimension --target-schema public --target-table date_dimension --rejection-schema public --rejection-table date_dimension_rej --load-spec load_date_dimension_spec --add-source date_dimension --add-source-cluster kafka_cluster ${kafka_config}
echo "Create Microbatch Complete!"
# Launch the Scheduler
/opt/vertica/packages/kafka/bin/vkconfig launch --instance-name load_date_dimension_scheduler ${kafka_config} &
echo "Launch the Scheduler Complete!"
echo "Done!"Look at an example configuration:
# Vertica 8.1.0
--schema 定义
kafka_config=" --config-schema SCHMA_KAFKA_TO_VERTICA --dbhost 192.168.20.132 --username dbadmin --password vertica11"
# shutdown instance
/opt/vertica/packages/kafka/bin/vkconfig shutdown --instance-name SCHMA_KAFKA_TO_VERTICA ${kafka_config}
echo "Shutdown Instance Complete!"
#truncate table
#$VSQL <<- EOF
#drop schema SHCMA_TOPIC_CA_CUST_INFO_AL cascade;
#truncate table public.tb_svr_cntmng_day;
#EOF
# Create and Configure Scheduler
/opt/vertica/packages/kafka/bin/vkconfig scheduler --create --add ${kafka_config} --frame-duration '00:00:30' --eof-timeout-ms 3000 --operator dbadmin
echo "Create and Configure Scheduler Complete!"
# Create a Cluster
/opt/vertica/packages/kafka/bin/vkconfig cluster --create --cluster kafka_cluster --hosts 192.168.100.105:9092,192.168.100.106:9092,192.168.100.107:9092 ${kafka_config}
echo "Create Cluster Complete!"
# Create a Data Table
# Create a Source
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CA_LOG --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CA_GZT_LOG --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_AI_INTER_LOG --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CT_CALL_COUNT --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CT_LOG --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CT_PICCOMPARE --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CT_PICCOMPARE_HIS --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CT_VERIFYCARD --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CT_VERIFYCARD_HIS --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CO_PIC_COMPARE_INFO --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CO_FACE_LIVE_INFO --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CA_CUST_INFO_GZT --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CA_CUST_INFO_PY --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CA_CUST_INFO_XHN --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CA_CUST_INFO_TT --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CA_CUST_INFO_TT_SIMPLE --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CA_CUST_INFO_GZT_SPECIAL --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CA_CUST_INFO_NFC --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_NFC_LOG --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_NFC_PARSE_ERR_INFO --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_NFC_PARSE_INFO --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_VI_VIDEO_PROCESSING_INFO --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_VI_SPEECH_CHECK_INFO --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_REG_OCR_PIC_INFO --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_REG_CUST_VEDIO_HIS --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_CO_IDENTIFY_PIC_INFO --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_REG_CUST_PIC_HIS --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source TOPIC_USER_HITS_LOG --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source KAFKA_CSAP_SERVERCORE_SMPL_CALLRESULT_TOPIC --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source KAFKA_CSAP_SERVERCORE_SMPL_CNTMNGT_TOPIC --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source KAFKA_CSAP_SERVERCORE_SMPL_DETAIL_TOPIC --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source KAFKA_CSAP_SERVERCORE_SMPL_PHONE_STATUS_TOPIC --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source KAFKA_CSAP_SERVERCORE_SMPL_PHONE_TOPIC --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source KAFKA_CSAP_SERVERCORE_USMPL_CNTMNGT_TOPIC --cluster kafka_cluster --partitions 3 ${kafka_config}
#/opt/vertica/packages/kafka/bin/vkconfig source --delete --source znyd_irp_msgLog --cluster kafka_cluster --partitions 3 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source znyd_irp_msgLog --cluster kafka_cluster --partitions 3 ${kafka_config}
#/opt/vertica/packages/kafka/bin/vkconfig source --delete --source calllog --cluster kafka_cluster --partitions 1 ${kafka_config}
#/opt/vertica/packages/kafka/bin/vkconfig source --delete --source mgtlog --cluster kafka_cluster --partitions 1 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source calllog --cluster kafka_cluster --partitions 1 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source mgtlog --cluster kafka_cluster --partitions 1 ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig source --create --source contactlog --cluster kafka_cluster --partitions 1 ${kafka_config}
echo "Create Kafka Source Complete!"
# Create a Target
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_CA_LOG ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table OL_GZT_LOG ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_AI_INTER_LOG ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_CT_CALL_COUNT ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_CT_LOG ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_CT_PICCOMPARE ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_CT_PICCOMPARE_HIS ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_CT_VERIFYCARD ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_CT_VERIFYCARD_HIS ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_CO_PIC_COMPARE_INFO ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_CO_FACE_LIVE_INFO ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table OL_CUST_INFO_GZT ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table OL_CUST_INFO_PY ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table OL_CUST_INFO_XHN ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table OL_CUST_INFO_TT ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table OL_CUST_INFO_TT_SIMPLE ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table OL_CUST_INFO_GZT_SPECIAL ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table OL_CUST_INFO_NFC ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_NFC_LOG ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_NFC_PARSE_ERR_INFO ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_NFC_PARSE_INFO ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_VI_VIDEO_PROCESSING_INFO ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_VI_SPEECH_CHECK_INFO ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_REG_OCR_PIC_INFO ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_REG_CUST_VEDIO_HIS ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_CO_IDENTIFY_PIC_INFO ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_REG_CUST_PIC_HIS ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_REG_USER_HITS_LOG ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema csapsmpl --target-table tb_svr_callresult_day ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema csapsmpl --target-table tb_svr_cntmng_day ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema csapsmpl --target-table tb_svr_sample_detail_day ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema csapsmpl --target-table tb_svr_phone_status_day ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema csapsmpl --target-table tb_svr_phone ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema csapsmpl --target-table tb_svr_u_cntmng ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema csapsi --target-table user_msg_log ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema csapdw --target-table tb_dw_ct_ngcs_call_log_list_min_tmp ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema csapdw --target-table tb_dw_ct_ngcs_mgt_log_list_min_tmp ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema csapdw --target-table tb_dw_ct_ngcs_contact_cct_list_min_tmp ${kafka_config}
echo "Create Target Complete!"
# Create a Load-Spec
/opt/vertica/packages/kafka/bin/vkconfig load-spec --create --load-spec kafka_load --parser KafkaJSONParser --parser-parameters flatten_arrays=False,flatten_maps=true ${kafka_config}
echo "Create Load-Spec Complete!"
# Create a Microbatch
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_CA_LOG --target-schema qinchaofeng --target-table T_CA_LOG --rejection-schema qinchaofeng --rejection-table T_CA_LOG_rej --load-spec kafka_load --add-source TOPIC_CA_LOG --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch OL_GZT_LOG --target-schema qinchaofeng --target-table OL_GZT_LOG --rejection-schema qinchaofeng --rejection-table OL_GZT_LOG_rej --load-spec kafka_load --add-source TOPIC_CA_GZT_LOG --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_AI_INTER_LOG --target-schema qinchaofeng --target-table T_AI_INTER_LOG --rejection-schema qinchaofeng --rejection-table T_AI_INTER_LOG_rej --load-spec kafka_load --add-source TOPIC_AI_INTER_LOG --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_CT_CALL_COUNT --target-schema qinchaofeng --target-table T_CT_CALL_COUNT --rejection-schema qinchaofeng --rejection-table T_CT_CALL_COUNT_rej --load-spec kafka_load --add-source TOPIC_CT_CALL_COUNT --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_CT_LOG --target-schema qinchaofeng --target-table T_CT_LOG --rejection-schema qinchaofeng --rejection-table T_CT_LOG_rej --load-spec kafka_load --add-source TOPIC_CT_LOG --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_CT_PICCOMPARE --target-schema qinchaofeng --target-table T_CT_PICCOMPARE --rejection-schema qinchaofeng --rejection-table T_CT_PICCOMPARE_rej --load-spec kafka_load --add-source TOPIC_CT_PICCOMPARE --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_CT_PICCOMPARE_HIS --target-schema qinchaofeng --target-table T_CT_PICCOMPARE_HIS --rejection-schema qinchaofeng --rejection-table T_CT_PICCOMPARE_HIS_rej --load-spec kafka_load --add-source TOPIC_CT_PICCOMPARE_HIS --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_CT_VERIFYCARD --target-schema qinchaofeng --target-table T_CT_VERIFYCARD --rejection-schema qinchaofeng --rejection-table T_CT_VERIFYCARD_rej --load-spec kafka_load --add-source TOPIC_CT_VERIFYCARD --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_CT_VERIFYCARD_HIS --target-schema qinchaofeng --target-table T_CT_VERIFYCARD_HIS --rejection-schema qinchaofeng --rejection-table T_CT_VERIFYCARD_HIS_rej --load-spec kafka_load --add-source TOPIC_CT_VERIFYCARD_HIS --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_CO_PIC_COMPARE_INFO --target-schema qinchaofeng --target-table T_CO_PIC_COMPARE_INFO --rejection-schema qinchaofeng --rejection-table T_CO_PIC_COMPARE_INFO_rej --load-spec kafka_load --add-source TOPIC_CO_PIC_COMPARE_INFO --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_CO_FACE_LIVE_INFO --target-schema qinchaofeng --target-table T_CO_FACE_LIVE_INFO --rejection-schema qinchaofeng --rejection-table T_CO_FACE_LIVE_INFO_rej --load-spec kafka_load --add-source TOPIC_CO_FACE_LIVE_INFO --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch OL_CUST_INFO_GZT --target-schema qinchaofeng --target-table OL_CUST_INFO_GZT --rejection-schema qinchaofeng --rejection-table OL_CUST_INFO_GZT_rej --load-spec kafka_load --add-source TOPIC_CA_CUST_INFO_GZT --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch OL_CUST_INFO_PY --target-schema qinchaofeng --target-table OL_CUST_INFO_PY --rejection-schema qinchaofeng --rejection-table OL_CUST_INFO_PY_rej --load-spec kafka_load --add-source TOPIC_CA_CUST_INFO_PY --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch OL_CUST_INFO_XHN --target-schema qinchaofeng --target-table OL_CUST_INFO_XHN --rejection-schema qinchaofeng --rejection-table OL_CUST_INFO_XHN_rej --load-spec kafka_load --add-source TOPIC_CA_CUST_INFO_XHN --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch OL_CUST_INFO_TT --target-schema qinchaofeng --target-table OL_CUST_INFO_TT --rejection-schema qinchaofeng --rejection-table OL_CUST_INFO_TT_rej --load-spec kafka_load --add-source TOPIC_CA_CUST_INFO_TT --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch OL_CUST_INFO_TT_SIMPLE --target-schema qinchaofeng --target-table OL_CUST_INFO_TT_SIMPLE --rejection-schema qinchaofeng --rejection-table OL_CUST_INFO_TT_SIMPLE_rej --load-spec kafka_load --add-source TOPIC_CA_CUST_INFO_TT_SIMPLE --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch OL_CUST_INFO_GZT_SPECIAL --target-schema qinchaofeng --target-table OL_CUST_INFO_GZT_SPECIAL --rejection-schema qinchaofeng --rejection-table OL_CUST_INFO_GZT_SPECIAL_rej --load-spec kafka_load --add-source TOPIC_CA_CUST_INFO_GZT_SPECIAL --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch OL_CUST_INFO_NFC --target-schema qinchaofeng --target-table OL_CUST_INFO_NFC --rejection-schema qinchaofeng --rejection-table OL_CUST_INFO_NFC_rej --load-spec kafka_load --add-source TOPIC_CA_CUST_INFO_NFC --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_NFC_LOG --target-schema qinchaofeng --target-table T_NFC_LOG --rejection-schema qinchaofeng --rejection-table T_NFC_LOG_rej --load-spec kafka_load --add-source TOPIC_NFC_LOG --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_NFC_PARSE_ERR_INFO --target-schema qinchaofeng --target-table T_NFC_PARSE_ERR_INFO --rejection-schema qinchaofeng --rejection-table T_NFC_PARSE_ERR_INFO_rej --load-spec kafka_load --add-source TOPIC_NFC_PARSE_ERR_INFO --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_NFC_PARSE_INFO --target-schema qinchaofeng --target-table T_NFC_PARSE_INFO --rejection-schema qinchaofeng --rejection-table T_NFC_PARSE_INFO_rej --load-spec kafka_load --add-source TOPIC_NFC_PARSE_INFO --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_VI_VIDEO_PROCESSING_INFO --target-schema qinchaofeng --target-table T_VI_VIDEO_PROCESSING_INFO --rejection-schema qinchaofeng --rejection-table T_VI_VIDEO_PROCESSING_INFO_rej --load-spec kafka_load --add-source TOPIC_VI_VIDEO_PROCESSING_INFO --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_VI_SPEECH_CHECK_INFO --target-schema qinchaofeng --target-table T_VI_SPEECH_CHECK_INFO --rejection-schema qinchaofeng --rejection-table T_VI_SPEECH_CHECK_INFO_rej --load-spec kafka_load --add-source TOPIC_VI_SPEECH_CHECK_INFO --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_REG_OCR_PIC_INFO --target-schema qinchaofeng --target-table T_REG_OCR_PIC_INFO --rejection-schema qinchaofeng --rejection-table T_REG_OCR_PIC_INFO_rej --load-spec kafka_load --add-source TOPIC_REG_OCR_PIC_INFO --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_REG_CUST_VEDIO_HIS --target-schema qinchaofeng --target-table T_REG_CUST_VEDIO_HIS --rejection-schema qinchaofeng --rejection-table T_REG_CUST_VEDIO_HIS_rej --load-spec kafka_load --add-source TOPIC_REG_CUST_VEDIO_HIS --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_CO_IDENTIFY_PIC_INFO --target-schema qinchaofeng --target-table T_CO_IDENTIFY_PIC_INFO --rejection-schema qinchaofeng --rejection-table T_CO_IDENTIFY_PIC_INFO_rej --load-spec kafka_load --add-source TOPIC_CO_IDENTIFY_PIC_INFO --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_REG_CUST_PIC_HIS --target-schema qinchaofeng --target-table T_REG_CUST_PIC_HIS --rejection-schema qinchaofeng --rejection-table T_REG_CUST_PIC_HIS_rej --load-spec kafka_load --add-source TOPIC_REG_CUST_PIC_HIS --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_REG_USER_HITS_LOG --target-schema qinchaofeng --target-table T_REG_USER_HITS_LOG --rejection-schema qinchaofeng --rejection-table T_REG_USER_HITS_LOG_rej --load-spec kafka_load --add-source TOPIC_USER_HITS_LOG --add-source-cluster kafka_cluster ${kafka_config}
#/opt/vertica/packages/kafka/bin/vkconfig microbatch --delete --microbatch tb_svr_callresult_day ${kafka_config}
#/opt/vertica/packages/kafka/bin/vkconfig microbatch --delete --microbatch tb_svr_cntmng_day ${kafka_config}
#/opt/vertica/packages/kafka/bin/vkconfig microbatch --delete --microbatch tb_svr_sample_detail_day ${kafka_config}
#/opt/vertica/packages/kafka/bin/vkconfig microbatch --delete --microbatch tb_svr_phone_status_day ${kafka_config}
#/opt/vertica/packages/kafka/bin/vkconfig microbatch --delete --microbatch tb_dw_ct_ngcs_call_log_list_min_tmp ${kafka_config}
#/opt/vertica/packages/kafka/bin/vkconfig microbatch --delete --microbatch tb_dw_ct_ngcs_mgt_log_list_min_tmp ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch tb_svr_callresult_day --target-schema csapsmpl --target-table tb_svr_callresult_day --rejection-schema csapsmpl --rejection-table tb_svr_callresult_day_rej --load-spec kafka_load --add-source KAFKA_CSAP_SERVERCORE_SMPL_CALLRESULT_TOPIC --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch tb_svr_cntmng_day --target-schema csapsmpl --target-table tb_svr_cntmng_day --rejection-schema csapsmpl --rejection-table tb_svr_cntmng_day_rej --load-spec kafka_load --add-source KAFKA_CSAP_SERVERCORE_SMPL_CNTMNGT_TOPIC --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch tb_svr_sample_detail_day --target-schema csapsmpl --target-table tb_svr_sample_detail_day --rejection-schema csapsmpl --rejection-table tb_svr_sample_detail_day_rej --load-spec kafka_load --add-source KAFKA_CSAP_SERVERCORE_SMPL_DETAIL_TOPIC --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch tb_svr_phone_status_day --target-schema csapsmpl --target-table tb_svr_phone_status_day --rejection-schema csapsmpl --rejection-table tb_svr_phone_status_day_rej --load-spec kafka_load --add-source KAFKA_CSAP_SERVERCORE_SMPL_PHONE_STATUS_TOPIC --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch tb_svr_phone --target-schema csapsmpl --target-table tb_svr_phone --rejection-schema csapsmpl --rejection-table tb_svr_phone_rej --load-spec kafka_load --add-source KAFKA_CSAP_SERVERCORE_SMPL_PHONE_TOPIC --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch tb_svr_u_cntmng --target-schema csapsmpl --target-table tb_svr_u_cntmng --rejection-schema csapsmpl --rejection-table tb_svr_u_cntmng_rej --load-spec kafka_load --add-source KAFKA_CSAP_SERVERCORE_USMPL_CNTMNGT_TOPIC --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch tb_dw_ct_ngcs_call_log_list_min_tmp --target-schema csapdw --target-table tb_dw_ct_ngcs_call_log_list_min_tmp --rejection-schema csapdw --rejection-table t_dw_ct_ngcs_call_log_list_min_tmp_rej --load-spec kafka_load --add-source calllog --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch tb_dw_ct_ngcs_mgt_log_list_min_tmp --target-schema csapdw --target-table tb_dw_ct_ngcs_mgt_log_list_min_tmp --rejection-schema csapdw --rejection-table tb_dw_ct_ngcs_mgt_log_list_min_tmp_rej --load-spec kafka_load --add-source mgtlog --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch user_msg_log --target-schema csapsi --target-table user_msg_log --rejection-schema csapsi --rejection-table user_msg_log_rej --load-spec kafka_load --add-source znyd_irp_msgLog --add-source-cluster kafka_cluster ${kafka_config}
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch tb_dw_ct_ngcs_contact_cct_list_min_tmp --target-schema csapdw --target-table tb_dw_ct_ngcs_contact_cct_list_min_tmp --rejection-schema csapdw --rejection-table tb_dw_ct_ngcs_contact_cct_list_min_tmp_rej --load-spec kafka_load --add-source contactlog --add-source-cluster kafka_cluster ${kafka_config}
echo "Create Microbatch Complete!"
# Launch the Scheduler
/opt/vertica/packages/kafka/bin/vkconfig launch --instance-name kafka_load ${kafka_config} &
echo "Launch the Scheduler Complete!"
echo "Done!"The strength of the nine cattle and two tigers finally succeeded.
192.168.125.199 LY1F-R021110-VM05.local LY1F-R021110-VM05
192.168.125.112 LY1F-R020503-VM04.local LY1F-R020503-VM04
192.168.125.140 LY1F-R020601-VM04.local LY1F-R020601-VM04
192.168.125.138 LY1F-R020601-VM02.local LY1F-R020601-VM02
192.168.125.136 LY1F-R020602-VM04.local LY1F-R020602-VM04
192.168.125.110 LY1F-R021203-VM02.local LY1F-R021203-VM02
/etc/hosts/: host name to mapping ip
~/.bashrc: change the environment variables of the current user
# Vertica 8.1.0
--schema 定义
kafka_config=" --config-schema SCHMA_KAFKA_TO_VERTICA --dbhost 192.168.20.132 --username dbadmin --password vertica11"
# shutdown instance
/opt/vertica/packages/kafka/bin/vkconfig shutdown --instance-name SCHMA_KAFKA_TO_VERTICA ${kafka_config}
echo "Shutdown Instance Complete!"
#truncate table
#$VSQL <<- EOF
#drop schema SHCMA_TOPIC_CA_CUST_INFO_AL cascade;
#truncate table public.tb_svr_cntmng_day;
#EOF
# Create and Configure Scheduler
/opt/vertica/packages/kafka/bin/vkconfig scheduler --create --add ${kafka_config} --frame-duration '00:00:30' --eof-timeout-ms 3000 --operator dbadmin
echo "Create and Configure Scheduler Complete!"
# Create a Cluster
/opt/vertica/packages/kafka/bin/vkconfig cluster --create --cluster kafka_cluster --hosts 192.168.100.105:9092,192.168.100.106:9092,192.168.100.107:9092 ${kafka_config}
echo "Create Cluster Complete!"
# Create a Data Table
# Create a Source
/opt/vertica/packages/kafka/bin/vkconfig source --create --source KAFKA_CSAP_SERVERCORE_USMPL_CNTMNGT_TOPIC --cluster kafka_cluster --partitions 1 ${kafka_config}
echo "Create Kafka Source Complete!"
# Create a Target
/opt/vertica/packages/kafka/bin/vkconfig target --create --target-schema qinchaofeng --target-table T_CA_LOG ${kafka_config}
echo "Create Target Complete!"
# Create a Microbatch
/opt/vertica/packages/kafka/bin/vkconfig microbatch --create --microbatch T_CA_LOG --target-schema qinchaofeng --target-table T_CA_LOG --rejection-schema qinchaofeng --rejection-table T_CA_LOG_rej --load-spec kafka_load --add-source TOPIC_CA_LOG --add-source-cluster kafka_cluster ${kafka_config}
# Create a Load-Spec
/opt/vertica/packages/kafka/bin/vkconfig load-spec --create --load-spec kafka_load --parser KafkaJSONParser --parser-parameters flatten_arrays=False,flatten_maps=true ${kafka_config}
echo "Create Load-Spec Complete!"
# Launch the Scheduler
/opt/vertica/packages/kafka/bin/vkconfig launch --instance-name kafka_load ${kafka_config} &
echo "Launch the Scheduler Complete!"
echo "Done!"