background
The second phase of development of a large-scale project has been completed and launched online. A large number of data sources need to be initialized, and the data sources provided by the third-party BPM system are not completely in the format and table structure we need, so it is necessary Do a data migration. The database used is SqlServer.This migration idea can be used for any migration where the table structure is inconsistent between databases.
Prerequisites and requirements
- Clarify the scope of local data migration
- Maintain data consistency during data migration
- The business table adds a migration data identifier, and the business table migration data field ISMOVE='Y'
logic
The data source is provided by a third party, which is cleaned, filtered, and written into its own business library and business tables.
accomplish
1. Create four migration setting tables QY_Tables, Qy_Maping, Qy_FieldDic, and Qy_ErrorMsg
--迁移表设置
CREATE TABLE QY_Tables
(
ID INT NOT NULL -- 主键
, Type VARCHAR (255) NULL -- 类型【试图、表，一般皆为表】
, Business VARCHAR (255) NULL -- 业务模块
, TableName VARCHAR (255) NULL -- 表名中文
, TableCode VARCHAR (255) NULL -- 第三方提供的数据表表名【用于存储清洗后的数据】
, BPMTable VARCHAR (255) NULL -- 从第三方数据库同步过来之后的数据源表名[用于第一次同步第三方数据源到自己的库中]
, BPMVIEW VARCHAR (255) NULL -- 将第三方数据源表字段更换为中文所用试图，便于查看
, CostTable VARCHAR (255) NULL -- 自己的业务表
, Mark VARCHAR (255) NULL -- 备注
, Doing VARCHAR (255) NULL -- 是否同步
)
--迁移字段设置表
--有可能QyTableFieldName字段在QyTableFieldCodeBPM中没有，那么设置QyDefaultValue默认值即可
--这个表，是设置我们所需要的业务字段，用于清洗数据使用
CREATE TABLE Qy_Maping
(
Id INT IDENTITY NOT NULL --主键
, QyTableName NVARCHAR (50) NULL -- 迁移表名
, QyTableFieldName NVARCHAR (50) NULL -- 迁移字段中文名称
, QyTableFieldCode NVARCHAR (50) NULL -- 迁移字段英文
, QyTableFieldCodeBPM NVARCHAR (50) NULL -- 第三方迁移表字段英文
, QyRemark NVARCHAR (200) NULL --字段备注
, QyPOPTYPE NVARCHAR (200) NULL -- 下拉框字典，对应 Qy_FieldDic.POPTYPE
, QyPOPTYPEValueSouce NVARCHAR (50) NULL --下拉框字段具体值，对应 Qy_FieldDic.TITLEVALUE
, QyDefaultValue NVARCHAR (50) NULL -- 默认值
, QyRelaceNullValue NVARCHAR (50) NULL -- 如果第三方字段为空，默认替代值
, QyTableFieldType NVARCHAR (50) NULL -- 字段类型【'int'、'nvarchar'等】
, QyTableFieldDataType NVARCHAR (50) NULL -- 字段类型，一般如果是枚举值时不为空，其他为空
, QyIsNull NVARCHAR (50) NULL -- 是否为null
, QyIsNullBPM NVARCHAR (50) NULL -- 第三方数据源是否为null
, QyDeaultBPMValue NVARCHAR (50) NULL -- 第三方数据源默认值
)
GO
-- 字段字典枚举值
CREATE TABLE dbo.Qy_FieldDic
(
ID INT IDENTITY NOT NULL -- 主键
, POPUPLSTELEMENTID INT NULL -- 对应自己业务系统中已有的字段枚举值表ID，没有填0
, TITLECN NVARCHAR (50) NULL -- 中文含义
, TITLEVALUE NVARCHAR (50) NULL -- 编码
, POPTYPE NVARCHAR (50) NULL -- 枚举类型
, BPMCODE NVARCHAR (50) NULL -- 第三方数据源值编码
, BPMCLOUMN NVARCHAR (50) NULL -- 第三方数据源值对应的数据源列
, BPMTABLE NVARCHAR (50) NULL -- 所在迁移表表名
, CONSTRAINT PK_Qy_FieldDic PRIMARY KEY (ID)
)
GO
--错误日志表，主要用于迁移失败的日志记录，可根据自己的业务自由扩展字段
CREATE TABLE dbo.Qy_ErrorMsg
(
Id INT IDENTITY NOT NULL -- 主键
, FromTableName NVARCHAR (50) NULL -- 来源表
, ToTableName NVARCHAR (50) NULL -- 业务目标表
, ErrorMsg NVARCHAR (max) NULL -- 错误信息
, ErrorSqlCmd NVARCHAR (max) NULL -- 执行的sql语句
, CONSTRAINT PK_Qy_ErrorMsg PRIMARY KEY (Id)
)
GO
2.BPM only provides an attempt to provide data sources, and the data sources are still in their system. Due to the large amount of migration, all the data sources they provide are first migrated to the database through database bridging. The intermediate library [intermediate table] of your own database; try to batch generate SQL statements that synchronize data to your own intermediate library through VIEW_SQL_Proc_BPM_IN.You can also further divide the queried SQL into stored procedures Proc_IN_BPM, so It is more convenient to perform synchronization.
create VIEW VIEW_SQL_Proc_BPM_IN AS
SELECT
'IF OBJECT_ID (''dbo.'+BPMTable+''') IS NOT NULL DROP TABLE dbo.'+BPMTable+';' AS SQL
FROM VIEW_QY_Tables
WHERE BPMTable <> ''
UNION ALL
SELECT
'SELECT * INTO '+BPMTable+' from openquery(BPM, ''select * from sgbpm_prd.'+BPMTable + ''') TEMP;' AS SQL
FROM VIEW_QY_Tables
WHERE BPMTable <> ''
GO
--中间如果有报错，根据实际情况更改下sql的查询方式即可
CREATE proc [dbo].[Proc_IN_BPM]
as
begin
--DROP TABLE bpm_fkjl_heyue_view;
DROP TABLE bpm_QY_BudgetMapKM_view ;
DROP TABLE bpm_QY_BudgetMapZT_view ;
DROP TABLE bpm_QY_BudgetZT_view ;
DROP TABLE bpm_QY_Company_view ;
DROP TABLE bpm_QY_ContractAppendices_view ;
DROP TABLE bpm_QY_ContractBasic_view ;
DROP TABLE bpm_QY_ContractCost_view ;
DROP TABLE bpm_QY_ContractMatter_view ;
DROP TABLE bpm_QY_ContractParty_view ;
DROP TABLE bpm_QY_ContractPayment_view ;
DROP TABLE bpm_QY_ContractType_view ;
DROP TABLE bpm_QY_Department_view ;
DROP TABLE bpm_QY_Legal_view ;
DROP TABLE bpm_QY_LX_Basic_view ;
DROP TABLE bpm_QY_LX_File_view ;
DROP TABLE bpm_QY_LX_Release_view ;
DROP TABLE bpm_QY_LX_Subject_view ;
DROP TABLE bpm_QY_Person_view ;
DROP TABLE bpm_QY_Project_view ;
DROP TABLE bpm_QY_Provision_gys_view ;
DROP TABLE bpm_QY_Provision_jtmx_view ;
DROP TABLE bpm_QY_Provision_view ;
DROP TABLE bpm_QY_Subject_view ;
DROP TABLE bpm_QY_YS_File_view ;
DROP TABLE bpm_QY_YS_PayPlan_view ;
DROP TABLE bpm_QY_YS_Settlement_view ;
DROP TABLE bpm_QY_YS_Subject_view ;
DROP TABLE bpm_QY_YS_view ;
--DROP TABLE BPM_JITICHONGXIAO_NEW_VIEW ;
--DROP TABLE BPM_JITICHONGXIAOMAIN_NEW_VIEW ;
--DROP TABLE bpm_toheyue_htfkjlkemu_view;
--DROP TABLE bpm_lxiangyanshoukemu_view;
--DROP TABLE bpm_htyufukuankemu_view;
SELECT * INTO bpm_QY_ContractType_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_ContractType_view') TEMP;
SELECT * INTO bpm_QY_Legal_view
from openquery(BPM,'select
NBFRZTID, GSMC, NSRSBH, FDDLR, GSDZ, YZBM, LXDH, BZ, NBFRBH, KAIHUHANG, KAIHUMINGCHENG, KAIHUZHUANGHAO, CJR, CJSJ,
XGR, XGSJ, LIANHANCODE, AREACODEINFO, REC_PROVINCE, REC_CITY, USAGE, TYPES ,cast(RecordStatus AS varchar(10)) AS RecordStatus
from sgbpm_prd.bpm_QY_Legal_view') a;
SELECT * INTO bpm_QY_Subject_view from openquery(BPM, 'select
YUSUANKEMUID ,
YUSUANKEMUMINGCHENG ,
YUSUANKEMUBIANHAO ,
FUJIKEMUID ,
KEMUJIBIE ,
cast(SHIFOUMOJIKEMU AS varchar(10)) AS SHIFOUMOJIKEMU,
CHANNELTYPE ,
ISBAOXIAO ,
COSTTYPE ,
ISCHECKEXTRACT ,
ISCHECKPACT ,
ISHOUSE ,
ISNEGATIVE ,
ISKEMUADJUST ,
MINLXZHAOBIAOJINE ,
MAXFEIHETONGLXJINE ,
MAXLXYANSHOUJINE ,
YUSUANLEIXING ,
PROJECTTYPE ,
SUBJECTTYPE ,
ASSESSMENT_TYPE ,
YUSUANKEMUSHUILV ,
GXMC ,
YUSUANKEMUSHUOMING ,
XIUGAIREN ,
XIUGAISHIJIAN ,
cast(RECORDSTATUS AS varchar(10)) AS RECORDSTATUS
from sgbpm_prd.bpm_QY_Subject_view') TEMP;
SELECT * INTO bpm_QY_BudgetMapKM_view
from openquery(BPM,'select
YUSUANKEMUID ,
NEWKMMC ,
NEWKMBH ,
CHANNELTYPE ,
COSTTYPE ,
PROJECTTYPE ,
SUBJECTTYPE ,
OLDKMMC ,
OLDKMBH ,
CJR ,
CJSJ ,
XGR ,
XGSJ ,
cast(RecordStatus AS varchar(10)) AS RecordStatus
from sgbpm_prd.bpm_QY_BudgetMapKM_view') a;
SELECT * INTO bpm_QY_Project_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_Project_view') TEMP;
SELECT * INTO bpm_QY_BudgetZT_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_BudgetZT_view') TEMP;
SELECT * INTO bpm_QY_BudgetMapZT_view from openquery(BPM, 'select
ID, OLDZTBH, OLDZTMC, NEWZTID, NEWZTBH, NEWZTMC, CJR, CJSJ, XGR, XGSJ, TYPE, STATUS, TYPENAME
,
cast(RecordStatus AS varchar(10)) AS RecordStatus
from sgbpm_prd.bpm_QY_BudgetMapZT_view') TEMP;
SELECT * INTO bpm_QY_ContractBasic_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_ContractBasic_view') TEMP;
SELECT * INTO bpm_QY_ContractParty_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_ContractParty_view') TEMP;
SELECT * INTO bpm_QY_ContractPayment_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_ContractPayment_view') TEMP;
SELECT * INTO bpm_QY_ContractAppendices_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_ContractAppendices_view') TEMP;
SELECT * INTO bpm_QY_ContractCost_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_ContractCost_view') TEMP;
SELECT * INTO bpm_QY_YS_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_YS_view') TEMP;
SELECT * INTO bpm_QY_YS_PayPlan_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_YS_PayPlan_view') TEMP;
SELECT * INTO bpm_QY_YS_Subject_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_YS_Subject_view') TEMP;
SELECT * INTO bpm_QY_YS_Settlement_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_YS_Settlement_view') TEMP;
SELECT * INTO bpm_QY_YS_File_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_YS_File_view') TEMP;
--SELECT * INTO bpm_fkjl_heyue_view from openquery(BPM, 'select * from sgbpm_prd.bpm_fkjl_heyue_view') TEMP;
SELECT * INTO bpm_QY_LX_Basic_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_LX_Basic_view') TEMP;
SELECT * INTO bpm_QY_LX_Subject_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_LX_Subject_view') TEMP;
SELECT * INTO bpm_QY_LX_File_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_LX_File_view') TEMP;
SELECT * INTO bpm_QY_LX_Release_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_LX_Release_view') TEMP;
SELECT * INTO bpm_QY_ContractMatter_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_ContractMatter_view') TEMP;
SELECT * INTO bpm_QY_Provision_gys_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_Provision_gys_view') TEMP;
SELECT * INTO bpm_QY_Provision_jtmx_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_Provision_jtmx_view') TEMP;
SELECT * INTO bpm_QY_Provision_view from openquery(BPM, 'select * from sgbpm_prd.bpm_QY_Provision_view') TEMP;
SELECT * INTO bpm_QY_Department_view from VIEW_SS_bpm_QY_Department_view TEMP;
SELECT * INTO bpm_QY_Company_view from VIEW_SS_bpm_QY_Company_view TEMP;
SELECT * INTO bpm_QY_Person_view from VIEW_EAS_PERSON TEMP;
--SELECT * INTO BPM_JITICHONGXIAO_NEW_VIEW from openquery(BPM,'select * from sgbpm_prd.BPM_JITICHONGXIAO_NEW_VIEW where xmbh
--IN(''HSHS379'',''ZJZJ457'',''XHS288'',''HSHS370'') and ZTLX=''1''') a;
--SELECT * INTO BPM_JITICHONGXIAOMAIN_NEW_VIEW from openquery(BPM,'select * from sgbpm_prd.BPM_JITICHONGXIAOMAIN_NEW_VIEW') a;
--SELECT
--* INTO bpm_toheyue_htfkjlkemu_view
-- from openquery(BPM,'select * from sgbpm_prd.bpm_toheyue_htfkjlkemu_view') a;
--SELECT * FROM QY_Project WHERE [项目名称] IN ('成都温江合生汇','北京朝阳合生汇','广州海珠合生广场南区','广州海珠合生广场北区')
--SELECT * INTO bpm_lxiangyanshoukemu_view from openquery(BPM,'select * from sgbpm_prd.bpm_lxiangyanshoukemu_view') a;
--SELECT * INTO bpm_htyufukuankemu_view from openquery(BPM,'select * from sgbpm_prd.bpm_htyufukuankemu_view') a;
end
GO
3. After executing the above Proc_IN_BPM stored procedure, we will find that there are many more tables under the BPMTable column in the QY_Tables table in our intermediate library. This is the data source table we have synchronized. All subsequent operations are performed on these tables. deal with
4. Cleaning process:
- Write the view VIEW_SQL_Proc_IN to obtain the data source table of the Chinese field of the third-party data source. The main purpose of obtaining the Chinese field is to facilitate data verification.
create VIEW VIEW_SQL_Proc_IN AS SELECT 'CREATE proc Proc_IN_'+QyTableName+' AS begin INSERT INTO '+QyTableName+ ' ('+LEFT(BPMList2,LEN(BPMList2)-1)+') '+ ' SELECT ' +LEFT(BPMList,LEN(BPMList)-1) + ' FROM '+BPMVIEW + ' END GO ' AS SQL2 FROM ( SELECT QyTableName,BPMTable,BPMVIEW, ( SELECT SQL1+',' FROM VIEW_CheckSQL WHERE QyTableName=A.QyTableName FOR XML PATH('') ) AS BPMList, ( SELECT QyTableFieldName+',' FROM VIEW_CheckSQL WHERE QyTableFieldCodeBPM IS NOT NULL AND QyTableName=A.QyTableName FOR XML PATH('') ) AS BPMList2 FROM VIEW_CheckSQL A --WHERE QyTableName='QY_ContractType' GROUP BY QyTableName,BPMTable,BPMVIEW )TEMP WHERE BPMList <> '' GO create VIEW VIEW_CheckSQL AS SELECT T.Business, T.TableName, T.BPMTable, T.BPMVIEW, M.QyTableName, M.QyTableFieldName, M.QyTableFieldCodeBPM, M.QyIsNull, M.QyIsNullBPM, CASE WHEN M.QyDeaultBPMValue IS NULL THEN M.QyTableFieldCodeBPM+ +' AS ['+M.QyTableFieldName+']' ELSE 'ISNULL('+M.QyTableFieldCodeBPM+','''+M.QyDeaultBPMValue+''')'+' AS ['+M.QyTableFieldName+']' END AS SQL1 FROM VIEW_Qy_Maping M LEFT JOIN VIEW_QY_Tables T ON T.TableCode=M.QyTableName WHERE QyTableFieldName IS NOT NULL GO - Querying VIEW_SQL_Proc_IN will get the execution statement of a stored procedure for all tables. After copying the query results, execute them in batches and then encapsulate them into stored procedures.
CREATE proc [Proc_IN_QY] as BEGIN DELETE FROM QY_BudgetMapKM ; DELETE FROM QY_BudgetMapZT ; DELETE FROM QY_BudgetZT ; DELETE FROM QY_ContractAppendices ; DELETE FROM QY_ContractBasic ; DELETE FROM QY_ContractCost ; DELETE FROM QY_ContractParty ; DELETE FROM QY_ContractPayment ; DELETE FROM QY_ContractType ; --DELETE FROM QY_Department ; --DELETE FROM QY_Legal ; DELETE FROM QY_LX_Basic ; DELETE FROM QY_LX_File ; DELETE FROM QY_LX_Release ; DELETE FROM QY_LX_Subject ; DELETE FROM QY_Project ; DELETE FROM QY_ProjectLegal ; DELETE FROM QY_Subject ; DELETE FROM QY_YS ; DELETE FROM QY_YS_File ; DELETE FROM QY_YS_PayPlan ; DELETE FROM QY_YS_Settlement ; DELETE FROM QY_YS_Subject ; DELETE FROM QY_YS_Pay ; DELETE FROM QY_Provision ; DELETE FROM QY_Provision_jtmx ; DELETE FROM QY_Provision_gys ; DELETE FROM QY_ContractMatter ; DELETE FROM QY_Company ; --DELETE FROM QY_Person ; --DELETE FROM QY_ProjectLegal ; DELETE FROM QY_ProjectMap ; --DELETE FROM QY_JITICHONGXIAOMAIN ; exec Proc_IN_QY_BudgetMapKM ; exec Proc_IN_QY_BudgetMapZT ; exec Proc_IN_QY_BudgetZT ; exec Proc_IN_QY_ContractAppendices ; exec Proc_IN_QY_ContractBasic ; exec Proc_IN_QY_ContractCost ; exec Proc_IN_QY_ContractParty ; exec Proc_IN_QY_ContractPayment ; exec Proc_IN_QY_ContractType ; --exec Proc_IN_QY_Legal ; exec Proc_IN_QY_LX_Basic ; exec Proc_IN_QY_LX_File ; exec Proc_IN_QY_LX_Release ; exec Proc_IN_QY_LX_Subject ; exec Proc_IN_QY_Project ; exec Proc_IN_QY_Subject ; exec Proc_IN_QY_YS ; exec Proc_IN_QY_YS_File ; exec Proc_IN_QY_YS_PayPlan ; exec Proc_IN_QY_YS_Settlement ; exec Proc_IN_QY_YS_Subject ; exec Proc_IN_QY_YS_Pay exec Proc_IN_QY_ContractMatter ; exec Proc_IN_QY_Provision ; exec Proc_IN_QY_Provision_jtmx ; exec Proc_IN_QY_Provision_gys ; --exec Proc_IN_QY_Department ; --exec Proc_IN_QY_Person ; exec Proc_IN_QY_Company ; --exec Proc_IN_QY_ProjectLegal ; exec Proc_IN_QY_ProjectMap ; --exec Proc_IN_QY_JITICHONGXIAOMAIN ; exec Proc_BPM_Clear ; end GO -
After executing the Proc_IN_QY stored procedure, we will get the Chinese data source tables of all migrated data sources. The main purpose is to facilitate viewing of the data sources.
-
Next, write the view VIEW_SQL_Proc_OUT. We write the data source of the current intermediate library to the target library.
CREATE VIEW VIEW_SQL_Proc_OUT AS SELECT 'if OBJECT_ID(''dbo.PROC_OUT_'+TableCode+''',''procedure'') is not null BEGIN DROP PROC dbo.PROC_OUT_' + TableCode + ' END GO' AS SQL1 FROM VIEW_QY_Tables WHERE BPMTable <> '' UNION SELECT 'CREATE proc Proc_OUT_'+TableCode+' AS begin INSERT INTO [HY].[HopsonCost].[dbo].'+TableCode+ ' SELECT * FROM '+TableCode + ' END GO ' AS SQL1 FROM VIEW_QY_Tables WHERE BPMTable <> '' GO -
Executing VIEW_SQL_Proc_OUT will get all the written stored procedure codes, paste and copy them, and execute them.
-
Similarly, package the above stored procedure into Proc_OUT_QY for easy execution
CREATE proc [dbo].[Proc_OUT_QY] as begin DELETE FROM [HY].[HopsonCost].[dbo].QY_BudgetMapKM; DELETE FROM [HY].[HopsonCost].[dbo].QY_BudgetMapZT; DELETE FROM [HY].[HopsonCost].[dbo].QY_BudgetZT; DELETE FROM [HY].[HopsonCost].[dbo].QY_ContractAppendices; DELETE FROM [HY].[HopsonCost].[dbo].QY_ContractBasic; DELETE FROM [HY].[HopsonCost].[dbo].QY_ContractCost; DELETE FROM [HY].[HopsonCost].[dbo].QY_ContractParty; DELETE FROM [HY].[HopsonCost].[dbo].QY_ContractPayment; DELETE FROM [HY].[HopsonCost].[dbo].QY_ContractType; --DELETE FROM [HY].[HopsonCost].[dbo].QY_Department; --DELETE FROM [HY].[HopsonCost].[dbo].QY_Legal; DELETE FROM [HY].[HopsonCost].[dbo].QY_LX_Basic; DELETE FROM [HY].[HopsonCost].[dbo].QY_LX_File; DELETE FROM [HY].[HopsonCost].[dbo].QY_LX_Release; DELETE FROM [HY].[HopsonCost].[dbo].QY_LX_Subject; DELETE FROM [HY].[HopsonCost].[dbo].QY_Project; --DELETE FROM [HY].[HopsonCost].[dbo].QY_ProjectLegal; DELETE FROM [HY].[HopsonCost].[dbo].QY_Subject; DELETE FROM [HY].[HopsonCost].[dbo].QY_YS; DELETE FROM [HY].[HopsonCost].[dbo].QY_YS_File; DELETE FROM [HY].[HopsonCost].[dbo].QY_YS_PayPlan; DELETE FROM [HY].[HopsonCost].[dbo].QY_YS_Settlement; DELETE FROM [HY].[HopsonCost].[dbo].QY_YS_Subject; DELETE FROM [HY].[HopsonCost].[dbo].QY_YS_Pay; DELETE FROM [HY].[HopsonCost].[dbo].QY_Company; --DELETE FROM [HY].[HopsonCost].[dbo].QY_Person; --DELETE FROM [HY].[HopsonCost].[dbo].QY_ProjectLegal; DELETE FROM [HY].[HopsonCost].[dbo].QY_ProjectMap; DELETE FROM [HY].[HopsonCost].[dbo].QY_ContractMatter; DELETE FROM [HY].[HopsonCost].[dbo].QY_Provision; DELETE FROM [HY].[HopsonCost].[dbo].QY_Provision_jtmx; DELETE FROM [HY].[HopsonCost].[dbo].QY_Provision_gys; --DELETE FROM [HY].[HopsonCost].[dbo].QY_JITICHONGXIAOMAIN ; exec Proc_OUT_QY_BudgetMapKM ; exec Proc_OUT_QY_BudgetMapZT ; exec Proc_OUT_QY_BudgetZT ; exec Proc_OUT_QY_ContractAppendices ; exec Proc_OUT_QY_ContractBasic ; exec Proc_OUT_QY_ContractCost ; exec Proc_OUT_QY_ContractParty ; exec Proc_OUT_QY_ContractPayment ; exec Proc_OUT_QY_ContractType ; --exec Proc_OUT_QY_Legal ; exec Proc_OUT_QY_Project ; exec Proc_OUT_QY_Subject ; exec Proc_OUT_QY_YS ; exec Proc_OUT_QY_YS_File ; exec Proc_OUT_QY_YS_PayPlan ; exec Proc_OUT_QY_YS_Settlement ; exec Proc_OUT_QY_YS_Subject ; exec Proc_OUT_QY_YS_Pay; exec Proc_OUT_QY_LX_Basic ; exec Proc_OUT_QY_LX_Subject ; exec Proc_OUT_QY_LX_File ; exec Proc_OUT_QY_LX_Release ; --exec Proc_OUT_QY_Department ; --exec Proc_OUT_QY_ProjectLegal ; exec Proc_OUT_QY_Company ; --exec Proc_OUT_QY_Person ; exec Proc_OUT_QY_ProjectMap ; exec Proc_OUT_QY_ContractMatter ; exec Proc_OUT_QY_Provision ; exec Proc_OUT_QY_Provision_gys ; exec Proc_OUT_QY_Provision_jtmx ; --exec Proc_OUT_QY_JITICHONGXIAOMAIN ; --DROP TABLE [HY].[HopsonCost].[dbo].bpm_fkjl_heyue_view; --DROP TABLE [HY].[HopsonCost].[dbo].bpm_toheyue_htfkjlkemu_view; --SELECT * INTO [HY].[HopsonCost].[dbo].bpm_fkjl_heyue_view FROM bpm_fkjl_heyue_view; --SELECT * INTO [HY].[HopsonCost].[dbo].bpm_toheyue_htfkjlkemu_view FROM bpm_toheyue_htfkjlkemu_view; --DROP TABLE [HY].[HopsonCost].[dbo].bpm_htyufukuankemu_view; --DROP TABLE [HY].[HopsonCost].[dbo].bpm_lxiangyanshoukemu_view; --SELECT * INTO [HY].[HopsonCost].[dbo].bpm_htyufukuankemu_view FROM bpm_htyufukuankemu_view; --SELECT * INTO [HY].[HopsonCost].[dbo].bpm_lxiangyanshoukemu_view FROM bpm_lxiangyanshoukemu_view; end GO -
After executing the above stored procedure, we will find that the data source table of our Chinese field is generated in our target library
5. Data writing
1. Based on the obtained data source table, code can be coded, and Qy_FieldDic and Qy_ErrorMsg can be used to convert related field dictionaries and log records.
2. Introduction to the code of a certain business
/// <summary>
/// 付款申请数据迁移
/// 1.1 主表记录
/// </summary>
/// <returns></returns>
public virtual HttpResponseMessage UpPaymentRequest()
{
try
{
int totalCount = 0, successCount = 0, errorCout = 0;
string fromTableName = "QY_PaymentRequest";
string toTableName = "FM_PaymentRequest";
string qyBUSSID = "PaymentRequestBussID";
string qyCreateUserID = "CreateUserID";
string qyModifyUserID = "ModifyUserID";
string qyProjectID = "ProjectID";
string qyContractID = "ContractID";
string qyProAppID = "ProAppID";
string qyImprestID = "ImprestID";
using (var context = new MasterDbContext())
{
List<Qy_Maping> mapings = context.Qy_Maping.Where(ss => ss.QyTableName == fromTableName).ToList();
List<Qy_FieldDic> fieldDics = context.Qy_FieldDic.Where(x=>x.BPMTABLE == fromTableName).ToList();
var BUSSID = mapings.FirstOrDefault(ss => ss.QyTableFieldCode == qyBUSSID);
if (BUSSID == null)
{
return WebApiHelper.GetErrorHttpResponseMessage(new { Messsage = "不存在业务ID" });
}
DataTable dataTable = context.ExecuteDataTable("select * from " + fromTableName);
totalCount = dataTable.Rows.Count;
LogTxtInfo("需要迁移" + totalCount + "条");
var CreateUserID = mapings.FirstOrDefault(ss => ss.QyTableFieldCode == qyCreateUserID);
var ModifyUserID = mapings.FirstOrDefault(ss => ss.QyTableFieldCode == qyModifyUserID);
var ProjectID = mapings.FirstOrDefault(ss => ss.QyTableFieldCode == qyProjectID);
var ProAppID = mapings.FirstOrDefault(ss => ss.QyTableFieldCode == qyProAppID);
var ContractID = mapings.FirstOrDefault(ss => ss.QyTableFieldCode == qyContractID);
var ImprestID = mapings.FirstOrDefault(ss => ss.QyTableFieldCode == qyImprestID);
for (int i = 0; i < dataTable.Rows.Count; i++)
{
DataRow row = dataTable.Rows[i];
object obj = row[BUSSID.QyTableFieldName];
if (obj == null)
{
//记录日志
errorCout++;
WriteLog(context, fromTableName, toTableName, "业务ID为空");
continue;
}
object objCreateUserID = row[CreateUserID.QyTableFieldName];
object objModifyUserID = row[ModifyUserID.QyTableFieldName];
object objProjectID = row[ProjectID.QyTableFieldName];
object objProAppID = row[ProAppID.QyTableFieldName];
string objContractID = row[ContractID.QyTableFieldName].ToStr();
string objImprestID = row[ImprestID.QyTableFieldName].ToStr();
int ProBUSSID = 0;
if (objProjectID != null)
{
ProBUSSID = context.Database.SqlQuery<int>("SELECT ProjectID FROM dbo.CMN_Project WHERE ProBUSSID = '" + objProjectID.ToString() + "'").FirstOrDefault();
ProjectID.QyDefaultValue = ProBUSSID.ToStr();
}
int Plixiangid = 0;
if(ProAppID != null){
Plixiangid = context.Database.SqlQuery<int>("SELECT ProAppID FROM CMN_SyncProjectApproval WHERE ProAppBUSSID = '" + objProAppID + "'").FirstOrDefault();
ProAppID.QyDefaultValue = Plixiangid.ToString();
}
int CUserID = 0;
if (objCreateUserID != null)
{
CUserID = context.Database.SqlQuery<int>("SELECT USERID FROM dbo.CF_USER WHERE USERNO = '" + objCreateUserID + "'").FirstOrDefault();
CreateUserID.QyDefaultValue = CUserID.ToString();
}
int MUserID = 0;
if (objCreateUserID != null)
{
MUserID = context.Database.SqlQuery<int>("SELECT USERID FROM dbo.CF_USER WHERE USERNO = '" + objModifyUserID + "'").FirstOrDefault();
ModifyUserID.QyDefaultValue = MUserID.ToString();
}
if (objContractID != "") {
var tempContract = context.CON_Contract.FirstOrDefault(x => x.ContractIDBUSSID == objContractID);
if (tempContract != null)
{
ContractID.QyDefaultValue = tempContract.ContractID.ToStr();
}
else {
LogTxtInfo("迁移失败： 未找到合同信息：objContractID=" + objContractID);
//记录日志
errorCout++;
WriteLog(context, fromTableName, toTableName, "迁移失败： 未找到合同信息：objContractID=" + objContractID);
continue;
}
}
if (objImprestID != "") {
var tempImprest = context.FM_Imprest.FirstOrDefault(x => x.BPMmainId == objImprestID);
if (tempImprest != null)
{
ImprestID.QyDefaultValue = tempImprest.ImprestID.ToStr();
}
else
{
LogTxtInfo("迁移失败： 未找到备用金信息：objImprestID=" + objImprestID);
//记录日志
errorCout++;
WriteLog(context, fromTableName, toTableName, "迁移失败： 未找到备用金信息：objImprestID=" + objImprestID);
continue;
}
}
string updateWhere = string.Empty;
if (obj.ToString() == "NULL")
{
updateWhere = " WHERE 1=2";
}
else
{
updateWhere = " WHERE " + qyBUSSID + "='" + obj.ToString() + "'";
}
string sqlCmdInsert = string.Empty;
string sqlCmdUpdate = string.Empty;
try
{
using (var dbContextTransaction = context.Database.BeginTransaction())
{
sqlCmdInsert = CreateInsertSqlCmd(row, toTableName, mapings, fieldDics);
sqlCmdUpdate = CreateUpdateSqlCmd(row, toTableName, mapings, fieldDics) + updateWhere;
List<SqlParameter> paramArray = new List<SqlParameter>();
paramArray.Add(new SqlParameter("@bussid", obj.ToString()));
paramArray.Add(new SqlParameter("@insertCmd", sqlCmdInsert));
paramArray.Add(new SqlParameter("@updateCmd", sqlCmdUpdate));
paramArray.Add(new SqlParameter("@tableName", toTableName));
context.Database.ExecuteSqlCommand("[dbo].[Qy_InsertOrUpdateAndInsertPrimarykeyRelation] @bussid , @insertCmd , @updateCmd , @tableName ", paramArray.ToArray());
dbContextTransaction.Commit();
successCount++;
}
}
catch (Exception ex)
{
//记录日志
errorCout++;
WriteLog(context, fromTableName, toTableName, ex.ToString(), sqlCmdInsert, sqlCmdUpdate);
continue;
}
}
}
LogTxtInfo("迁移完毕,总计：" + totalCount + "条，成功：" + successCount + "条，失败：" + errorCout + "条");
return WebApiHelper.GetOkHttpResponseMessage(new { Messsage = "迁移完毕,总计：" + totalCount + "条，成功：" + successCount + "条，失败：" + errorCout + "条" });
}
catch (Exception ex)
{
LogTxtInfo("迁移异常：" + ex.Message.ToString());
return WebApiHelper.GetErrorHttpResponseMessage(new { Messsage = ex.Message.ToString() });
}
}
/// <summary>
/// 创建插入数据SQL
/// </summary>
/// <param name="dataRow"></param>
/// <param name="tableName"></param>
/// <param name="mapings"></param>
/// <param name="fieldDics"></param>
/// <returns></returns>
private string CreateInsertSqlCmd(DataRow dataRow, string tableName, List<Qy_Maping> mapings, List<Qy_FieldDic> fieldDics)
{
mapings = mapings.Where(ss => ss.QyTableFieldCode != null).ToList();
StringBuilder sqlAppend = new StringBuilder();
sqlAppend.Append("INSERT INTO " + tableName + "");
sqlAppend.Append("( ");
List<string> fields = new List<string>();
foreach (var item in mapings)
{
fields.Add(item.QyTableFieldCode);
}
sqlAppend.Append(string.Join(",", fields));
sqlAppend.Append(")");
sqlAppend.Append("VALUES ");
sqlAppend.Append("( ");
List<string> values = new List<string>();
for (int k = 0; k < mapings.Count; k++)
{
Qy_Maping maping = mapings[k];
try
{
string QyTableFieldName = maping.QyTableFieldName;
if (string.IsNullOrEmpty(QyTableFieldName))
{
if (string.IsNullOrEmpty(maping.QyDefaultValue))
{
values.Add("NULL");
}
else
{
//values.Add("'" + mapings[k].QyDefaultValue + "'");
values.Add(GetFieldTypeForValue(maping.QyTableFieldType, maping.QyDefaultValue));
}
}
else
{
//如果设置默认值，则强制覆盖
if (!string.IsNullOrEmpty(maping.QyDefaultValue))
{
//values.Add("'" + mapings[k].QyDefaultValue + "'");
values.Add(GetFieldTypeForValue(maping.QyTableFieldType, maping.QyDefaultValue));
}
else
{
object obj = dataRow[QyTableFieldName];
if (obj != null && !string.IsNullOrEmpty(obj.ToString()))
{
string value = GetValue(obj, maping.QyPOPTYPE, maping.QyPOPTYPEValueSouce, fieldDics);
if (value == "NULL")
values.Add(value);
else
//values.Add(value);
values.Add(GetFieldTypeForValue(maping.QyTableFieldType, value));
}
else
{
if (!string.IsNullOrEmpty(maping.QyRelaceNullValue))
{
values.Add(GetFieldTypeForValue(maping.QyTableFieldType, maping.QyRelaceNullValue));
}
else
values.Add(" NULL ");
}
}
}
}
catch (Exception ex)
{
throw new Exception("错误字段：" + maping.QyTableFieldName + ",错误消息：" + ex.Message);
}
}
sqlAppend.Append(string.Join(",", values));
sqlAppend.Append(")");
return sqlAppend.ToString();
}
/// <summary>
/// 创建更新数据SQL
/// </summary>
/// <param name="dataRow"></param>
/// <param name="tableName"></param>
/// <param name="mapings"></param>
/// <param name="fieldDics"></param>
/// <returns></returns>
private string CreateUpdateSqlCmd(DataRow dataRow, string tableName, List<Qy_Maping> mapings, List<Qy_FieldDic> fieldDics)
{
mapings = mapings.Where(ss => ss.QyTableFieldCode != null).ToList();
List<string> sqlCmdUpdateTabel = new List<string>();
StringBuilder sqlAppend = new StringBuilder();
sqlAppend.Append("UPDATE " + tableName + " SET ");
List<string> values = new List<string>();
for (int k = 0; k < mapings.Count; k++)
{
Qy_Maping maping = mapings[k];
try
{
string QyTableFieldName = maping.QyTableFieldName;
if (string.IsNullOrEmpty(QyTableFieldName))
{
if (string.IsNullOrEmpty(maping.QyDefaultValue))
{
values.Add(maping.QyTableFieldCode + "= NULL ");
}
else
{
values.Add(mapings[k].QyTableFieldCode + " = " + GetFieldTypeForValue(maping.QyTableFieldType, maping.QyDefaultValue));
}
}
else
{
//如果设置默认值，则强制覆盖
if (!string.IsNullOrEmpty(maping.QyDefaultValue))
{
values.Add(maping.QyTableFieldCode + "= " + GetFieldTypeForValue(maping.QyTableFieldType, maping.QyDefaultValue));
}
else
{
object obj = dataRow[QyTableFieldName];
if (obj != null && !string.IsNullOrEmpty(obj.ToString()))
{
string value = GetValue(obj, maping.QyPOPTYPE, maping.QyPOPTYPEValueSouce, fieldDics);
if (value == "NULL")
values.Add(maping.QyTableFieldCode + "= NULL ");
else
values.Add(maping.QyTableFieldCode + " = " + GetFieldTypeForValue(maping.QyTableFieldType, value));
}
else
{
if (!string.IsNullOrEmpty(maping.QyRelaceNullValue))
{
values.Add(maping.QyTableFieldCode + " = " + GetFieldTypeForValue(maping.QyTableFieldType, maping.QyRelaceNullValue));
}
else
values.Add(maping.QyTableFieldCode + "= NULL ");
}
}
}
}
catch (Exception ex)
{
throw new Exception("错误字段：" + maping.QyTableFieldName + ",错误消息：" + ex.Message);
}
}
sqlAppend.Append(string.Join(",", values));
return sqlAppend.ToString();
}
/// <summary>
/// 根据字段类型创建SQL语句的Values
/// </summary>
/// <param name="QyTableFieldType"></param>
/// <param name="Value"></param>
/// <returns></returns>
private string GetFieldTypeForValue(string QyTableFieldType, string Value)
{
try
{
if (string.IsNullOrEmpty(QyTableFieldType))
return "'" + Value + "'";
string fieldValue = string.Empty;
switch (QyTableFieldType)
{
case "char": fieldValue = "'" + Value + "'"; break;
case "date": fieldValue = "'" + Value + "'"; break;
case "datetime": fieldValue = "'" + Convert.ToDateTime(Value).ToString("yyyy-MM-dd HH:mm:ss") + "'"; break;
case "int": fieldValue = Convert.ToInt32(Value).ToString(); break;
case "decimal": fieldValue = Convert.ToDecimal(Value).ToString(); break;
case "nvarchar": fieldValue = "'" + Value.Replace("'", "’") + "'"; break;
case "VARCHAR2": fieldValue = "'" + Value.Replace("'", "’") + "'"; break;
case "NUMBER": fieldValue = Convert.ToInt32(Value).ToString(); break;
case "NUMBER(20,4)": fieldValue = Convert.ToDecimal(Value).ToString(); break;
case "TIMESTAMP": fieldValue = "to_timestamp('" + Convert.ToDateTime(Value).ToString() + "','YYYY-MM-DDHH24:MI:SS')"; break;
case "DATE": fieldValue = "to_date('" + Convert.ToDateTime(Value).ToString() + "','YYYY-MM-DDHH24:MI:SS')"; break;
default: fieldValue = "'" + Value + "'"; break;
}
return fieldValue;
}
catch (Exception ex)
{
throw new Exception(ex.Message);
}
}
/// <summary>
/// 记录错误日志
/// </summary>
/// <param name="context"></param>
/// <param name="fromTable"></param>
/// <param name="toTable"></param>
/// <param name="errorMsg"></param>
/// <param name="sqlCmdInsert"></param>
/// <param name="sqlCmdUpdate"></param>
private void WriteLog(MasterDbContext context, string fromTable, string toTable, string errorMsg, string sqlCmdInsert = "", string sqlCmdUpdate = "")
{
string sqlCmdLog = "INSERT INTO[dbo].[Qy_ErrorMsg]([FromTableName],[ToTableName],[ErrorMsg],[ErrorInsertSqlCmd],[ErrorUpdateSqlCmd]) VALUES(@FromTableName,@ToTableName,@ErrorMsg,@ErrorInsertSqlCmd,@ErrorUpdateSqlCmd)";
List<SqlParameter> paramArray = new List<SqlParameter>();
paramArray.Add(new SqlParameter("@FromTableName", fromTable));
paramArray.Add(new SqlParameter("@ToTableName", toTable));
paramArray.Add(new SqlParameter("@ErrorMsg", errorMsg));
paramArray.Add(new SqlParameter("@ErrorInsertSqlCmd", sqlCmdInsert));
paramArray.Add(new SqlParameter("@ErrorUpdateSqlCmd", sqlCmdUpdate));
context.Database.ExecuteSqlCommand(sqlCmdLog, paramArray.ToArray());
}Summarize
At this point, the overall migration is completed. In the migration process, there are actually not many difficulties, it just requires more patienceandcare< a i=4>. Similarly, the above migration ideas can also be simplified. However, I personally think that this is safer and easier to verify later data. Of course, there are many other solutions, welcome to discuss~~~