大容量导入和导出数据 (SQL Server)

其他 2020-05-21 09:16:02 阅读次数: 0

前言

最近手头上接到一个临时任务，使用C++或者Nodejs编写程序，从redis服务器中获取100个站点的keys(env:hbxd:421000401:*),获取站点编码，然后根据站点编码如421000401，遍历获取前缀为env:hbxd:421000401的所有该站点的keys(比如env:hbxd:421000401:20200519:101025)，然后按照某种格式生成这100个站点的数据文件(txt格式),然后再使用bcp命令将数据文件导入到对应的SQLServer数据库中。

使用hiredis封装的Redis客户端类

// RedisClient.h 头文件

#pragma once
#include <type.h>
#include <my_log.h>
#include <Locker.h>

class RedisClient : public my_log
{
public:
	RedisClient(sig_log& lg);
	~RedisClient();

	void set_connect_info(const String& host, const uint16 port, const String& password = "", const uint16 index = 0);

	int connect();

	bool SET(const String& key, const String& value);

	bool SETEX(const String& key, const String& value, const uint32 seconds);

	bool HSET(const String& key, const String& field, const String& value);

	bool EXPIRE(const String& key, int32 seconds);

	bool KEYS(StringArray& _return, const String& pattern);

	bool GET(String& _return, const String& key);

	bool HGETALL(StringMap& _return, const String& key);

	bool DEL(const String& key);

	bool DEL(const StringArray& keys);

	bool HDEL(const String& key, const String& field);

	bool HDEL(const String& key, const StringArray& fields);

protected:
	bool AUTH(struct redisContext* ctx, const String& password);

	bool SELECT(struct redisContext* ctx, const int32 index);

	bool PING(struct redisContext* ctx);

	bool refresh();


private:
	struct redisContext*	redis_ctx_ = nullptr;
	RWMutex		mutex_;

	String host_ = "localhost"; 
	uint16 port_ = 6379; 
	String password_; 
	uint16 index_ = 0;
};

// RedisClient.cpp 实现文件

#include "RedisClient.h"
#include <hiredis/hiredis.h>
#ifdef WIN32
#pragma comment(lib, "hiredis.lib")
#endif
#include <func.h>


RedisClient::RedisClient(sig_log& lg)
	: my_log(lg)
{
}


RedisClient::~RedisClient()
{
}

void RedisClient::set_connect_info(const String& host, const uint16 port, const String& password /*= ""*/, const uint16 index /*= 0*/)
{
	host_ = host;
	port_ = port;
	password_ = password;
	index_ = index;
}

int RedisClient::connect()
{
	redis_ctx_ = redisConnect(host_.c_str(), port_);
	if (!redis_ctx_) {
		return -2;
	}
	if (redis_ctx_->err != REDIS_OK) {
		EmitLog(error, "Redis Connect: %s\n", redis_ctx_->errstr);
		goto failed;
	}
	if (redisEnableKeepAlive(redis_ctx_) != REDIS_OK) {
		EmitLog(error, "redisEnableKeepAlive: %s\n", redis_ctx_->errstr);
		goto failed;
	}
	if (password_.size() && !AUTH(redis_ctx_, password_)) {
		EmitLog(error, "AUTH: %s\n", redis_ctx_->errstr);
		goto failed;
	}
	if (!PING(redis_ctx_)) {
		EmitLog(error, "PING: %s\n", redis_ctx_->errstr);
		goto failed;
	}
	if (!SELECT(redis_ctx_, index_)) {
		EmitLog(error, "SELECT: %s\n", redis_ctx_->errstr);
		goto failed;
	}
	return 0;

failed:
	redisFree(redis_ctx_);
	redis_ctx_ = nullptr;
	return -1;
}

bool RedisClient::refresh()
{
	if (!redis_ctx_) {
		return connect() == 0;
	}
	if (password_.size() && !AUTH(redis_ctx_, password_)) {
		EmitLog(error, "AUTH: %s\n", redis_ctx_->errstr);
		goto failed;
	}
	if (!PING(redis_ctx_)) {
		EmitLog(error, "PING: %s\n", redis_ctx_->errstr);
		goto failed;
	}
	if (!SELECT(redis_ctx_, index_)) {
		EmitLog(error, "SELECT: %s\n", redis_ctx_->errstr);
		goto failed;
	}
	return true;

failed:
	redisFree(redis_ctx_);
	redis_ctx_ = nullptr;
	return false;
}

bool RedisClient::PING(struct redisContext* ctx)
{
	bool rc = false;
	redisReply* reply = (redisReply *)redisCommand(ctx, "PING");
	if (reply) {
		if (reply->type == REDIS_REPLY_ERROR) {
			rc = false;
			EmitLog(error, "PING: %s\n", reply->str);
		}
		else if (reply->type == REDIS_REPLY_STATUS) {
			rc = (strcmp(reply->str, "PONG") == 0);
		}
	}
	freeReplyObject(reply);
	return rc;
}

bool RedisClient::AUTH(struct redisContext* ctx, const String & password)
{
	bool rc = false;
	redisReply* reply = (redisReply *)redisCommand(ctx, "AUTH %s", password.c_str());	
	if (reply) {
		if (reply->type == REDIS_REPLY_ERROR) {
			rc = false;
			EmitLog(error, "AUTH: %s\n", reply->str);
		}
		else if (reply->type == REDIS_REPLY_STATUS) {
			rc = (strcmp(reply->str, "OK") == 0);
		}
	}
	freeReplyObject(reply);
	return rc;
}

bool RedisClient::SELECT(struct redisContext* ctx, const int32 index)
{
	bool rc = false;
	redisReply* reply = (redisReply *)redisCommand(ctx, "SELECT %d", index);
	if (reply) {
		if (reply->type == REDIS_REPLY_ERROR) {
			rc = false;
			EmitLog(error, "SELECT: %s\n", reply->str);
		}
		else if (reply->type == REDIS_REPLY_STATUS) {
			rc = (strcmp(reply->str, "OK") == 0);
		}
	}
	freeReplyObject(reply);
	return rc;
}

bool RedisClient::SET(const String& key, const String& value)
{
	WriteLocker locker(mutex_);
	bool rc = false;
	redisReply* reply = nullptr;
	int retry_count = 3;
	while (!reply && retry_count) {
		if (redis_ctx_) {
			reply = (redisReply *)redisCommand(redis_ctx_, "SET %s %s", key.c_str(), value.c_str());
		}
		if (reply) {
			if (reply->type == REDIS_REPLY_ERROR) {
				rc = false;
				EmitLog(error, "SET: %s\n", reply->str);
			}
			else if (reply->type == REDIS_REPLY_INTEGER) {
				rc = (reply->integer == 1);
			}
		}
		else if (!redis_ctx_ || redis_ctx_->err != REDIS_OK) {
			refresh();
			retry_count--;
		}
		freeReplyObject(reply);
	}
	return rc;
}

bool RedisClient::SETEX(const String& key, const String& value, const uint32 seconds)
{
	WriteLocker locker(mutex_);
	bool rc = false;
	redisReply* reply = nullptr;
	int retry_count = 3;
	while (!reply && retry_count) {
		if (redis_ctx_) {
			reply = (redisReply *)redisCommand(redis_ctx_, "SETEX %s %d %s", key.c_str(), seconds, value.c_str());
		}
		if (reply) {
			if (reply->type == REDIS_REPLY_ERROR) {
				rc = false;
				EmitLog(error, "SETEX: %s\n", reply->str);
			}
			else if (reply->type == REDIS_REPLY_INTEGER) {
				rc = (reply->integer == 1);
			}
		}
		else if (!redis_ctx_ || redis_ctx_->err != REDIS_OK) {
			refresh();
			retry_count--;
		}
		freeReplyObject(reply);
	}
	return rc;
}

bool RedisClient::HSET(const String & key, const String & field, const String & value)
{
	WriteLocker locker(mutex_);
	bool rc = false;
	redisReply* reply = nullptr;
	int retry_count = 3;
	while (!reply && retry_count) {
		if (redis_ctx_) {
			reply = (redisReply *)redisCommand(redis_ctx_, "HSET %s %s %s", key.c_str(), field.c_str(), value.c_str());
		}
		if (reply) {
			if (reply->type == REDIS_REPLY_ERROR) {
				rc = false;
				EmitLog(error, "HSET: %s\n", reply->str);
			}
			else if (reply->type == REDIS_REPLY_INTEGER) {
				rc = (reply->integer == 1);
			}
		}
		else if (!redis_ctx_ || redis_ctx_->err != REDIS_OK) {
			refresh();
			retry_count--;
		}
		freeReplyObject(reply);
	}
	return rc;
}

bool RedisClient::EXPIRE(const String & key, int32 seconds)
{
	WriteLocker locker(mutex_);
	bool rc = false;
	redisReply* reply = nullptr;
	int retry_count = 3;
	while (!reply && retry_count) {
		if (redis_ctx_) {
			reply = (redisReply *)redisCommand(redis_ctx_, "EXPIRE %s %d", key.c_str(), seconds);
		}
		if (reply) {
			if (reply->type == REDIS_REPLY_ERROR) {
				rc = false;
				EmitLog(error, "EXPIRE: %s\n", reply->str);
			}
			else if (reply->type == REDIS_REPLY_INTEGER) {
				rc = (reply->integer == 1);
			}
		}
		else if (!redis_ctx_ || redis_ctx_->err != REDIS_OK) {
			refresh();
			retry_count--;
		}
		freeReplyObject(reply);
	}
	return rc;
}

bool RedisClient::KEYS(StringArray & _return, const String & pattern)
{
	WriteLocker locker(mutex_);
	bool rc = false;
	redisReply* reply = nullptr;
	int retry_count = 3;
	while (!reply && retry_count) {
		if (redis_ctx_) {
			reply = (redisReply *)redisCommand(redis_ctx_, "KEYS %s", pattern.c_str());
		}
		if (reply) {
			if (reply->type == REDIS_REPLY_ERROR) {
				rc = false;
				EmitLog(error, "KEYS: %s\n", reply->str);
			}
			else if (reply->type == REDIS_REPLY_ARRAY) {
				for (size_t i = 0; i < reply->elements; i++) {
					_return.push_back(reply->element[i]->str);
				}
				rc = true;
			}
		}
		else if (!redis_ctx_ || redis_ctx_->err != REDIS_OK) {
			refresh();
			retry_count--;
		}
		freeReplyObject(reply);
	}
	return rc;
}

bool RedisClient::GET(String& _return, const String& key)
{
	WriteLocker locker(mutex_);
	bool rc = false;
	redisReply* reply = nullptr;
	int retry_count = 3;
	while (!reply && retry_count) {
		if (redis_ctx_) {
			reply = (redisReply *)redisCommand(redis_ctx_, "GET %s", key.c_str());
		}
		if (reply) {
			if (reply->type == REDIS_REPLY_ERROR) {
				rc = false;
				EmitLog(error, "GET: %s\n", reply->str);
			}
			else if (reply->type == REDIS_REPLY_STRING) {
				_return = reply->str;
				rc = true;
			}
			else if (reply->type == REDIS_REPLY_INTEGER) {
				_return = std::to_string(reply->integer);
				rc = true;
			}
		}
		else if (!redis_ctx_ || redis_ctx_->err != REDIS_OK) {
			refresh();
			retry_count--;
		}
		freeReplyObject(reply);
	}
	return rc;
}

bool RedisClient::HGETALL(StringMap & _return, const String & key)
{
	WriteLocker locker(mutex_);
	bool rc = false;
	redisReply* reply = nullptr;
	int retry_count = 3;
	while (!reply && retry_count) {
		if (redis_ctx_) {
			reply = (redisReply *)redisCommand(redis_ctx_, "HGETALL %s", key.c_str());
		}
		if (reply) {
			if (reply->type == REDIS_REPLY_ERROR) {
				rc = false;
				EmitLog(error, "HGETALL: %s\n", reply->str);
			}
			else if (reply->type == REDIS_REPLY_ARRAY) {
				for (size_t i = 0; i < reply->elements / 2; i++) {
					_return.insert(std::make_pair(reply->element[i * 2]->str, reply->element[i * 2 + 1]->str));
				}
				rc = true;
			}
		}
		else if (!redis_ctx_ || redis_ctx_->err != REDIS_OK) {
			refresh();
			retry_count--;
		}
		freeReplyObject(reply);
	}
	return rc;
}

bool RedisClient::DEL(const String & key)
{
	WriteLocker locker(mutex_);
	bool rc = false;
	redisReply* reply = nullptr;
	int retry_count = 3;
	while (!reply && retry_count) {
		if (redis_ctx_) {
			reply = (redisReply *)redisCommand(redis_ctx_, "DEL %s", key.c_str());
		}
		if (reply) {
			if (reply->type == REDIS_REPLY_ERROR) {
				rc = false;
				EmitLog(error, "DEL: %s\n", reply->str);
			}
			else if (reply->type == REDIS_REPLY_INTEGER) {
				rc = (reply->integer == 1);
			}
		}
		else if (!redis_ctx_ || redis_ctx_->err != REDIS_OK) {
			refresh();
			retry_count--;
		}
		freeReplyObject(reply);
	}
	return rc;
}

bool RedisClient::DEL(const StringArray & keys)
{
	const String arrstr = Math::Tools::join(keys, " ");
	return DEL(arrstr);
}

bool RedisClient::HDEL(const String & key, const String & field)
{
	WriteLocker locker(mutex_);
	bool rc = false;
	redisReply* reply = nullptr;
	int retry_count = 3;
	while (!reply && retry_count) {
		if (redis_ctx_) {
			reply = (redisReply *)redisCommand(redis_ctx_, "HDEL %s %s", key.c_str(), field.c_str());
		}
		if (reply) {
			if (reply->type == REDIS_REPLY_ERROR) {
				rc = false;
				EmitLog(error, "HDEL: %s\n", reply->str);
			}
			else if (reply->type == REDIS_REPLY_INTEGER) {
				rc = (reply->integer == 1);
			}
		}
		else if (!redis_ctx_ || redis_ctx_->err != REDIS_OK) {
			refresh();
			retry_count--;
		}
		freeReplyObject(reply);
	}
	return rc;
}

bool RedisClient::HDEL(const String & key, const StringArray & fields)
{
	for (auto& field : fields)
	{
		HDEL(key, field);
	}
	return true;
}

上述使用C++封装了hiredis库，实现常用的redis命令如
AUTH、SELECT、GET、HKEYS、HGETALL、DEL、HDEL等。

关于在Windows下使用hiredis可以参考hiredis在windows下的编译以及使用，去microsoft/hiredis下载windows版本的hiredis，然后使用对应的Visual Studio比如VS2017编译对应的hiredis.lib库，再导入到项目中使用。

main.cpp

// RedisToTxtApp.cpp : 此文件包含 "main" 函数。程序执行将在此处开始并结束。
//

#include <iostream>
#include <type.h>
#include <mutex>
#include <func.h>
#include <unordered_map>

#include "RedisClient.h"
#include "ado/CustomADO.h"

// 查询站点映射表
StringMap g_stationMap;

// 从SQLServer数据库中查询UniqueCode->SStation的映射转换关系表
StringMap getStationMap(sig_log& lg)
{
	StringMap mapResult;

	// 连接SQLServer数据库
	CCustomADO ado(lg);
	ado.SetConnectInfo("192.19.0.9", "AirData", "sa", "123456");
	if (!ado.OpenConnect())
	{
		return mapResult;
	}

	char sql[10240] = { 0 };

	// 构造查询SQL脚本
	sprintf(sql, "select SStation, UniqueCode from t_AllSStation;");

	if (!ado.ExecuteProcess(sql, true))
	{
		return mapResult;
	}

	// 获取站点的所有站点的总个数
	long recordSum = ado.GetRecordsetCount();

	for (long i = 0; i < recordSum; ++i)
	{
		// 将查询到的站点信息保存到stbInfoItem中
		String strStation;
		String strUniqueCode;
		// String strZDStationCode;

		ado.GetFieldValue("SStation", strStation);
		ado.GetFieldValue("UniqueCode", strUniqueCode);
		// ado.GetFieldValue("ZDStationCode", strZDStationCode);

		mapResult.insert({ strUniqueCode, strStation });

		ado.MoveToNext();
	}

	return mapResult;
}


int main(int argc, char* argv[])
{
	std::mutex mutex_;

	sig_log log_;
	log_.connect([&mutex_](const char* file, const char* func, long line, log_level level, bool need_send, const char* msg) {
		std::lock_guard<std::mutex> locker(mutex_);
#ifdef _DEBUG
		printf("filelog===>[%d]%s:%ld %s: %s\n", level, file, line, func, msg);
#else
		char str[4096];
		sprintf(str, "[%s][%d] %s\n", Math::Date::getnow().c_str(), level, msg);
		printf("%s", str);
		if (Math::File::mk_dirs("./log"))
		{
			if (level != log_level::info)
			{
				String logfile = "./log/error_" + Math::Date::getnow("%04d_%02d") + ".log";
				FILE* f = fopen(logfile.c_str(), "a+");
				if (f)
				{
					fwrite(str, 1, strlen(str), f);
					fclose(f);
				}
			}
		}
#endif
	});

    // 1.从SQLServer数据库中获取站点映射关系表
	g_stationMap = getStationMap(log_);

	// 2. 从redis中获取某个站点的keys
	// keys env:hbxd:* 所有站点的站房数据
	// keys env:hbxd:421000401:* uniquecode编码为421000401的站点的站房数据
	RedisClient redis_(log_);
	redis_.set_connect_info("192.19.0.9", 7001, "123456", 1);
	redis_.connect();
	// 获取包含所有站点站房数据的所有key
	StringArray stationKeys;
	// redis_.KEYS(stationKeys, "env:hbxd:421000401:*");
	redis_.KEYS(stationKeys, "env:hbxd:*");

	// 获取env:hbxd:* 中所有的站点编码
	std::unordered_map<String, StringArray> strStcodeMap;
	for (auto &key1 : stationKeys)
	{
		StringArray strKeysArr = Math::Tools::split(key1, ":");
		String strStcode = strKeysArr[2];
		// 在strStcodeVec中查询是否已经存在该站点
		auto fi = strStcodeMap.find(strStcode);
		if (fi == strStcodeMap.end())  {
			strStcodeMap.insert(std::make_pair(strStcode, StringArray()));
		}
		strStcodeMap.at(strStcode).emplace_back(key1);
	}

	// 遍历所有站点，获取该站点env:hbxd:421000401:*下的所有keys，然后一个个站点生成txt监测数据
	for (auto &i : strStcodeMap)
	{
		auto& stcode = i.first;
		// 将站点编码转换成SStation即SS打头的站点编码格式
		auto iter = g_stationMap.find(stcode);
		if (iter == g_stationMap.end()){
			continue;
		}
		String strConvertStcode = iter->second;

		// 3.根据站点的key，获取对应的站点监测因子map数据
		String strFileName = strConvertStcode.empty() ? "tmp.txt" : strConvertStcode + ".txt";
		String strFolderPath = "./data";
		Math::File::mk_dirs(strFolderPath.c_str());

		String strFullPath = strFolderPath + "/" + strFileName;
		FILE *fp = fopen(strFullPath.c_str(), "w+");
		if (fp == NULL) {
			std::cout << "打开文件失败！";
			continue;
		}

		for (auto &key : i.second)
		{
			// std::cout << key << std::endl;
			// 4.从对应的key中找到对应的站点监测数据，然后再转换成txt文件，每个站点生成一个(20200512-20200518)
			StringMap monitorMap;
			redis_.HGETALL(monitorMap, key);
			// 从key中获取对应的站点编码，日期，时间
			StringArray strKeysArr = Math::Tools::split(key, ":");
			String strDate = strKeysArr[3];
			String strTime = strKeysArr[4];
			String strTimeTmp = strDate + strTime;
			String strDateTime = Math::Date::convertfmt(strTimeTmp.c_str(),
				"%04d%02d%02d%02d%02d%02d", "%04d-%02d-%02d %02d:%02d:%02d");
			// 获取输入字符串时间时间戳，默认格式 2018-10-01 01:10:20, %04d-%02d-%02d
			uint64 tt = Math::Date::getunix(strDateTime.c_str());
			// 遍历monitorMap，按照该站点421000401，将每个监测因子数据写入对应的421000401.txt文件中
			// 每行数据格式为：
			// SS4201051\tEP121\t1589526600\t2020-05-15 15:10:00\t62.0\tN\tM03\r\n
			String items;
			for (auto &jt : monitorMap)
			{
				// std::cout << "filed:" << jt.first << ",value:" << jt.second << "\n";
				// 解析得到的value(IA|10.01||EP117|)，从中获取对应的监测因子的EPID如EP117和值10.01，还有标记位和单位
				StringArray strArr = Math::Tools::split(jt.second, "|");
				// 监测数值
				double dVal;
				Math::Tools::to_float(dVal, strArr[1]);
				// 监测标记位
				String strMark = strArr[2].empty() ? "N" : strArr[2];
				// 监测因子 EPId
				String strParamId = strArr[3];
				// 监测单位
				String strUnit = strArr[4].empty() ? " " : strArr[3];

				char strLineBuf[1024] = { 0 };
				sprintf(strLineBuf, "%s\t%s\t%lld\t%s\t%lf\t%s\t%s\r\n",
					strConvertStcode.c_str(), strParamId.c_str(), tt,
					strDateTime.c_str(), dVal, strMark.c_str(), strUnit.c_str());
				items += strLineBuf;
			}
			if (fp && items.size() && fwrite(items.data(), sizeof(char), items.size(), fp) > 0) {
				if (!redis_.DEL(key)) {
					printf("%s删除失败", key.c_str());
				}
			}
		}
		if (fp) {
			fclose(fp);
		}
		std::cout << "站点文件" << strFileName << "生成完毕！" << std::endl;
	}
	
	return 0;
}

参考资料

使用格式化文件批量导入数据 (SQL Server)

非 XML 格式化文件 (SQL Server)

bcp 实用工具

大容量导入和导出数据 (SQL Server)

格式化文件以导入或导出数据 (SQL Server)

批量导入和导出 XML 文档的示例 (SQL Server)

猜你喜欢

转载自blog.csdn.net/ccf19881030/article/details/106209198
今日推荐
NetBSD 禁止提交由 AI 生成的代码
Apache Doris 2.0.10 版本正式发布！
开源日报 | 大模型开战；大模型独角兽被曝卖身；周鸿祎建议谷歌开源所有产品；最大开源AI社区提供1000万美元共享GPU
开源日报 | Chrome内置Gemini的意义不在于Gemini；中国AI追随之路的五大误区；ECharts创始人“下海”养鱼；谷歌I/O开发者大会什么都有，只是没有惊喜
微软回应中国区AI团队“打包赴美”传闻
基于大语言模型的开源知识库问答系统 MaxKB GitHub Star 数量突破 5,000 个！
周排行
static方法和非static方法的区别（java）
如何查找计算机专业paper
java.lang.ClassFormatError: Incompatible magic value 0 in class file com/sitecha
跳跃游戏II
stm32_之【建立工程】
TeaWeb v0.0.9 发布，统计底层优化、主机监控功能改进
事件分发 -----控制字体大小
JavaScript DOM练习（动态表格添加） December 25，2019
JSF Scope & CDI
实现从零搭建一个登录注册页面（附源代码）
每日归档
更多
2024-05-19(0)
2024-05-18(4)
2024-05-17(34)
2024-05-16(6)
2024-05-15(24)
2024-05-14(0)
2024-05-13(18)
2024-05-12(0)
2024-05-11(38)
2024-05-10(38)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022