1 tsquery
esqueryのTypeScriptバージョンであるtsqueryライブラリを見つけて導入しました。これにより、css セレクターを使用して、指定された条件を満たす TypeScript ast ノードをすばやくクエリできます (JavaScript もサポートされています)。
目的、すべてのクエリ データを返す:
class TsQueryApi:
def __init__(self,
wid_org_id: str, # 位置编号
metric_api: RpcMetricApiStub, # 频次
ts_api: RpcTimeseriesApiStub): # 时间
self._org_id = wid_org_id
self._RpcMetricApiStub = metric_api
self._RpcTimeseriesApiStub = ts_api
def metric_query(self,
metric_name: str = None,
gateway_tag_name: str = None,
business_types: list[str] = [],
metric_table_ids: list[str] = [],
metric_source_types: list[str] = [],
metric_types: list[str] = [],
metric_table_types: list[str] = [],
gateway_tag_names: list[str] = [],
metric_ids: list[str] = []) -> list[dict]:
"""
查询指标信息，如果所有参数都留空，那么回返回系统中的所有的指标信息
:param metric_name: 指标名称，支持模糊查询
:param gateway_tag_name: 模糊查询，网关上传的点位tagId，用于是网关出，指标的唯一标识
:param business_types: 指标的业务类型
:param metric_table_ids: 指标所属的站点
:param metric_source_types: 指标的数据源类型,raw为原始数据，processed为虚拟指标
:param metric_types: 信号类型, DI(数字输入)、DO(数字输出)、AI（模拟输入）、AO（模拟输出）、DA（报警）、T（阈值）
:param metric_table_types: 通过指标所属站点的类型，来查询所属指标类型
:param gateway_tag_names: 指标在网关上指定的tagId
:param metric_ids: 指标id
:return: 指标信息的dict
"""
query_dto = {
'metricName': metric_name, # 指标名称
'gatewayTagName': gateway_tag_name, # 模糊查询
'businessTypes': business_types, # 指标的业务类型
'metricTableIds': metric_table_ids, # 指标所属的站点
'metricSourceTypes': metric_source_types, # 指标的数据源类型,raw为原始数据，processed为虚拟指标
'metricTypes': metric_types, # 信号类型
'metricTableTypes': metric_table_types, # 通过指标所属站点的类型，来查询所属指标类型
'gatewayTagNames': gateway_tag_names, # 指标在网关上指定的tagId
'metricIds': metric_ids # 指标id
}
request = RpcMetricInfoQuery(org_id=self._org_id, query_dto_json=json.dumps(query_dto, ensure_ascii=False))
resp_json = self._RpcMetricApiStub.MetricQuery(request).metric_info_list_json
return json.loads(resp_json)
def query_ts_data(self,
from_time: datetime,
to_time: datetime = datetime.now(),
timeseries_name: list[str] = [],
down_sampling: RpcDownSampling = None,
result_tz: Union[str, pytz.timezone, dateutil.tz.tzfile] = None,
tags: dict[str, str] = None) -> Iterable[TsResp]:
"""
:param from_time: 开始时间
:param to_time: 结束时间
:param timeseries_name: 时间序列名，注意这里不是tag_id
:param down_sampling: 降采样参数
:param result_tz: 结果的时区信息，不设置默认为utc时间
:param tags:
:return: 返回查询的数据，会yield返回
"""
from_time = handle_datetime_without_tz(from_time)
to_time = handle_datetime_without_tz(to_time)
request = QueryTsDataRequest(org_id=self._org_id,
from_time=from_time.isoformat(),
to_time=to_time.isoformat(),
timeseries_name=timeseries_name,
down_sampling=down_sampling,
tags=tags)
for resp in self._RpcTimeseriesApiStub.QueryTsData(request):
length = len(resp.points)
# time = numpy.zeros(shape=[length], dtype='datetime64[ms]')
time = []
value = numpy.empty(shape=[length])
for i in range(length):
p = resp.points[i]
# time[i] = datetime.utcfromtimestamp(p.time)
# dt = datetime.fromtimestamp(p.time)
# time[i] = numpy.datetime64(p.time, "ms")
t = pd.Timestamp(p.time, unit='ms', tz=result_tz)
time.append(t)
if p.WhichOneof("valueOrNull") == "value":
value[i] = p.value
else:
value[i] = numpy.NaN
yield TsResp(timeseries_name=resp.timeseries_name,
data=pd.Series(data=value, index=time))
def query_dict_bind(self, bind_dict: dict,
from_time: datetime,
to_time: datetime = datetime.now(),
down_sampling: RpcDownSampling = None,
result_tz: Union[str, pytz.timezone, dateutil.tz.tzfile] = None) -> dict:
"""
查询通过dict来绑定并查询的业务，dict的key为scada的tag_id,该方法会自动将metric_info和ts_data绑定到对于的key上
例如输入一个dict:
{
"AI_01_IN_FLOW": {}
}
最终结果是:
{
"AI_01_IN_FLOW": {
"metric_info": dict
"ts_data": pandas.Series
}
"AI_02_FLOW1": {
"metric_info": dict
"ts_data": pandas.Series
}
}
:return: 包含metric_info和ts_data的dict
"""
result_dict = bind_dict.copy()
tag_ids = []
for key, value in bind_dict.items():
tag_ids.append(key)
# 如果不是dict，改为一个dict
if not isinstance(result_dict[key], dict):
result_dict[key] = {}
metric_info_result = self.metric_query(gateway_tag_names=tag_ids)
timeseries_names = []
# 绑定scada数据
for metric_info in metric_info_result:
tag_id = metric_info['gatewayTagName']
result_dict[tag_id]['metric_info'] = metric_info
timeseries_names.append(metric_info['timeSerialName'])
# 查询时序数据
ts_data = self.query_ts_data(from_time=from_time, to_time=to_time, timeseries_name=timeseries_names,
down_sampling=down_sampling, result_tz=result_tz)
for t in ts_data:
resp: TsResp = t
for tag_id, item in result_dict.items():
if 'metric_info' in item:
if resp.timeseries_name == item['metric_info']['timeSerialName']:
item['ts_data'] = resp.data
return result_dict
def query_scada_bind_file(self, scada_json_file: str,
from_time: datetime,
to_time: datetime = datetime.now(),
down_sampling: RpcDownSampling = None,
result_tz: Union[str, pytz.timezone, dateutil.tz.tzfile] = None) -> dict:
"""
:param result_tz: 结果的时区信息，不设置默认为utc时间
"""
scada_json_dict = json.loads(scada_json_file)
param_dict = {
'node': scada_json_dict['node'],
'link': scada_json_dict['link'],
'subcatchment': scada_json_dict['subcatchment']
}
# 先查询指标信息 metric_info
parms = [] # 去掉外层的绑定信息
misc.loop_into_swmm_scada_dict(param_dict,
lambda obj_id, swmm_obj_bind_param: parms.append(swmm_obj_bind_param))
metric_info_result = self.metric_query(gateway_tag_names=list(map(lambda p: p['tag_ID'], parms)))
misc.loop_into_swmm_scada_dict(param_dict,
lambda obj_id, swmm_obj_bind_param: self.__metric_to_dict(obj_id,
swmm_obj_bind_param,
metric_info_result))
for key, value in param_dict.items():
scada_json_dict[key] = value
# 查询时序数据
timeseries_names = list(map(lambda m: m['timeSerialName'], metric_info_result))
ts_data = self.query_ts_data(from_time=from_time, to_time=to_time, timeseries_name=timeseries_names,
down_sampling=down_sampling, result_tz=result_tz)
misc.loop_into_swmm_scada_dict(param_dict,
lambda obj_id, swmm_obj_bind_param: self.__ts_data_to_dict(obj_id,
swmm_obj_bind_param,
list(ts_data)))
return scada_json_dict
def __metric_to_dict(self, obj_id: str, swmm_obj_bind_param: dict,
metric_info_list: list[dict]):
for metric_info in metric_info_list:
if metric_info['gatewayTagName'] == swmm_obj_bind_param['tag_ID']:
swmm_obj_bind_param['metric_info'] = metric_info
def __ts_data_to_dict(self, obj_id: str, swmm_obj_bind_param: dict,
ts_data: list[TsResp]):
for ts_resp in ts_data:
if 'metric_info' in swmm_obj_bind_param:
if ts_resp.timeseries_name == swmm_obj_bind_param['metric_info']['timeSerialName']:
swmm_obj_bind_param['ts_data'] = ts_resp.data2 SwmmApi
class SwmmApi:
def __init__(self,
org_id: str,
stub: RpcWmSwmmApiStub):
self.__stub = stub # 编号
self.org_id = org_id # 服务器
def upload_swmm_processing_status(self,
model_id: str,
percent_complete: float):
"""
上传模型计算状态
:param model_id: 模型id
:param percent_complete: 运行的百分比
:return:
"""
request = RpcSwmmProcessingStatus(model_id=model_id, org_id=self.org_id, percent_complete=percent_complete)
return self.__stub.UploadSwmmProcessingStatus(request)
def query_wm_config(self, model_id: str) -> dict[str, str]:
"""
查询对应模型的配置信息
:param model_id: 模型id
:return: 返回模型的配置文件信息，key为config_name，value为config文件内容
"""
request = RpcQueryWmConfigRequest(model_id=model_id, org_id=self.org_id)
resp = self.__stub.QueryWmConfig(request)
config_dict = {}
for config in resp.configs:
config_dict[config.config_name] = config.content
return config_dict
def query_wm_swmm_model_inp_file(self, model_id: str, target_file_path: str):
"""
将模型inp文件写入到指定路径
:param model_id: 模型id
:param target_file_path: 路径地址
:return:
"""
request = QueryWmSwmmModelInpFileRequest(model_id=model_id, org_id=self.org_id)
resp = self.__stub.QueryWmSwmmModelInpFile(request)
with open(target_file_path, "wb") as f:
f.write(resp.inp_file_bytes)
def upload_swmm_result(self,
model_id: str,
sim_start_time: datetime = None,
sim_end_time: datetime = None,
task_start_time: datetime = None,
task_end_time: datetime = None,
back_cal_time: timedelta = None,
save_as_forecast: bool = True,
out_file_name: str = None,
rpt_file_name: str = None):
"""
上传swmm运算结果
:param model_id: 模型id
:param sim_start_time: 模型模拟的开始时间
:param sim_end_time: 模型模拟的结束时间
:param task_start_time: 计算任务开始时间，用于记录
:param task_end_time: 计算任务结束时间，用于记录
:param back_cal_time: 回算时间，如果为空，数据不会保存到时序数据库。 例如datetime.timedelta(hours=10)
:param save_as_forecast: 是否应该保存为预测数据，否则只当作回算数据
:param out_file_name: .out文件的路径
:param rpt_file_name: .rpt文件的路径
:return:
"""
sim_start_time = handle_datetime_without_tz(sim_start_time)
sim_end_time = handle_datetime_without_tz(sim_end_time)
task_start_time = handle_datetime_without_tz(task_start_time)
task_end_time = handle_datetime_without_tz(task_end_time)
result = RpcUploadSwmmResultRequest(model_id=model_id, org_id=self.org_id)
if sim_start_time is not None:
result.sim_start_time = sim_start_time.isoformat()
if sim_end_time is not None:
result.sim_end_time = sim_end_time.isoformat()
if task_start_time is not None:
result.task_start_time = task_start_time.isoformat()
if task_end_time is not None:
result.task_end_time = task_end_time.isoformat()
# 处理out文件
if (out_file_name is not None) or (out_file_name != ''):
self.__handle_out_file(back_cal_time=back_cal_time, save_as_forecast=save_as_forecast,
out_file_name=out_file_name, result=result)
# 上传rpt文件
if (rpt_file_name is not None) or (out_file_name != ''):
result.rpt_rpt_file = open(rpt_file_name, 'rb').read()
return self.__stub.UploadSwmmResult(result)
# 处理上传计算结果过程中的.out文件
@staticmethod
def __handle_out_file(back_cal_time: timedelta, save_as_forecast: bool, out_file_name, result):
# 处理.out
out = read_out_file(out_file_name)
out_frame = out.to_frame()
# 保存回算数据
if back_cal_time is not None:
result.save_to_tsdb = True
# 把需要保存到时序数据库的数据选出
back_cal_out_frame = out_frame.loc[:(out_frame.index[0] + back_cal_time)]
for column in back_cal_out_frame:
series = result.out_series.add()
series.type = str(column[0])
series.obj_id = str(column[1])
series.field = str(column[2]) # 这里拿到pandas的series
for index, value in back_cal_out_frame[column].items():
p = series.points.add()
p.time = int(index.timestamp() * 1000) # 时间，毫秒
p.value = value # 值
if save_as_forecast:
# 保存.out的csv格式文件
with tempfile.NamedTemporaryFile(mode='w+b') as temp_out, \
tempfile.NamedTemporaryFile() as temp_zip:
# 从multiIndex替换为字符串
out_frame.columns = out_frame.columns.map(lambda cl: f'{cl[0]}/{cl[1]}/{cl[2]}')
out_frame.to_csv(temp_out)
temp_out.seek(0) # 没有这一步，文件无法读取
result.out_csv_file = temp_out.read()3 ファイル API
class FileApi:
def __init__(self,
org_id: str,
file_stub: RpcFileApiStub):
self._file_stub = file_stub # 文件
self.org_id = org_id # 编号
def get_and_write_file(self, path: str, target: Union[str, IO, IOBase]):
"""
从中台获取文件，并写入到对应文件或IO中
:param path: 中台返回的文件路径
:param target: 如果为str，则当作路径写入到对应位置。如果是IO对象（例如使用open打开的file对象），会写入到对应的对象中
"""
request = RpcGetFileRequest(path=path)
if isinstance(target, str):
with open(target, "wb") as file:
for resp in self._file_stub.GetFile(request):
c: RpcGetFileResponse = resp
file.write(c.content)
elif isinstance(target, IO):
for resp in self._file_stub.GetFile(request):
c: RpcGetFileResponse = resp
target.write(c.content)
elif isinstance(target, IOBase):
for resp in self._file_stub.GetFile(request):
c: RpcGetFileResponse = resp
target.write(c.content)
else:
raise Exception('unsupported target type:' + type(target).__name__)
@staticmethod
def __write_to_io(target: IO, c: RpcGetFileResponse):
target.write(c.content)