supervisor-spec.json如下:
{
"type": "kafka",
"dataSchema": {
"dataSource": "metrics-kafka",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp",
"format": "auto" },
"dimensionsSpec": {
"dimensions": [],
"dimensionExclusions": [ "timestamp", "value" ] }
}
},
"metricsSpec": [
{
"name": "count",
"type": "count"
},
{
"name": "value_sum",
"fieldName": "value",
"type": "doubleSum"
},
{
"name": "value_min",
"fieldName": "value",
"type": "doubleMin"
},
{
"name": "value_max",
"fieldName": "value",
"type": "doubleMax"
}
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "HOUR",
"queryGranularity": "NONE"
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 5000000
},
"ioConfig": {
"topic": "metrics",
"consumerProperties": {
"bootstrap.servers": "localhost:9092"
},
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H"
}
}
创建Kafka-index-service任务
curl -X POST -H 'Content-Type: application/json' -d @supervisor-spec.json http://192.168.165.130:8090/druid/indexer/v1/supervisor
返回结果:
{"id":"metrics-kafka"}
获取supervisor的描述文件:
curl -X GET http://192.168.165.130:18090/druid/indexer/v1/supervisor/metrics-kafka
返回结果集:
{"id":"metrics-kafka"}[root@emr-header-1 kafka-index-service]# curl -X GET -H 'Content-Type: application/json' -d @supervisor-spec1.json http://192.168.165.130:18090/druid/indexer/v1/supervisor/metrics-kafka
{"type":"kafka","dataSchema":{"dataSource":"metrics-kafka","parser":{"type":"string","parseSpec":{"format":"json","timestampSpec":{"column":"timestamp","format":"auto"},"dimensionsSpec":{"dimensions":[],"dimensionExclusions":["timestamp","value"]}}},"metricsSpec":[{"type":"count","name":"count"},{"type":"doubleSum","name":"value_sum","fieldName":"value","expression":null},{"type":"doubleMin","name":"value_min","fieldName":"value","expression":null},{"type":"doubleMax","name":"value_max","fieldName":"value","expression":null}],"granularitySpec":{"type":"uniform","segmentGranularity":"HOUR","queryGranularity":{"type":"none"},"rollup":true,"intervals":null}},"tuningConfig":{"type":"kafka","maxRowsInMemory":75000,"maxRowsPerSegment":5000000,"intermediatePersistPeriod":"PT10M","basePersistDirectory":"/opt/apps/ecm/service/druid/0.11.0-1.2.0/package/druid-0.11.0-1.2.0/var/tmp/1534915930015-0","maxPendingPersists":0,"indexSpec":{"bitmap":{"type":"concise"},"dimensionCompression":"lz4","metricCompression":"lz4","longEncoding":"longs"},"buildV9Directly":true,"reportParseExceptions":false,"handoffConditionTimeout":0,"resetOffsetAutomatically":false,"workerThreads":null,"chatThreads":null,"chatRetries":8,"httpTimeout":"PT10S","shutdownTimeout":"PT80S","offsetFetchPeriod":"PT30S"},"ioConfig":{"topic":"metrics","replicas":1,"taskCount":1,"taskDuration":"PT3600S","consumerProperties":{"bootstrap.servers":"localhost:9092"},"startDelay":"PT5S","period":"PT30S","useEarliestOffset":false,"completionTimeout":"PT1800S","lateMessageRejectionPeriod":null,"earlyMessageRejectionPeriod":null,"skipOffsetGaps":false},"context":null}
获取kafka-index-service任务状态:
curl -X GET http://192.168.165.130:18090/druid/indexer/v1/supervisor/metrics-kafka/status
返回结果:
{"id":"metrics-kafka","generationTime":"2018-08-22T05:33:59.253Z","payload":{"dataSource":"metrics-kafka","topic":"metrics","partitions":0,"replicas":1,"durationSeconds":3600,"activeTasks":[],"publishingTasks":[],"minimumLag":{},"aggregateLag":0,"offsetsLastUpdated":null}}
查询kafaka-index-service任务的历史记录:
curl -X GET http://192.168.165.130:18090/druid/indexer/v1/supervisor/metrics-kafka/history
返回结果:
[{"spec":{"type":"kafka","dataSchema":{"dataSource":"metrics-kafka","parser":{"type":"string","parseSpec":{"format":"json","timestampSpec":{"column":"timestamp","format":"auto"},"dimensionsSpec":{"dimensions":[],"dimensionExclusions":["timestamp","value"]}}},"metricsSpec":[{"type":"count","name":"count"},{"type":"doubleSum","name":"value_sum","fieldName":"value","expression":null},{"type":"doubleMin","name":"value_min","fieldName":"value","expression":null},{"type":"doubleMax","name":"value_max","fieldName":"value","expression":null}],"granularitySpec":{"type":"uniform","segmentGranularity":"HOUR","queryGranularity":{"type":"none"},"rollup":true,"intervals":null}},"tuningConfig":{"type":"kafka","maxRowsInMemory":75000,"maxRowsPerSegment":5000000,"intermediatePersistPeriod":"PT10M","basePersistDirectory":"/opt/apps/ecm/service/druid/0.11.0-1.2.0/package/druid-0.11.0-1.2.0/var/tmp/1534915930015-0","maxPendingPersists":0,"indexSpec":{"bitmap":{"type":"concise"},"dimensionCompression":"lz4","metricCompression":"lz4","longEncoding":"longs"},"buildV9Directly":true,"reportParseExceptions":false,"handoffConditionTimeout":0,"resetOffsetAutomatically":false,"workerThreads":null,"chatThreads":null,"chatRetries":8,"httpTimeout":"PT10S","shutdownTimeout":"PT80S","offsetFetchPeriod":"PT30S"},"ioConfig":{"topic":"metrics","replicas":1,"taskCount":1,"taskDuration":"PT3600S","consumerProperties":{"bootstrap.servers":"localhost:9092"},"startDelay":"PT5S","period":"PT30S","useEarliestOffset":false,"completionTimeout":"PT1800S","lateMessageRejectionPeriod":null,"earlyMessageRejectionPeriod":null,"skipOffsetGaps":false},"context":null},"version":"2018-08-22T05:32:10.018Z"}]