Flink作业详情获取

Flink Dashboard

基于Flink Dashboard自带的Web UI能直接查询

数据写入条数和数据大小基于Source Reader的Records Sent和Bytes Sent获取

数据读取条数和数据大小基于Sink Writer的Records Recevied 和 Bytes Recevied获取

Flink已完成/已结束任务

需要flink-conf.yaml上需要配置historyserver.archive.fs.dir和jobmanager.archive.fs.dir

如果需要详情页链接，需要配置historyserver.web相关

jobmanager.archive.fs.dir: hdfs://XXX/flink-history/
historyserver.archive.fs.dir: hdfs://XXX/flink-history/
historyserver.web.address: 0.0.0.0
historyserver.web.port: 8082
historyserver.archive.fs.refresh-interval: 10000

如果启动了Flink的HistoryServer，可以通过HistoryServer的REST API获取具体的作业详情和数据，具体URLs如下

在flink使用了jobmanager的归档功能，则通过hdfs获取任务结束的状态和作业的指标数据，即Flink HistoryServer上展示的数据

分析json文件中的/jobs/{jobId}路径下的作用运行详细

样例：

{
    "jid": "295a03a068984f836581bdd64bb55904",
    "name": "flinkcdc_job",
    "isStoppable": false,
    "state": "CANCELED",
    "start-time": 1720767118813,
    "end-time": 1721029766416,
    "duration": 262647603,
    "maxParallelism": -1,
    "now": 1721029766680,
    "timestamps": {
        "CANCELED": 1721029766416,
        "RUNNING": 1720767119311,
        "FINISHED": 0,
        "SUSPENDED": 0,
        "FAILING": 0,
        "FAILED": 0,
        "CANCELLING": 1721029765826,
        "RESTARTING": 0,
        "RECONCILING": 0,
        "INITIALIZING": 1720767118813,
        "CREATED": 1720767118906
    },
    "vertices": [
        {
            "id": "bc764cd8ddf7a0cff126f51c16239658",
            "name": "Source: Flink CDC Event Source: mysql",
            "maxParallelism": 128,
            "parallelism": 1,
            "status": "CANCELED",
            "start-time": 1720767138357,
            "end-time": 1721029766412,
            "duration": 262628055,
            "tasks": {
                "CREATED": 0,
                "CANCELING": 0,
                "DEPLOYING": 0,
                "FINISHED": 0,
                "INITIALIZING": 0,
                "SCHEDULED": 0,
                "RUNNING": 0,
                "CANCELED": 1,
                "FAILED": 0,
                "RECONCILING": 0
            },
            "metrics": {
                "read-bytes": 0,
                "read-bytes-complete": true,
                "write-bytes": 6744,
                "write-bytes-complete": true,
                "read-records": 0,
                "read-records-complete": true,
                "write-records": 22,
                "write-records-complete": true,
                "accumulated-backpressured-time": 0,
                "accumulated-idle-time": 266841686,
                "accumulated-busy-time": 0
            }
        },
        {
            "id": "9899a42c64d67ef3172b7e3be3c1bbb9",
            "name": "SchemaOperator",
            "maxParallelism": 128,
            "parallelism": 1,
            "status": "CANCELED",
            "start-time": 1720767138363,
            "end-time": 1721029765928,
            "duration": 262627565,
            "tasks": {
                "CREATED": 0,
                "CANCELING": 0,
                "DEPLOYING": 0,
                "FINISHED": 0,
                "INITIALIZING": 0,
                "SCHEDULED": 0,
                "RUNNING": 0,
                "CANCELED": 1,
                "FAILED": 0,
                "RECONCILING": 0
            },
            "metrics": {
                "read-bytes": 1004704,
                "read-bytes-complete": true,
                "write-bytes": 6800,
                "write-bytes-complete": true,
                "read-records": 22,
                "read-records-complete": true,
                "write-records": 24,
                "write-records-complete": true,
                "accumulated-backpressured-time": 0,
                "accumulated-idle-time": 288212995,
                "accumulated-busy-time": 0
            }
        },
        {
            "id": "78be0dd8677bc2711e2a56947a5ea048",
            "name": "PrePartition",
            "maxParallelism": 128,
            "parallelism": 1,
            "status": "CANCELED",
            "start-time": 1720767138373,
            "end-time": 1721029765958,
            "duration": 262627585,
            "tasks": {
                "CREATED": 0,
                "CANCELING": 0,
                "DEPLOYING": 0,
                "FINISHED": 0,
                "INITIALIZING": 0,
                "SCHEDULED": 0,
                "RUNNING": 0,
                "CANCELED": 1,
                "FAILED": 0,
                "RECONCILING": 0
            },
            "metrics": {
                "read-bytes": 1004760,
                "read-bytes-complete": true,
                "write-bytes": 6736,
                "write-bytes-complete": true,
                "read-records": 24,
                "read-records-complete": true,
                "write-records": 24,
                "write-records-complete": true,
                "accumulated-backpressured-time": 0,
                "accumulated-idle-time": 288219639,
                "accumulated-busy-time": 0
            }
        },
        {
            "id": "fffb571f3c6daa1956cc08d8e61e216a",
            "name": "PostPartition",
            "maxParallelism": 128,
            "parallelism": 1,
            "status": "CANCELED",
            "start-time": 1720767138374,
            "end-time": 1721029765958,
            "duration": 262627584,
            "tasks": {
                "CREATED": 0,
                "CANCELING": 0,
                "DEPLOYING": 0,
                "FINISHED": 0,
                "INITIALIZING": 0,
                "SCHEDULED": 0,
                "RUNNING": 0,
                "CANCELED": 1,
                "FAILED": 0,
                "RECONCILING": 0
            },
            "metrics": {
                "read-bytes": 1004696,
                "read-bytes-complete": true,
                "write-bytes": 6640,
                "write-bytes-complete": true,
                "read-records": 24,
                "read-records-complete": true,
                "write-records": 24,
                "write-records-complete": true,
                "accumulated-backpressured-time": 0,
                "accumulated-idle-time": 288234470,
                "accumulated-busy-time": 0
            }
        },
        {
            "id": "4f68b812c5c08515a44082f028175e7d",
            "name": "Sink Writer: Flink CDC Event Sink: doris",
            "maxParallelism": 128,
            "parallelism": 1,
            "status": "CANCELED",
            "start-time": 1720767138377,
            "end-time": 1721029765957,
            "duration": 262627580,
            "tasks": {
                "CREATED": 0,
                "CANCELING": 0,
                "DEPLOYING": 0,
                "FINISHED": 0,
                "INITIALIZING": 0,
                "SCHEDULED": 0,
                "RUNNING": 0,
                "CANCELED": 1,
                "FAILED": 0,
                "RECONCILING": 0
            },
            "metrics": {
                "read-bytes": 1004600,
                "read-bytes-complete": true,
                "write-bytes": 0,
                "write-bytes-complete": true,
                "read-records": 24,
                "read-records-complete": true,
                "write-records": 0,
                "write-records-complete": true,
                "accumulated-backpressured-time": 0,
                "accumulated-idle-time": 288208909,
                "accumulated-busy-time": 0
            }
        }
    ],
    "status-counts": {
        "CREATED": 0,
        "CANCELING": 0,
        "DEPLOYING": 0,
        "FINISHED": 0,
        "INITIALIZING": 0,
        "SCHEDULED": 0,
        "RUNNING": 0,
        "CANCELED": 5,
        "FAILED": 0,
        "RECONCILING": 0
    },
    "plan": {
        "jid": "295a03a068984f836581bdd64bb55904",
        "name": "flinkcdc_job",
        "type": "STREAMING",
        "nodes": [
            {
                "id": "4f68b812c5c08515a44082f028175e7d",
                "parallelism": 1,
                "operator": "",
                "operator_strategy": "",
                "description": "Sink Writer: Flink CDC Event Sink: doris<br/>",
                "inputs": [
                    {
                        "num": 0,
                        "id": "fffb571f3c6daa1956cc08d8e61e216a",
                        "ship_strategy": "FORWARD",
                        "exchange": "pipelined_bounded"
                    }
                ],
                "optimizer_properties": {

                }
            },
            {
                "id": "fffb571f3c6daa1956cc08d8e61e216a",
                "parallelism": 1,
                "operator": "",
                "operator_strategy": "",
                "description": "PostPartition<br/>",
                "inputs": [
                    {
                        "num": 0,
                        "id": "78be0dd8677bc2711e2a56947a5ea048",
                        "ship_strategy": "CUSTOM",
                        "exchange": "pipelined_bounded"
                    }
                ],
                "optimizer_properties": {

                }
            },
            {
                "id": "78be0dd8677bc2711e2a56947a5ea048",
                "parallelism": 1,
                "operator": "",
                "operator_strategy": "",
                "description": "PrePartition<br/>",
                "inputs": [
                    {
                        "num": 0,
                        "id": "9899a42c64d67ef3172b7e3be3c1bbb9",
                        "ship_strategy": "FORWARD",
                        "exchange": "pipelined_bounded"
                    }
                ],
                "optimizer_properties": {

                }
            },
            {
                "id": "9899a42c64d67ef3172b7e3be3c1bbb9",
                "parallelism": 1,
                "operator": "",
                "operator_strategy": "",
                "description": "SchemaOperator<br/>",
                "inputs": [
                    {
                        "num": 0,
                        "id": "bc764cd8ddf7a0cff126f51c16239658",
                        "ship_strategy": "FORWARD",
                        "exchange": "pipelined_bounded"
                    }
                ],
                "optimizer_properties": {

                }
            },
            {
                "id": "bc764cd8ddf7a0cff126f51c16239658",
                "parallelism": 1,
                "operator": "",
                "operator_strategy": "",
                "description": "Source: Flink CDC Event Source: mysql<br/>",
                "optimizer_properties": {

                }
            }
        ]
    }
}