指标
kubevirt指标
vmiCreationToRunningSecondsP50
vmiCreationToRunningSecondsP90
vmiCreationToRunningSecondsP99
k8s api server指标
apiserver_request_total
kubelet指标
pod资源使用情况指标
container_cpu_usage_seconds_total
container_memory_rss
场景
测试方案
测试批量创建虚机、运行虚机、删除虚机，指标变化情况
测试规模
- <input disabled="" type="checkbox" /> 100 VM
- <input disabled="" type="checkbox" /> 200 VM
- <input disabled="" type="checkbox" /> 1000 VM
- <input disabled="" type="checkbox" /> 4000 VM
准备
结果获取百分位
50 90 99
| 镜像 | 存储 | CPU | 内存 |
|---|---|---|---|
| cirros | 50M | 0.1(100m) | 231(60Mi+171Mi) |
测试工具：
perfscale-load-generator
收集结果工具：
perfscale-audit
高负载集群可能的存在问题
etcd cpu占用高
api-server内存高
vm开关机timeout问题
测试的集群配置
测试方法
perfscale-load-generator工具
- 拉取源码，执行
make bazel-build构建perfscale-load-generator测试工具
- 获取执行性能测试的配置文件，包括kubeconfig、workloadconfig等，workloadconfig修改成我们需要配置
- 修改并确认配置正确后执行性能测试
perfscale-audit工具
使用perfscale-audit工具审计kubevirt控制面指标
已知kubevirt指标通过virt-handler提供metrics接口获取，每个边缘集群都已接入prometheus，查看servicemonitor可知指标已上报边缘集群prometheus
- 拉取源码，执行
make bazel-build构建perfscale-audit审计工具 - 配置perf-audit-results.json审计的输入信息，包括Prometheus地址，审计时间段等
- 执行监控指标审计工作
- 分析审计指标
{
"Values": {
// 对象CREATE/GET/DELETE/LIST/PATCH请求计数
"CREATE-controllerrevisions-count": {
"value": 1.0003473428273706
},
"CREATE-events-count": {
"value": 828.3349850933264
},
"CREATE-pods-count": {
"value": 213.07398402222992
},
"CREATE-virtualmachineinstances-count": {
"value": 1.0003473428273706
},
"DELETE-controllerrevisions-count": {
"value": 1.0003473428273706
},
"DELETE-pods-count": {
"value": 213.07398402222992
},
"DELETE-virtualmachineinstances-count": {
"value": 1.0003473428273706
},
"GET-configmaps-count": {
"value": 144.05001736714138
},
"GET-endpoints-count": {
"value": 139856.561306009
},
"GET-network-attachment-definitions-count": {
"value": 1.0003473428273706
},
"GET-nodes-count": {
"value": 8416.922542549497
},
"GET-virtualmachineinstances-count": {
"value": 1.0003473428273706
},
"GET-virtualmachines-count": {
"value": 1.0003473428273706
},
"LIST-cdiconfigs-count": {
"value": 196.06807919416462
},
"LIST-cdis-count": {
"value": 192.06668982285515
},
"LIST-clusterrolebindings-count": {
"value": 388.13476901701983
},
"LIST-clusterroles-count": {
"value": 389.1351163598472
},
"LIST-configmaps-count": {
"value": 3277.1378951024662
},
"LIST-controllerrevisions-count": {
"value": 193.06703716568254
},
"LIST-customresourcedefinitions-count": {
"value": 2895.00521014241
},
"LIST-daemonsets-count": {
"value": 385.13372698853766
},
"LIST-datasources-count": {
"value": 390.13546370267454
},
"LIST-datavolumes-count": {
"value": 193.06703716568254
},
"LIST-deployments-count": {
"value": 387.1344216741924
},
"LIST-jobs-count": {
"value": 382.1326849600556
},
"LIST-kubevirts-count": {
"value": 2506.8704411253902
},
"LIST-limitranges-count": {
"value": 388.1347690170198
},
"LIST-migrationpolicies-count": {
"value": 192.06668982285515
},
"LIST-mutatingwebhookconfigurations-count": {
"value": 383.13303230288295
},
"LIST-namespaces-count": {
"value": 381.1323376172282
},
"LIST-nodes-count": {
"value": 195.06773185133727
},
"LIST-persistentvolumeclaims-count": {
"value": 197.068426536992
},
"LIST-poddisruptionbudgets-count": {
"value": 570.1979854116012
},
"LIST-pods-count": {
"value": 768.2667592914206
},
"LIST-prometheusrules-count": {
"value": 387.1344216741924
},
"LIST-rolebindings-count": {
"value": 380.13199027440083
},
"LIST-roles-count": {
"value": 387.1344216741924
},
"LIST-secrets-count": {
"value": 391.1358110455019
},
"LIST-serviceaccounts-count": {
"value": 392.13615838832925
},
"LIST-servicemonitors-count": {
"value": 385.13372698853766
},
"LIST-services-count": {
"value": 381.1323376172282
},
"LIST-storageclasses-count": {
"value": 190.06599513720042
},
"LIST-validatingwebhookconfigurations-count": {
"value": 379.1316429315734
},
"LIST-virtualmachineclusterflavors-count": {
"value": 575.1997221257382
},
"LIST-virtualmachineflavors-count": {
"value": 581.2018061827023
},
"LIST-virtualmachineinstancemigrations-count": {
"value": 193.06703716568254
},
"LIST-virtualmachineinstancepresets-count": {
"value": 382.13268496005554
},
"LIST-virtualmachineinstancereplicasets-count": {
"value": 191.06634248002777
},
"LIST-virtualmachineinstances-count": {
"value": 3275.137200416811
},
"LIST-virtualmachinepools-count": {
"value": 195.06773185133727
},
"LIST-virtualmachinerestores-count": {
"value": 577.2004168113929
},
"LIST-virtualmachines-count": {
"value": 191.06634248002777
},
"LIST-virtualmachinesnapshotcontents-count": {
"value": 191.06634248002777
},
"LIST-virtualmachinesnapshots-count": {
"value": 196.06807919416462
},
"LIST-volumesnapshotclasses-count": {
"value": 197.068426536992
},
"LIST-volumesnapshots-count": {
"value": 190.06599513720042
},
"PATCH-events-count": {
"value": 164.05682578382329
},
"PATCH-nodes-count": {
"value": 6263.1747134421685
},
"PATCH-pods-count": {
"value": 220.0764154220215
},
"PATCH-secrets-count": {
"value": 5.001736714136853
},
"PATCH-virtualmachineinstances-count": {
"value": 152.05279610976035
},
"PATCH-virtualmachines-count": {
"value": 1.0003473428273706
},
"UPDATE-endpoints-count": {
"value": 85950.84404307051
},
"UPDATE-virtualmachineinstances-count": {
"value": 2425.866701002138
},
"UPDATE-virtualmachines-count": {
"value": 24261.424105592218
},
// running状态计数
"running-phase-count": {
"value": 31
},
// 调度状态计数
"scheduling-phase-count": {
"value": 1
},
// 50分位时间
"vmiCreationToRunningSecondsP50": {
"value": 108.125
},
// 95分位时间
"vmiCreationToRunningSecondsP95": {
"value": 172.60975609756096
},
// 99分位时间
"vmiCreationToRunningSecondsP99": {
"value": 178.52195121951218
}
}
}
