by 云网产品事业部-弹性存储-王伟

创建快照

- 创建流程

1.  查询csi.snaps.default中是否有快照信息
2.  生成快照的uuid, 并保存信息到csi.snaps.default, 键值对为"csi.snap."+uuid
3.  创建元数据对象csi.snap.+uuid并保存其他信息到其中, csi.imagename(ceph快照的名字,  "csi-snap-"+uuid), csi.snapname(k8s的快照名字), csi.source(快照父卷的名字, 也是ceph的image name)
4. 先创建父卷的快照, 再根据快照创建克隆卷, 克隆卷的名字等于快照名字, 然后删除父卷快照, 再根据克隆卷创建最终快照, 最终会保留一个与快照同名的卷以及快照
5.  写入ceph快照id到omap中 csi.imageid(ceph image的id)

- k8s 查询快照信息

[root@node1 vdisk]# kubectl get volumesnapshot
NAME                       READYTOUSE   SOURCEPVC         SOURCESNAPSHOTCONTENT   RESTORESIZE   SNAPSHOTCLASS                     SNAPSHOTCONTENT                                    CREATIONTIME   AGE
cstor-pvc-snapshot-vdisk   true         cstor-pvc-vdisk                           1Gi           cstor-csiplugin-vdisk-snapclass   snapcontent-976516a4-56a6-4ebb-bee6-ab069da551c8   4m39s          4m44s
[root@node1 vdisk]# kubectl get volumesnapshotcontent
NAME                                               READYTOUSE   RESTORESIZE   DELETIONPOLICY   DRIVER                VOLUMESNAPSHOTCLASS               VOLUMESNAPSHOT             AGE
snapcontent-976516a4-56a6-4ebb-bee6-ab069da551c8   true         1073741824    Delete           vdisk.csi.cstor.com   cstor-csiplugin-vdisk-snapclass   cstor-pvc-snapshot-vdisk   4m51s

-  查看volumesnapshotcontent详细信息, 查看snapHandle中的uuid

Spec:
  Deletion Policy:  Delete
  Driver:           vdisk.csi.cstor.com
  Source:
    Volume Handle:             0101-24-4a9e463a-4853-4237-a5c5-9ae9d25bacda-00000001-768a1126-f064-11ec-9a2d-36614e068d81
  Volume Snapshot Class Name:  cstor-csiplugin-vdisk-snapclass
  Volume Snapshot Ref:
    API Version:       snapshot.storage.k8s.io/v1beta1
    Kind:              VolumeSnapshot
    Name:              cstor-pvc-snapshot-vdisk
    Namespace:         default
    Resource Version:  6485105
    UID:               976516a4-56a6-4ebb-bee6-ab069da551c8
Status:
  Creation Time:    1655723953278633948
  Ready To Use:     true
  Restore Size:     1073741824
  Snapshot Handle:  0101-24-4a9e463a-4853-4237-a5c5-9ae9d25bacda-00000001-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a
Events:             <none>

根据Snapshot Handle获取uuid为cd9f5f0e-f08a-11ec-8806-1ed64b5d608a

- 查看ceph的快照

(ceph-mon)[root@node2 /]# rbd ls -p rbd
csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a   //克隆卷
csi-vol-768a1126-f064-11ec-9a2d-36614e068d81
(ceph-mon)[root@node2 /]# rbd snap ls -p rbd csi-vol-768a1126-f064-11ec-9a2d-36614e068d81 // 无快照信息
(ceph-mon)[root@node2 /]# rbd snap ls -p rbd csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a // 有快照信息
SNAPID NAME                                          SIZE  PROTECTED TIMESTAMP                
    51 csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a 1 GiB           Mon Jun 20 19:19:14 2022

- 查看rados对象新增加的对象

(ceph-mon)[root@node2 /]# rados -p rbd ls | grep ^csi
csi.snap.cd9f5f0e-f08a-11ec-8806-1ed64b5d608a
csi.snaps.default

- 查看csi.snaps.default新增加的键值对信息

(ceph-mon)[root@node2 /]#  rados  -p rbd listomapvals csi.snaps.default
csi.snap.snapshot-976516a4-56a6-4ebb-bee6-ab069da551c8
value (36 bytes) :
00000000  63 64 39 66 35 66 30 65  2d 66 30 38 61 2d 31 31  |cd9f5f0e-f08a-11|
00000010  65 63 2d 38 38 30 36 2d  31 65 64 36 34 62 35 64  |ec-8806-1ed64b5d|
00000020  36 30 38 61                                       |608a|
00000024

- 查看csi.snap.cd9f5f0e-f08a-11ec-8806-1ed64b5d608a包含omap信息

(ceph-mon)[root@node2 /]#  rados  -p rbd listomapvals csi.snap.cd9f5f0e-f08a-11ec-8806-1ed64b5d608a
csi.imageid
value (14 bytes) :
00000000  32 61 61 33 61 61 33 33  35 66 38 31 32 62        |2aa3aa335f812b|
0000000e

csi.imagename
value (45 bytes) :
00000000  63 73 69 2d 73 6e 61 70  2d 63 64 39 66 35 66 30  |csi-snap-cd9f5f0|
00000010  65 2d 66 30 38 61 2d 31  31 65 63 2d 38 38 30 36  |e-f08a-11ec-8806|
00000020  2d 31 65 64 36 34 62 35  64 36 30 38 61           |-1ed64b5d608a|
0000002d

csi.snapname
value (45 bytes) :
00000000  73 6e 61 70 73 68 6f 74  2d 39 37 36 35 31 36 61  |snapshot-976516a|
00000010  34 2d 35 36 61 36 2d 34  65 62 62 2d 62 65 65 36  |4-56a6-4ebb-bee6|
00000020  2d 61 62 30 36 39 64 61  35 35 31 63 38           |-ab069da551c8|
0000002d

csi.source
value (44 bytes) :
00000000  63 73 69 2d 76 6f 6c 2d  37 36 38 61 31 31 32 36  |csi-vol-768a1126|
00000010  2d 66 30 36 34 2d 31 31  65 63 2d 39 61 32 64 2d  |-f064-11ec-9a2d-|
00000020  33 36 36 31 34 65 30 36  38 64 38 31              |36614e068d81|
0000002c

- csi 日志

got omap values: (pool="rbd", namespace="", name="csi.volume.768a1126-f064-11ec-9a2d-36614e068d81"):
 map[csi.imageid:2aa2c1b62ed876 csi.imagename:csi-vol-768a1126-f064-11ec-9a2d-36614e068d81
 csi.volname:pvc-accbfd11-8b13-4a8a-96aa-cdd86b699223]
got omap values: (pool="rbd", namespace="", name="csi.snaps.default"): map[]
set omap keys (pool="rbd", namespace="", name="csi.snaps.default"):
map[csi.snap.snapshot-976516a4-56a6-4ebb-bee6-ab069da551c8:cd9f5f0e-f08a-11ec-8806-1ed64b5d608a])
set omap keys (pool="rbd", namespace="", name="csi.snap.cd9f5f0e-f08a-11ec-8806-1ed64b5d608a"):
map[csi.imagename:csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a
csi.snapname:snapshot-976516a4-56a6-4ebb-bee6-ab069da551c8
csi.source:csi-vol-768a1126-f064-11ec-9a2d-36614e068d81])
generated Volume ID (0101-24-4a9e463a-4853-4237-a5c5-9ae9d25bacda-00000001-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a)
 and image name (csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a)
 for request name (snapshot-976516a4-56a6-4ebb-bee6-ab069da551c8)
rbd: snap create rbd/csi-vol-768a1126-f064-11ec-9a2d-36614e068d81@csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a
 using mon 10.20.10.12,10.20.10.13
rbd: clone rbd/csi-vol-768a1126-f064-11ec-9a2d-36614e068d81@csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a
 csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a (features: [layering]) using mon 10.20.10.12,10.20.10.13
rbd: snap rm rbd/csi-vol-768a1126-f064-11ec-9a2d-36614e068d81@csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a
using mon 10.20.10.12,10.20.10.13
rbd: snap create rbd/csi-vol-768a1126-f064-11ec-9a2d-36614e068d81@csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a
 using mon 10.20.10.12,10.20.10.13
set omap keys (pool="rbd", namespace="", name="csi.snap.cd9f5f0e-f08a-11ec-8806-1ed64b5d608a"):
map[csi.imageid:2aa3aa335f812b])

从快照创建卷

- 创建流程

1.  先去csi.volumes.default中查询卷是否存在
2.  不存在则生成uuid, 然后写入csi.volumes.default, 键值对为"csi.volume."+pvName: uuid.
3.  创建元数据object, 写入元数据, 元数据信息包括csi.imagename(ceph中的卷name, 格式为"csi-vol"+uuid), csi.volname(pvName)
4.  创建卷, 并根据卷的信息写入omap csi.imageid(ceph image的id)
5.  根据快照创建克隆卷
6.  并根据卷的信息写入omap csi.imageid(ceph image的id)
   
   

- 查看k8s信息

[root@node1 vdisk]# kubectl get pvc
NAME                      STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS         AGE
cstor-pvc-restore-vdisk   Bound    pvc-3603c558-fd4c-4b49-988a-efc6bce11a7a   1Gi        RWO            cstor-csi-vdisk-sc   6s

[root@node1 vdisk]# kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                             STORAGECLASS         REASON   AGE
pvc-3603c558-fd4c-4b49-988a-efc6bce11a7a   1Gi        RWO            Delete           Bound    default/cstor-pvc-restore-vdisk   cstor-csi-vdisk-sc            8s

- 查看pv的详细信息, 确定uuid

Name:            pvc-3603c558-fd4c-4b49-988a-efc6bce11a7a
Labels:          <none>
Annotations:     pv.kubernetes.io/provisioned-by: vdisk.csi.cstor.com
Finalizers:      [kubernetes.io/pv-protection]
StorageClass:    cstor-csi-vdisk-sc
Status:          Bound
Claim:           default/cstor-pvc-restore-vdisk
Reclaim Policy:  Delete
Access Modes:    RWO
VolumeMode:      Filesystem
Capacity:        1Gi
Node Affinity:   <none>
Message:         
Source:
    Type:              CSI (a Container Storage Interface (CSI) volume source)
    Driver:            vdisk.csi.cstor.com
    FSType:            ext4
    VolumeHandle:      0101-24-4a9e463a-4853-4237-a5c5-9ae9d25bacda-00000001-566a36f5-f090-11ec-9a2d-36614e068d81
    ReadOnly:          false
    VolumeAttributes:      clusterID=4a9e463a-4853-4237-a5c5-9ae9d25bacda
                           driverType=vdisk.csi.cstor.com
                           imageName=csi-vol-566a36f5-f090-11ec-9a2d-36614e068d81
                           journalPool=rbd
                           pool=rbd
                           radosNamespace=
                           storage.kubernetes.io/csiProvisionerIdentity=1655707323013-8081-csi.cstor.com
Events:                <none>

根据VolumeHandle确定uuid为566a36f5-f090-11ec-9a2d-36614e068d81

- 查看ceph image信息

(ceph-mon)[root@node2 /]# rbd ls -p rbd
csi-vol-566a36f5-f090-11ec-9a2d-36614e068d81

- 查看csi.volumes.default的元数据信息

csi.volume.pvc-3603c558-fd4c-4b49-988a-efc6bce11a7a
value (36 bytes) :
00000000  35 36 36 61 33 36 66 35  2d 66 30 39 30 2d 31 31  |566a36f5-f090-11|
00000010  65 63 2d 39 61 32 64 2d  33 36 36 31 34 65 30 36  |ec-9a2d-36614e06|
00000020  38 64 38 31                                       |8d81|
00000024

- 查看csi.volume.566a36f5-f090-11ec-9a2d-36614e068d81的omap信息

(ceph-mon)[root@node2 /]#  rados  -p rbd listomapvals csi.volume.566a36f5-f090-11ec-9a2d-36614e068d81
csi.imageid
value (14 bytes) :
00000000  32 61 61 34 31 34 38 33  31 39 30 38 65 33        |2aa414831908e3|
0000000e

csi.imagename
value (44 bytes) :
00000000  63 73 69 2d 76 6f 6c 2d  35 36 36 61 33 36 66 35  |csi-vol-566a36f5|
00000010  2d 66 30 39 30 2d 31 31  65 63 2d 39 61 32 64 2d  |-f090-11ec-9a2d-|
00000020  33 36 36 31 34 65 30 36  38 64 38 31              |36614e068d81|
0000002c

csi.volname
value (40 bytes) :
00000000  70 76 63 2d 33 36 30 33  63 35 35 38 2d 66 64 34  |pvc-3603c558-fd4|
00000010  63 2d 34 62 34 39 2d 39  38 38 61 2d 65 66 63 36  |c-4b49-988a-efc6|
00000020  62 63 65 31 31 61 37 61                           |bce11a7a|
00000028

- 日志打印

got omap values: (pool="rbd", namespace="", name="csi.snap.cd9f5f0e-f08a-11ec-8806-1ed64b5d608a"):
map[csi.imageid:2aa3aa335f812b csi.imagename:csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a
csi.snapname:snapshot-976516a4-56a6-4ebb-bee6-ab069da551c8 csi.source:csi-vol-768a1126-f064-11ec-9a2d-36614e068d81]
got omap values: (pool="rbd", namespace="", name="csi.volumes.default"): map[]
set omap keys (pool="rbd", namespace="", name="csi.volumes.default"):
map[csi.volume.pvc-3603c558-fd4c-4b49-988a-efc6bce11a7a:566a36f5-f090-11ec-9a2d-36614e068d81])
set omap keys (pool="rbd", namespace="", name="csi.volume.566a36f5-f090-11ec-9a2d-36614e068d81"):
map[csi.imagename:csi-vol-566a36f5-f090-11ec-9a2d-36614e068d81 csi.volname:pvc-3603c558-fd4c-4b49-988a-efc6bce11a7a])
generated Volume ID (0101-24-4a9e463a-4853-4237-a5c5-9ae9d25bacda-00000001-566a36f5-f090-11ec-9a2d-36614e068d81)
and image name (csi-vol-566a36f5-f090-11ec-9a2d-36614e068d81) for request name (pvc-3603c558-fd4c-4b49-988a-efc6bce11a7a)
got omap values: (pool="rbd", namespace="", name="csi.snap.cd9f5f0e-f08a-11ec-8806-1ed64b5d608a"):
 map[csi.imageid:2aa3aa335f812b csi.imagename:csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a
  csi.snapname:snapshot-976516a4-56a6-4ebb-bee6-ab069da551c8 csi.source:csi-vol-768a1126-f064-11ec-9a2d-36614e068d81]
rbd: clone rbd/csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a@csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a
csi-vol-566a36f5-f090-11ec-9a2d-36614e068d81 (features: [layering]) using mon 10.20.10.12,10.20.10.13

删除快照回滚卷

- 删除流程跟删除普通卷和克隆卷一致

1.  检查是否还在使用
2.  生成临时克隆卷, 即卷的名称+temp
3.  尝试trash删除临时卷, 如果错误是没发现卷, 忽略该错误, 其他错误则报错返回
4.  trash删除卷
5.  移除omap对象, 移除csi.volums.default的记录

- csi日志

got omap values: (pool="rbd", namespace="", name="csi.volume.768a1126-f064-11ec-9a2d-36614e068d81"):
map[csi.imageid:2aa2c1b62ed876 csi.imagename:csi-vol-768a1126-f064-11ec-9a2d-36614e068d81
 csi.volname:pvc-accbfd11-8b13-4a8a-96aa-cdd86b699223]
command /usr/bin/rbd -m 10.20.10.12,10.20.10.13 --id test --keyfile=/tmp/csi/keys/keyfile-2661790888
 -c /etc/ceph/ceph.conf --format=json info rbd/csi-vol-768a1126-f064-11ec-9a2d-36614e068d81 start
the command /usr/bin/rbd -m 10.20.10.12,10.20.10.13 --id test --keyfile=/tmp/csi/keys/keyfile-2661790888
 -c /etc/ceph/ceph.conf --format=json info rbd/csi-vol-768a1126-f064-11ec-9a2d-36614e068d81 exits normally
Can not find rbd image: rbd/csi-vol-768a1126-f064-11ec-9a2d-36614e068d81-temp, but maybe no error
deleting image csi-vol-768a1126-f064-11ec-9a2d-36614e068d81
rbd: delete csi-vol-768a1126-f064-11ec-9a2d-36614e068d81 using mon 10.20.10.12,10.20.10.13, pool rbd
executing [rbd task add trash remove rbd/2aa2c1b62ed876 --id test --keyfile=/tmp/csi/keys/keyfile-2661790888
-m 10.20.10.12,10.20.10.13] for image (csi-vol-768a1126-f064-11ec-9a2d-36614e068d81)
using mon 10.20.10.12,10.20.10.13, pool rbd
command /usr/bin/ceph rbd task add trash remove rbd/2aa2c1b62ed876 --id test
--keyfile=/tmp/csi/keys/keyfile-2661790888 -m 10.20.10.12,10.20.10.13 start
run the command successfully: /usr/bin/ceph rbd task add trash remove rbd/2aa2c1b62ed876 --id test
--keyfile=/tmp/csi/keys/keyfile-2661790888 -m 10.20.10.12,10.20.10.13,
stdout: {"message": "Removing image rbd/2aa2c1b62ed876 from trash",
"id": "43152a6e-82a3-466f-b370-6173082501fc", "refs": {"action": "trash remove",
 "pool_name": "rbd", "pool_namespace": "", "image_id": "2aa2c1b62ed876"}, "sequence": 1}, stderr:
the command /usr/bin/ceph rbd task add trash remove rbd/2aa2c1b62ed876 --id test
 --keyfile=/tmp/csi/keys/keyfile-2661790888 -m 10.20.10.12,10.20.10.13 exits normally
removed omap keys (pool="rbd", namespace="", name="csi.volumes.default"):
 [csi.volume.pvc-accbfd11-8b13-4a8a-96aa-cdd86b699223]

删除快照

- 删除流程

1.  删除快照
2.  删除快照同名的volume
3.  删除元数据

- csi日志

got omap values: (pool="rbd", namespace="", name="csi.snap.cd9f5f0e-f08a-11ec-8806-1ed64b5d608a"):
map[csi.imageid:2aa3aa335f812b csi.imagename:csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a
csi.snapname:snapshot-976516a4-56a6-4ebb-bee6-ab069da551c8 csi.source:csi-vol-768a1126-f064-11ec-9a2d-36614e068d81]
deleting cloned rbd volume csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a
rbd: snap rm rbd/csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a@csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a
using mon 10.20.10.12,10.20.10.13
rbd: delete csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a using mon 10.20.10.12,10.20.10.13, pool rbd
executing [rbd task add trash remove rbd/2aa3aa335f812b --id test --keyfile=/tmp/csi/keys/keyfile-3289575306
-m 10.20.10.12,10.20.10.13] for image (csi-snap-cd9f5f0e-f08a-11ec-8806-1ed64b5d608a)
using mon 10.20.10.12,10.20.10.13, pool rbd
command /usr/bin/ceph rbd task add trash remove rbd/2aa3aa335f812b --id test
--keyfile=/tmp/csi/keys/keyfile-3289575306 -m 10.20.10.12,10.20.10.13 start
run the command successfully: /usr/bin/ceph rbd task add trash remove rbd/2aa3aa335f812b
--id test --keyfile=/tmp/csi/keys/keyfile-3289575306 -m 10.20.10.12,10.20.10.13,
stdout: {"message": "Removing image rbd/2aa3aa335f812b from trash",
"id": "2ef0fc5e-bca1-404f-b1d0-7afc57b1fba8", "refs": {"action": "trash remove",
"pool_name": "rbd", "pool_namespace": "", "image_id": "2aa3aa335f812b"}, "sequence": 2}, stderr:
the command /usr/bin/ceph rbd task add trash remove rbd/2aa3aa335f812b --id
test --keyfile=/tmp/csi/keys/keyfile-3289575306 -m 10.20.10.12,10.20.10.13 exits normally
removed omap keys (pool="rbd", namespace="", name="csi.snaps.default"):
[csi.snap.snapshot-976516a4-56a6-4ebb-bee6-ab069da551c8]