<center><font size=7 color=#186A3B >k8s etcd镜像集群方案</font></center>

# 1 背景

- k8s集群master分布于AZ1和AZ2

- AZ2分布了两个etcd节点，一旦AZ2挂掉，整个集群立即变为不可用状态
- 在已有etcd集群的基础上，再额外部署一个etcd镜像集群，当主集群故障不可用时，切换到备用etcd镜像集群

以下仅代表测试环境

|   用途   | IP |  AZ  | ETCD |
| :------: |:------: | :--: |:------:|
| master1  | 10.16.0.45 | AZ1  | 二进制 |
| master2  | 10.16.0.43 | AZ2  | 二进制 |
| master3  | 10.16.0.46 | AZ2  | 二进制 |
|  node1   | 10.16.0.42 | AZ1  | 无 |
|  node2   | 10.16.0.41 | 其他  | 二进制/待加入 |
| node3 | 10.16.0.58 | 其他 | 二进制/待加入 |
| node4 | 10.16.0.59 | 其他 | 二进制/待加入 |



# 2 远端etcd节点安装ETCD

```shell
将二进制etcd、etcdctl放到3个镜像etcd节点/usr/local/bin/下

# 设置为service
[root@node2 ~]# vi /etc/systemd/system/etcd.service
[Unit]
Description=etcd
After=network.target

[Service]
Type=notify
User=root
EnvironmentFile=/etc/etcd.env
ExecStart=/usr/local/bin/etcd
NotifyAccess=all
Restart=always
RestartSec=10s
LimitNOFILE=40000

[Install]
WantedBy=multi-user.target

# 设置环境变量
# 注意IP和ETCD_NAME
# 配置镜像集群第一个节点
[root@node2 ~]# vi /etc/etcd.env
# Environment file for etcd v3.5.0
ETCD_DATA_DIR=/var/lib/etcd
ETCD_ADVERTISE_CLIENT_URLS=h(体体)ps://10.16.0.41:2379
ETCD_INITIAL_ADVERTISE_PEER_URLS=h(体体)ps://10.16.0.41:2380
ETCD_INITIAL_CLUSTER_STATE=new
ETCD_METRICS=basic
ETCD_LISTEN_METRICS_URLS=h(体体)ps://0.0.0.0:2381
ETCD_LISTEN_CLIENT_URLS=h(体体)ps://10.16.0.41:2379,h(体体)ps://127.0.0.1:2379
ETCD_ELECTION_TIMEOUT=5000
ETCD_HEARTBEAT_INTERVAL=250
ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd
ETCD_LISTEN_PEER_URLS=h(体体)ps://10.16.0.41:2380
ETCD_NAME=etcd1
ETCD_PROXY=off
ETCD_INITIAL_CLUSTER=etcd1=h(体体)ps://10.16.0.41:2380
ETCD_AUTO_COMPACTION_RETENTION=8
ETCD_SNAPSHOT_COUNT=10000
ETCD_QUOTA_BACKEND_BYTES=6442450944
# Flannel need etcd v2 API
ETCD_ENABLE_V2=true

# TLS settings
ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem
ETCD_CERT_FILE=/etc/ssl/etcd/ssl/member-node2.pem
ETCD_KEY_FILE=/etc/ssl/etcd/ssl/member-node2-key.pem
ETCD_CLIENT_CERT_AUTH=true

ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem
ETCD_PEER_CERT_FILE=/etc/ssl/etcd/ssl/member-node2.pem
ETCD_PEER_KEY_FILE=/etc/ssl/etcd/ssl/member-node2-key.pem
ETCD_PEER_CLIENT_CERT_AUTH=True




# CLI settings
ETCDCTL_ENDPOINTS=h(体体)ps://127.0.0.1:2379
ETCDCTL_CACERT=/etc/ssl/etcd/ssl/ca.pem
ETCDCTL_KEY=/etc/ssl/etcd/ssl/admin-node2-key.pem
ETCDCTL_CERT=/etc/ssl/etcd/ssl/admin-node2.pem
```
# 3 ETCD准备

```shell
# 拷贝原集群etcd根证书
[root@node2 ~]# mkdir -p /etc/ssl/etcd/ssl
[root@node2 ~]# scp 10.16.0.46:/etc/ssl/etcd/ssl/* /etc/ssl/etcd/ssl
# 创建证书制作脚本目录
[root@node2 ~]# mkdir /etcddatabak
```

# 4 制作证书
## 制作证书脚本
```shell
[root@node2 ~]# vi /etcddatabak/make-ssl-etcd.sh
#!/bin/bash

# Author: Smana smainklh@gmail.com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o pipefail
usage()
{
    cat << EOF
Create self signed certificates

Usage : $(basename $0) -f <config> [-d <ssldir>]
      -h | --help         : Show this message
      -f | --config       : Openssl configuration file
      -d | --ssldir       : Directory where the certificates will be installed

               ex :
               $(basename $0) -f openssl.conf -d /srv/ssl
EOF
}

# Options parsing
while (($#)); do
    case "$1" in
        -h | --help)   usage;   exit 0;;
        -f | --config) CONFIG=${2}; shift 2;;
        -d | --ssldir) SSLDIR="${2}"; shift 2;;
        *)
            usage
            echo "ERROR : Unknown option"
            exit 3
        ;;
    esac
done

if [ -z ${CONFIG} ]; then
    echo "ERROR: the openssl configuration file is missing. option -f"
    exit 1
fi
if [ -z ${SSLDIR} ]; then
    SSLDIR="/etc/ssl/etcd"
fi

tmpdir=$(mktemp -d /tmp/etcd_cacert.XXXXXX)
trap 'rm -rf "${tmpdir}"' EXIT
cd "${tmpdir}"

mkdir -p "${SSLDIR}"

# Root CA
if [ -e "$SSLDIR/ca-key.pem" ]; then
    # Reuse existing CA
    cp $SSLDIR/{ca.pem,ca-key.pem} .
else
    openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1
    openssl req -x509 -new -nodes -key ca-key.pem -days 36500 -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1
fi

# ETCD member
if [ -n "$MASTERS" ]; then
    for host in $MASTERS; do
        cn="${host%%.*}"
        # Member key
        openssl genrsa -out member-${host}-key.pem 2048 > /dev/null 2>&1
        openssl req -new -key member-${host}-key.pem -out member-${host}.csr -subj "/CN=etcd-member-${cn}" -config ${CONFIG} > /dev/null 2>&1
        openssl x509 -req -in member-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member-${host}.pem -days 36500 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1

        # Admin key
        openssl genrsa -out admin-${host}-key.pem 2048 > /dev/null 2>&1
        openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=etcd-admin-${cn}" > /dev/null 2>&1
        openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days 36500 -extensions ssl_client  -extfile ${CONFIG} > /dev/null 2>&1
    done
fi

# Node keys
if [ -n "$HOSTS" ]; then
    for host in $HOSTS; do
        cn="${host%%.*}"
        openssl genrsa -out node-${host}-key.pem 2048 > /dev/null 2>&1
        openssl req -new -key node-${host}-key.pem -out node-${host}.csr -subj "/CN=etcd-node-${cn}" > /dev/null 2>&1
        openssl x509 -req -in node-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out node-${host}.pem -days 36500 -extensions ssl_client  -extfile ${CONFIG} > /dev/null 2>&1
    done
fi

# Install certs
if [ -e "$SSLDIR/ca-key.pem" ]; then
    # No pass existing CA
    rm -f ca.pem ca-key.pem
fi

mv *.pem ${SSLDIR}/



-------------------------------------------------------
# 生成openssl.conf
# 注意修改主机名和IP
[root@node2 ~]# vi /etcddatabak/openssl.conf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name

[req_distinguished_name]

[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names

[ ssl_client ]
extendedKeyUsage = clientAuth, serverAuth
basicConstraints = CA:FALSE
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
subjectAltName = @alt_names

[ v3_ca ]
basicConstraints = CA:TRUE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
authorityKeyIdentifier=keyid:always,issuer

[alt_names]
DNS.1 = localhost
DNS.2 = master1
DNS.3 = master2
DNS.4 = master3
DNS.5 = etcd.kube-system.svc.cluster.net
DNS.6 = etcd.kube-system.svc
DNS.7 = etcd.kube-system
DNS.8 = etcd
DNS.9 = node2
DNS.10 = node3
DNS.11 = node4
IP.1 = 10.16.0.45
IP.2 = 10.16.0.45
IP.3 = 10.16.0.43
IP.4 = 10.16.0.43
IP.5 = 10.16.0.46
IP.6 = 10.16.0.58
IP.7 = 10.16.0.41
IP.8 = 10.16.0.59
IP.9 = 127.0.0.1


------------------------

# 生成新的证书
[root@node2 ~]# env MASTERS='master1 master2 master3 node2 node3 node4' bash -x make-ssl-etcd.sh -f /root/etcd/openssl.conf -d /etc/ssl/etcd/ssl

# 将新证书拷贝到原ETCD集群
[root@node2 ~]# scp /etc/ssl/etcd/ssl/* 10.16.0.45:/etc/ssl/etcd/ssl
```

# 5 启动镜像集群第一个 etcd 节点

```shell
# 注意修改IP和add后的名称
[root@node2 ~]# systemctl enable etcd
[root@node2 ~]# systemctl start etcd
```

# 6 添加镜像集群第二个ETCD节点到集群中
```shell
# 添加第二个节点之后，集群会短暂无法访问，待第二个节点启动之后，集群恢复正常
[root@node2 ~]# etcdctl --cacert=/etc/ssl/etcd/ssl/ca.pem --cert=/etc/ssl/etcd/ssl/ca.pem --key=/etc/ssl/etcd/ssl/ca-key.pem --endpoints=h(体体)ps://10.16.0.41:2379 member add etcd2 --peer-urls=h(体体)ps://10.16.0.58:2380
```

# 7 配置镜像集群第二个 etcd 节点

```shell
# 设置为service
[root@node2 ~]# vi /etc/systemd/system/etcd.service
[Unit]
Description=etcd
After=network.target

[Service]
Type=notify
User=root
EnvironmentFile=/etc/etcd.env
ExecStart=/usr/local/bin/etcd
NotifyAccess=all
Restart=always
RestartSec=10s
LimitNOFILE=40000

[Install]
WantedBy=multi-user.target

# 设置环境变量
# 注意IP和ETCD_NAME
# 配置镜像集群第一个节点
[root@node2 ~]# vi /etc/etcd.env
# Environment file for etcd v3.5.0
ETCD_DATA_DIR=/var/lib/etcd
ETCD_ADVERTISE_CLIENT_URLS=h(体体)ps://10.16.0.58:2379
ETCD_INITIAL_ADVERTISE_PEER_URLS=h(体体)ps://10.16.0.58:2380
ETCD_INITIAL_CLUSTER_STATE=existing
ETCD_METRICS=basic
ETCD_LISTEN_METRICS_URLS=h(体体)ps://0.0.0.0:2381
ETCD_LISTEN_CLIENT_URLS=h(体体)ps://10.16.0.58:2379,h(体体)ps://127.0.0.1:2379
ETCD_ELECTION_TIMEOUT=5000
ETCD_HEARTBEAT_INTERVAL=250
ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd
ETCD_LISTEN_PEER_URLS=h(体体)ps://10.16.0.58:2380
ETCD_NAME=etcd2
ETCD_PROXY=off
ETCD_INITIAL_CLUSTER=etcd1=h(体体)ps://10.16.0.41:2380,etcd2=h(体体)ps://10.16.0.58:2380
ETCD_AUTO_COMPACTION_RETENTION=8
ETCD_SNAPSHOT_COUNT=10000
ETCD_QUOTA_BACKEND_BYTES=6442450944
# Flannel need etcd v2 API
ETCD_ENABLE_V2=true

# TLS settings
ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem
ETCD_CERT_FILE=/etc/ssl/etcd/ssl/member-node3.pem
ETCD_KEY_FILE=/etc/ssl/etcd/ssl/member-node3-key.pem
ETCD_CLIENT_CERT_AUTH=true

ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem
ETCD_PEER_CERT_FILE=/etc/ssl/etcd/ssl/member-node3.pem
ETCD_PEER_KEY_FILE=/etc/ssl/etcd/ssl/member-node3-key.pem
ETCD_PEER_CLIENT_CERT_AUTH=True




# CLI settings
ETCDCTL_ENDPOINTS=h(体体)ps://127.0.0.1:2379
ETCDCTL_CACERT=/etc/ssl/etcd/ssl/ca.pem
ETCDCTL_KEY=/etc/ssl/etcd/ssl/admin-node3-key.pem
ETCDCTL_CERT=/etc/ssl/etcd/ssl/admin-node3.pem
```

```shell
# 启动第二个etcd节点
[root@node3 ~]# systemctl enable etcd
[root@node3 ~]# systemctl start etcd

```

# 8 添加镜像集群第三个ETCD节点到集群中
```shell
# 添加第三个节点
[root@node2 ~]# etcdctl --cacert=/etc/ssl/etcd/ssl/ca.pem --cert=/etc/ssl/etcd/ssl/ca.pem --key=/etc/ssl/etcd/ssl/ca-key.pem --endpoints=h(体体)ps://10.16.0.41:2379 member add etcd3 --peer-urls=h(体体)ps://10.16.0.59:2380
```

# 9 配置镜像集群第三个 etcd 节点

```shell
# 设置为service
[root@node4 ~]# vi /etc/systemd/system/etcd.service
[Unit]
Description=etcd
After=network.target

[Service]
Type=notify
User=root
EnvironmentFile=/etc/etcd.env
ExecStart=/usr/local/bin/etcd
NotifyAccess=all
Restart=always
RestartSec=10s
LimitNOFILE=40000

[Install]
WantedBy=multi-user.target

# 设置环境变量
# 注意IP和ETCD_NAME
# 配置镜像集群第一个节点
[root@node4 ~]# vi /etc/etcd.env
# Environment file for etcd v3.5.0
ETCD_DATA_DIR=/var/lib/etcd
ETCD_ADVERTISE_CLIENT_URLS=h(体体)ps://10.16.0.59:2379
ETCD_INITIAL_ADVERTISE_PEER_URLS=h(体体)ps://10.16.0.59:2380
ETCD_INITIAL_CLUSTER_STATE=existing
ETCD_METRICS=basic
ETCD_LISTEN_METRICS_URLS=h(体体)p://0.0.0.0:2381
ETCD_LISTEN_CLIENT_URLS=h(体体)ps://10.16.0.59:2379,h(体体)ps://127.0.0.1:2379
ETCD_ELECTION_TIMEOUT=5000
ETCD_HEARTBEAT_INTERVAL=250
ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd
ETCD_LISTEN_PEER_URLS=h(体体)ps://10.16.0.59:2380
ETCD_NAME=etcd3
ETCD_PROXY=off
ETCD_INITIAL_CLUSTER=etcd1=h(体体)ps://10.16.0.41:2380,etcd2=h(体体)ps://10.16.0.58:2380,etcd3=h(体体)ps://10.16.0.59:2380
ETCD_AUTO_COMPACTION_RETENTION=8
ETCD_SNAPSHOT_COUNT=10000
ETCD_QUOTA_BACKEND_BYTES=6442450944
# Flannel need etcd v2 API
ETCD_ENABLE_V2=true

# TLS settings
ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem
ETCD_CERT_FILE=/etc/ssl/etcd/ssl/member-node4.pem
ETCD_KEY_FILE=/etc/ssl/etcd/ssl/member-node4-key.pem
ETCD_CLIENT_CERT_AUTH=true

ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem
ETCD_PEER_CERT_FILE=/etc/ssl/etcd/ssl/member-node4.pem
ETCD_PEER_KEY_FILE=/etc/ssl/etcd/ssl/member-node4-key.pem
ETCD_PEER_CLIENT_CERT_AUTH=True




# CLI settings
ETCDCTL_ENDPOINTS=h(体体)ps://127.0.0.1:2379
ETCDCTL_CACERT=/etc/ssl/etcd/ssl/ca.pem
ETCDCTL_KEY=/etc/ssl/etcd/ssl/admin-node4-key.pem
ETCDCTL_CERT=/etc/ssl/etcd/ssl/admin-node4.pem
```

```shell
# 启动第二个etcd节点
[root@node4 ~]# systemctl enable etcd
[root@node4 ~]# systemctl start etcd
```


# 10 配置镜像集群，开启同步，并且需要一致保持
```shell
[root@master1 ~]#  etcdctl --cacert=/etc/ssl/etcd/ssl/ca.pem --cert=/etc/ssl/etcd/ssl/ca.pem --key=/etc/ssl/etcd/ssl/ca-key.pem --endpoints=h(体体)ps://10.16.0.46:2379,h(体体)ps://10.16.0.45:2379,h(体体)ps://10.16.0.43:2379 make-mirror  --dest-cacert=/etc/ssl/etcd/ssl/ca.pem --dest-cert=/etc/ssl/etcd/ssl/ca.pem --dest-key=/etc/ssl/etcd/ssl/ca-key.pem h(体体)ps://10.16.0.41:2379
```