ETCD 安装指导

简介

此文档接收如何安装部署ETCD集群，以3节点为例。

安装

环境规划

系统类型：RHEL-7.3
配置：4c/8g/50g
地址：10.243.176.158/159/160 总计3台

安装依赖

配置内网yum源

安装包获取

从外网获取：etcd-3.3.11-2.el7.centos.x86_64.rpm

执行安装命令：

修改配置文件

在10.243.176.158上修改配置文件

 文件位置：/etc/etcd/etcd.conf

 #[Member]
 ETCD_DATA_DIR="/cloud/etcd/snb_etcd_1.etcd"
 ETCD_LISTEN_PEER_URLS="http://10.243.176.158:2380"
 ETCD_LISTEN_CLIENT_URLS="http://10.243.176.158:2379"
 ETCD_MAX_SNAPSHOTS="5"
 ETCD_NAME="snb_etcd_1"
 ETCD_SNAPSHOT_COUNT="1000"

 #[Cluster]
 ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.243.176.158:2380"
 ETCD_ADVERTISE_CLIENT_URLS="http://10.243.176.158:2379"

 ETCD_INITIAL_CLUSTER="snb_etcd_1=http://10.243.176.158:2380,snb_etcd_2=http://10.243.176.159:2380,snb_etc d_3=http://10.243.176.160:2380"
 ETCD_INITIAL_CLUSTER_TOKEN="snb-etcd-cluster"
 ETCD_INITIAL_CLUSTER_STATE="new"
 #ETCD_ENABLE_V2="true"

在10.243.176.158上修改配置文件

10.243.176.159上文件：/etc/etcd/etcd.conf

 #[Member]
 ETCD_DATA_DIR="/cloud/etcd/snb_etcd_2.etcd"
 ETCD_LISTEN_PEER_URLS="http://10.243.176.159:2380"
 ETCD_LISTEN_CLIENT_URLS="http://10.243.176.159:2379"
 ETCD_MAX_SNAPSHOTS="5"
 ETCD_NAME="snb_etcd_2"
 ETCD_SNAPSHOT_COUNT="1000"

 #[Cluster]
 ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.243.176.159:2380"
 ETCD_ADVERTISE_CLIENT_URLS="http://10.243.176.159:2379"

 ETCD_INITIAL_CLUSTER="snb_etcd_1=http://10.243.176.158:2380,snb_etcd_2=http://10.243.176.159:2380,snb_etc d_3=http://10.243.176.160:2380"
 ETCD_INITIAL_CLUSTER_TOKEN="snb-etcd-cluster"
 ETCD_INITIAL_CLUSTER_STATE="new"
 #ETCD_ENABLE_V2="true"

在10.243.176.158上修改配置文件

10.243.176.160上文件：/etc/etcd/etcd.conf

 #[Member]
 ETCD_DATA_DIR="/cloud/etcd/snb_etcd_3.etcd"
 ETCD_LISTEN_PEER_URLS="http://10.243.176.160:2380"
 ETCD_LISTEN_CLIENT_URLS="http://10.243.176.160:2379"
 ETCD_MAX_SNAPSHOTS="5"
 ETCD_NAME="snb_etcd_3"
 ETCD_SNAPSHOT_COUNT="1000"

 #[Cluster]
 ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.243.176.160:2380"
 ETCD_ADVERTISE_CLIENT_URLS="http://10.243.176.160:2379"

 ETCD_INITIAL_CLUSTER="snb_etcd_1=http://10.243.176.158:2380,snb_etcd_2=http://10.243.176.159:2380,snb_etc d_3=http://10.243.176.160:2380"
 ETCD_INITIAL_CLUSTER_TOKEN="snb-etcd-cluster"
 ETCD_INITIAL_CLUSTER_STATE="new"
 #ETCD_ENABLE_V2="true"

启动服务

在10.243.176.158上执行

 # systemctl start etcd

在10.243.176.159上执行

 # systemctl start etcd

在10.243.176.160上执行

 # systemctl start etcd

服务状态查看

在10.243.176.158上执行

 # systemctl status etcd

在10.243.176.159上执行

 # systemctl status etcd

在10.243.176.160上执行

 # systemctl status etcd

测试

创建flannel需要的网络配置

 # ETCD_API=2
 # etcdctl --endpoints http://10.243.176.160:2379 mk /snbcloud/network1/config "{\"Network\":\"172.31.0.0/16\",\"SubnetLen\":27,\"Backend\":{\"Type\":\"host-gw\"}}"

 etcdctl --endpoints http://10.243.176.160:2379 mk /snbcloud/network1/config "{\"Network\":\"172.31.0.0/16\",\"SubnetLen\":27,\"Backend\":{\"Type\":\"vxlan\"}}"

查询路径及value值

 # etcdctl --endpoints http://10.243.176.160:2379 ls /snbcloud/network1
 # etcdctl --endpoints http://10.243.176.160:2379 get /snbcloud/network1/config

从ETCD中获取k8s元数据

查看etcd的启动命令

ps -ef |grep etcd


/usr/local/bin/etcd --name=master1.smx-dev-k8s.smx.com --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem --peer-cert-file=/etc/etcd/ssl/etcd.pem --peer-key-file=/etc/etcd/ssl/etcd-key.pem --trusted-ca-file=/etc/kubernetes/ssl/ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem --initial-advertise-peer-urls=https://10.0.133.211:2380 --listen-peer-urls=https://10.0.133.211:2380 --listen-client-urls=https://10.0.133.211:2379,http://127.0.0.1:2379 --advertise-client-urls=https://10.0.133.211:2379 --initial-cluster-token=etcd-cluster-0 --initial-cluster=master3.smx-dev-k8s.smx.com=https://10.0.133.213:2380,master2.smx-dev-k8s.smx.com=https://10.0.133.212:2380,master1.smx-dev-k8s.smx.com=https://10.0.133.211:2380 --initial-cluster-state=new --data-dir=/var/lib/etcd

截取其中的：--trusted-ca-file=/etc/kubernetes/ssl/ca.pem 修改为：--cacert=/etc/kubernetes/ssl/ca.pem
设置etcdctl版本

 export ETCDCTL_API=3

执行如下命令获取etcd中的信息，只查询key，不反会数据

 etcdctl  --cacert=/etc/kubernetes/ssl/ca.pem --endpoints https://10.0.133.211:2379 get / --prefix --keys-only

获取指定前缀的key及对应的数据

 etcdctl  --cacert=/etc/kubernetes/ssl/ca.pem --endpoints https://10.0.133.211:2379 get  /registry/leases/  --prefix

容器化部署ETCD

集群部署-3节点

# 如下可根据实际情况修改，仓库地址变量
REGISTRY=quay.io/coreos/etcd
REGISTRY=gcr.io/etcd-development/etcd

# 在每个宿主机节点上配置
ETCD_VERSION=latest
TOKEN=my-etcd-token
CLUSTER_STATE=new
NAME_1=etcd-node-0
NAME_2=etcd-node-1
NAME_3=etcd-node-2
HOST_1=10.20.30.1
HOST_2=10.20.30.2
HOST_3=10.20.30.3
CLUSTER=${NAME_1}=http://${HOST_1}:2380,${NAME_2}=http://${HOST_2}:2380,${NAME_3}=http://${HOST_3}:2380

# 如下是etcd存放数据的目录，可根据实际现需要进行修改
DATA_DIR=/var/lib/etcd

# 节点1上启动ETCD服务
THIS_NAME=${NAME_1}
THIS_IP=${HOST_1}
docker run \
  -p 2379:2379 \
  -p 2380:2380 \
  --volume=${DATA_DIR}:/etcd-data \
  --name etcd ${REGISTRY}:${ETCD_VERSION} \
  /usr/local/bin/etcd \
  --data-dir=/etcd-data --name ${THIS_NAME} \
  --initial-advertise-peer-urls http://${THIS_IP}:2380 --listen-peer-urls http://0.0.0.0:2380 \
  --advertise-client-urls http://${THIS_IP}:2379 --listen-client-urls http://0.0.0.0:2379 \
  --initial-cluster ${CLUSTER} \
  --initial-cluster-state ${CLUSTER_STATE} --initial-cluster-token ${TOKEN}

# 节点2上启动ETCD服务
THIS_NAME=${NAME_2}
THIS_IP=${HOST_2}
docker run \
  -p 2379:2379 \
  -p 2380:2380 \
  --volume=${DATA_DIR}:/etcd-data \
  --name etcd ${REGISTRY}:${ETCD_VERSION} \
  /usr/local/bin/etcd \
  --data-dir=/etcd-data --name ${THIS_NAME} \
  --initial-advertise-peer-urls http://${THIS_IP}:2380 --listen-peer-urls http://0.0.0.0:2380 \
  --advertise-client-urls http://${THIS_IP}:2379 --listen-client-urls http://0.0.0.0:2379 \
  --initial-cluster ${CLUSTER} \
  --initial-cluster-state ${CLUSTER_STATE} --initial-cluster-token ${TOKEN}

# 节点3上启动ETCD服务
THIS_NAME=${NAME_3}
THIS_IP=${HOST_3}
docker run \
  -p 2379:2379 \
  -p 2380:2380 \
  --volume=${DATA_DIR}:/etcd-data \
  --name etcd ${REGISTRY}:${ETCD_VERSION} \
  /usr/local/bin/etcd \
  --data-dir=/etcd-data --name ${THIS_NAME} \
  --initial-advertise-peer-urls http://${THIS_IP}:2380 --listen-peer-urls http://0.0.0.0:2380 \
  --advertise-client-urls http://${THIS_IP}:2379 --listen-client-urls http://0.0.0.0:2379 \
  --initial-cluster ${CLUSTER} \
  --initial-cluster-state ${CLUSTER_STATE} --initial-cluster-token ${TOKEN}

ETCD常用命令

获取集群节点

$ etcdctl --endpoints=http://${NODE1}:2379 member list

生成快照

$ etcdctl --endpoints=http://${NODE1}:2379  snapshot save snapshot.db

恢复集群数据

# 导入数据
$ ETCDCTL_API=3 etcdctl snapshot restore snapshot.db \
  --name m1 \
  --initial-cluster m1=http://${NODE1}:2380,m2=http://${NODE2}:2380,m3=http://${NODE3}:2380 \
  --initial-cluster-token etcd-cluster-1 \
  --initial-advertise-peer-urls http://${NODE1}:2380
$ ETCDCTL_API=3 etcdctl snapshot restore snapshot.db \
  --name m2 \
  --initial-cluster m1=http://${NODE1}:2380,m2=http://${NODE2}:2380,m3=http://${NODE3}:2380 \
  --initial-cluster-token etcd-cluster-1 \
  --initial-advertise-peer-urls http://${NODE2}:2380
$ ETCDCTL_API=3 etcdctl snapshot restore snapshot.db \
  --name m3 \
  --initial-cluster m1=http://${NODE1}:2380,m2=http://${NODE2}:2380,m3=http://${NODE3}:2380 \
  --initial-cluster-token etcd-cluster-1 \
  --initial-advertise-peer-urls http://${NODE3}:2380

# 启动ETCD服务
$ etcd \
  --name m1 \
  --listen-client-urls http://host1:2379 \
  --advertise-client-urls http://host1:2379 \
  --listen-peer-urls http://host1:2380 &
$ etcd \
  --name m2 \
  --listen-client-urls http://host2:2379 \
  --advertise-client-urls http://host2:2379 \
  --listen-peer-urls http://host2:2380 &
$ etcd \
  --name m3 \
  --listen-client-urls http://host3:2379 \
  --advertise-client-urls http://host3:2379 \
  --listen-peer-urls http://host3:2380 &

结语

## 回首页

Keys	Action
`?`	Open this help
`←`	Previous page
`→`	Next page
`s`	Search