ビビリフクロウの足跡

いつもお世話になっているインターネットへの恩返し

kubeadmを用いたKubernetes HAクラスタ on AWS(v.1.11対応版)

前の記事を公開してすぐにv.1.11が出てしまい、構築方法が変わったので、泣きながら再検証...

構成

OS: CentOS7
サーバ群
- Master + etcd
  - 3台(aws-k8s-master[01-03])
- Node
  - 3台(aws-k8s-node[01-03])

前提

作業ユーザはroot

手順

NLBの作成(AWS ManagementConsole等)

(6443/TCPのリスナを作成、aws-k8s-master[01-03]のIPをターゲットグループに登録する)

Dockerのインストール(aws-k8s-master[01-03]、aws-k8s-node[01-03])

yum install -y docker
systemctl enable docker && systemctl start docker

kubeadm、kubelet、kubectlのインストール(aws-k8s-master[01-03]、aws-k8s-node[01-03])

cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
EOF
sed -i 's/^SELINUX=.*/SELINUX=permissive/g' /etc/selinux/config
setenforce 0
yum install -y kubelet kubeadm kubectl
systemctl enable kubelet && systemctl start kubelet

cat <<EOF >  /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sysctl --system

最初のMasterサーバの構築(aws-k8s-master01)

cat <<EOF > kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
kubernetesVersion: v1.11.0
apiServerCertSANs:
- "<NLBのDNS名>"
api:
  controlPlaneEndpoint: "<NLBのDNS名>:6443"
etcd:
  local:
    extraArgs:
      listen-client-urls: "https://127.0.0.1:2379,https://<aws-k8s-master01のIP>:2379"
      advertise-client-urls: "https://<aws-k8s-master01のIP>:2379"
      listen-peer-urls: "https://<aws-k8s-master01のIP>:2380"
      initial-advertise-peer-urls: "https://<aws-k8s-master01のIP>:2380"
      initial-cluster: "aws-k8s-master01=https://<aws-k8s-master01のIP>:2380"
    serverCertSANs:
      - aws-k8s-master01
      - <aws-k8s-master01のIP>
    peerCertSANs:
      - aws-k8s-master01
      - <aws-k8s-master01のIP>
networking:
  podSubnet: "192.168.0.0/16"
EOF

kubeadm init --config kubeadm-config.yaml
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

SSHキーペアの作成(aws-k8s-master02, aws-k8s-master03)

ssh-keygen -t rsa -b 4096 -C ""

cat ~/.ssh/id_rsa.pub
# 出力された内容をaws-k8s-master01の/root/.ssh/authorized_keysのエントリを追加する

証明書群、設定ファイル群の配布(aws-k8s-master02、aws-k8s-master03)

mkdir -p /etc/kubernetes/pki/etcd
scp root@<aws-k8s-master01のIP>:/etc/kubernetes/pki/ca.crt /etc/kubernetes/pki/ca.crt
scp root@<aws-k8s-master01のIP>:/etc/kubernetes/pki/ca.key /etc/kubernetes/pki/ca.key
scp root@<aws-k8s-master01のIP>:/etc/kubernetes/pki/sa.key /etc/kubernetes/pki/sa.key
scp root@<aws-k8s-master01のIP>:/etc/kubernetes/pki/sa.pub /etc/kubernetes/pki/sa.pub
scp root@<aws-k8s-master01のIP>:/etc/kubernetes/pki/front-proxy-ca.crt /etc/kubernetes/pki/front-proxy-ca.crt
scp root@<aws-k8s-master01のIP>:/etc/kubernetes/pki/front-proxy-ca.key /etc/kubernetes/pki/front-proxy-ca.key
scp root@<aws-k8s-master01のIP>:/etc/kubernetes/pki/etcd/ca.crt /etc/kubernetes/pki/etcd/ca.crt
scp root@<aws-k8s-master01のIP>:/etc/kubernetes/pki/etcd/ca.key /etc/kubernetes/pki/etcd/ca.key
scp root@<aws-k8s-master01のIP>:/etc/kubernetes/admin.conf /etc/kubernetes/admin.conf

2台目のMasterサーバの構築(aws-k8s-master02)

cat <<EOF > kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
kubernetesVersion: v1.11.0
apiServerCertSANs:
- "<NLBのDNS名>"
api:
  controlPlaneEndpoint: "<NLBのDNS名>:6443"
etcd:
  local:
    extraArgs:
      listen-client-urls: "https://127.0.0.1:2379,https://<aws-k8s-master02のIP>:2379"
      advertise-client-urls: "https://<aws-k8s-master02のIP>:2379"
      listen-peer-urls: "https://<aws-k8s-master02のIP>:2380"
      initial-advertise-peer-urls: "https://<aws-k8s-master02のIP>:2380"
      initial-cluster: "aws-k8s-master01=https://<aws-k8s-master01のIP>:2380,aws-k8s-master02=https://<aws-k8s-master02のIP>:2380"
      initial-cluster-state: existing
    serverCertSANs:
      - aws-k8s-master02
      - <aws-k8s-master02のIP>
    peerCertSANs:
      - aws-k8s-master02
      - <aws-k8s-master02のIP>
networking:
  podSubnet: "192.168.0.0/16"
EOF

kubeadm alpha phase certs all --config kubeadm-config.yaml
kubeadm alpha phase kubelet config write-to-disk --config kubeadm-config.yaml
kubeadm alpha phase kubelet write-env-file --config kubeadm-config.yaml
kubeadm alpha phase kubeconfig kubelet --config kubeadm-config.yaml
systemctl start kubelet

CP0_IP=<aws-k8s-master01のIP>
CP0_HOSTNAME=aws-k8s-master01
CP1_IP=<aws-k8s-master02のIP>
CP1_HOSTNAME=aws-k8s-master02

KUBECONFIG=/etc/kubernetes/admin.conf kubectl exec -n kube-system etcd-${CP0_HOSTNAME} -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://${CP0_IP}:2379 member add ${CP1_HOSTNAME} https://${CP1_IP}:2380
kubeadm alpha phase etcd local --config kubeadm-config.yaml

kubeadm alpha phase kubeconfig all --config kubeadm-config.yaml
kubeadm alpha phase controlplane all --config kubeadm-config.yaml
kubeadm alpha phase mark-master --config kubeadm-config.yaml

3台目のMasterサーバの構築(aws-k8s-master03)

cat <<EOF > kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
kubernetesVersion: v1.11.0
apiServerCertSANs:
- "<NLBのDNS名>"
api:
  controlPlaneEndpoint: "<NLBのDNS名>:6443"
etcd:
  local:
    extraArgs:
      listen-client-urls: "https://127.0.0.1:2379,https://<aws-k8s-master03のIP>:2379"
      advertise-client-urls: "https://<aws-k8s-master03のIP>:2379"
      listen-peer-urls: "https://<aws-k8s-master03のIP>:2380"
      initial-advertise-peer-urls: "https://<aws-k8s-master03のIP>:2380"
      initial-cluster: "aws-k8s-master01=https://<aws-k8s-master01のIP>:2380,aws-k8s-master02=https://<aws-k8s-master02のIP>:2380,aws-k8s-master03=https://<aws-k8s-master03のIP>:2380"
      initial-cluster-state: existing
    serverCertSANs:
      - aws-k8s-master03
      - <aws-k8s-master03のIP>
    peerCertSANs:
      - aws-k8s-master03
      - <aws-k8s-master03のIP>
networking:
  podSubnet: "192.168.0.0/16"
EOF

kubeadm alpha phase certs all --config kubeadm-config.yaml
kubeadm alpha phase kubelet config write-to-disk --config kubeadm-config.yaml
kubeadm alpha phase kubelet write-env-file --config kubeadm-config.yaml
kubeadm alpha phase kubeconfig kubelet --config kubeadm-config.yaml
systemctl start kubelet

CP0_IP=<aws-k8s-master01のIP>
CP0_HOSTNAME=aws-k8s-master01
CP2_IP=<aws-k8s-master03のIP>
CP2_HOSTNAME=aws-k8s-master03

KUBECONFIG=/etc/kubernetes/admin.conf kubectl exec -n kube-system etcd-${CP0_HOSTNAME} -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://${CP0_IP}:2379 member add ${CP2_HOSTNAME} https://${CP2_IP}:2380
kubeadm alpha phase etcd local --config kubeadm-config.yaml

kubeadm alpha phase kubeconfig all --config kubeadm-config.yaml
kubeadm alpha phase controlplane all --config kubeadm-config.yaml
kubeadm alpha phase mark-master --config kubeadm-config.yaml

flannelのインストール(aws-k8s-master01)

curl -O https://raw.githubusercontent.com/coreos/flannel/v0.10.0/Documentation/kube-flannel.yml
sed -i 's/10.244.0.0/192.168.0.0/g' kube-flannel.yml
kubectl apply -f kube-flannel.yml

Nodeのクラスタ参加(aws-k8s-node[01-03])

kubeadm join --token <token> <NLBのDNS名>:6443 --discovery-token-ca-cert-hash sha256:<hash>
# <token>、<hash>はkubeadm initをaws-k8s-master01で実行した際に表示されたものを使用