K8s-1.29.2二进制安装-第三章(Node组件 及其他插件安装)

1、Node 配置

1.1 复制相关证书至 node

# 在 master01 上将证书复制到 node 节点
cd /etc/kubernetes/
 
# for NODE in k8s-master02 k8s-master03 k8s-node01 k8s-node02; do ssh $NODE mkdir -p /etc/kubernetes/pki; for FILE in pki/ca.pem pki/ca-key.pem pki/front-proxy-ca.pem bootstrap-kubelet.kubeconfig kube-proxy.kubeconfig; do scp /etc/kubernetes/$FILE $NODE:/etc/kubernetes/${FILE}; done; done

for NODE in server171 server172 server173 server174; do ssh $NODE mkdir -p /etc/kubernetes/pki; for FILE in pki/ca.pem pki/ca-key.pem pki/front-proxy-ca.pem bootstrap-kubelet.kubeconfig kube-proxy.kubeconfig; do scp /etc/kubernetes/$FILE $NODE:/etc/kubernetes/${FILE}; done; done

1.2 配置 kubelet（所有节点）

# kubelet 配置
# 当使用 docker 作为 Runtime
cat > /usr/lib/systemd/system/kubelet.service << EOF

[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=network-online.target firewalld.service cri-docker.service docker.socket containerd.service
Wants=network-online.target
Requires=docker.socket containerd.service

[Service]
ExecStart=/usr/local/bin/kubelet \\
  --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig  \\
  --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \\
  --config=/etc/kubernetes/kubelet-conf.yml \\
  --container-runtime-endpoint=unix:///run/cri-dockerd.sock  \\
  --node-labels=node.kubernetes.io/node= 


[Install]
WantedBy=multi-user.target
EOF


cat > /etc/kubernetes/kubelet-conf.yml <<EOF
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 2m0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.pem
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 5m0s
    cacheUnauthorizedTTL: 30s
cgroupDriver: systemd
cgroupsPerQOS: true
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
containerLogMaxFiles: 5
containerLogMaxSize: 10Mi
contentType: application/vnd.kubernetes.protobuf
cpuCFSQuota: true
cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebuggingHandlers: true
enforceNodeAllocatable:
- pods
eventBurst: 10
eventRecordQPS: 5
evictionHard:
  imagefs.available: 15%
  memory.available: 100Mi
  nodefs.available: 10%
  nodefs.inodesFree: 5%
evictionPressureTransitionPeriod: 5m0s
failSwapOn: true
fileCheckFrequency: 20s
hairpinMode: promiscuous-bridge
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 20s
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMinimumGCAge: 2m0s
iptablesDropBit: 15
iptablesMasqueradeBit: 14
kubeAPIBurst: 10
kubeAPIQPS: 5
makeIPTablesUtilChains: true
maxOpenFiles: 1000000
maxPods: 110
nodeStatusUpdateFrequency: 10s
oomScoreAdj: -999
podPidsLimit: -1
registryBurst: 10
registryPullQPS: 5
resolvConf: /etc/resolv.conf
rotateCertificates: true
runtimeRequestTimeout: 2m0s
serializeImagePulls: true
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 4h0m0s
syncFrequency: 1m0s
volumeStatsAggPeriod: 1m0s
EOF

systemctl daemon-reload
systemctl enable --now kubelet.service
systemctl status kubelet.service

1.3 kube-proxy 配置（所有节点）

# 将 kubeconfig 发送至其他节点
# master-1 执行
# for NODE in k8s-master02 k8s-master03 k8s-node01 k8s-node02; do scp /etc/kubernetes/kube-proxy.kubeconfig $NODE:/etc/kubernetes/kube-proxy.kubeconfig; done
for NODE in server171 server172 server173 server174; do scp /etc/kubernetes/kube-proxy.kubeconfig $NODE:/etc/kubernetes/kube-proxy.kubeconfig; done

# 所有 k8s 节点添加 kube-proxy 的 service 文件
cat >  /usr/lib/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Kubernetes Kube Proxy
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-proxy \\
 --config=/etc/kubernetes/kube-proxy.yaml \\
 --cluster-cidr=172.16.0.0/12 \\
 --v=2
Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target

EOF


# 所有 k8s节点添加 kube-proxy 的配置
cat > /etc/kubernetes/kube-proxy.yaml << EOF
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
clientConnection:
  acceptContentTypes: ""
  burst: 10
  contentType: application/vnd.kubernetes.protobuf
  kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
  qps: 5
clusterCIDR: 172.16.0.0/12
configSyncPeriod: 15m0s
conntrack:
  max: null
  maxPerCore: 32768
  min: 131072
  tcpCloseWaitTimeout: 1h0m0s
  tcpEstablishedTimeout: 24h0m0s
enableProfiling: false
healthzBindAddress: 0.0.0.0:10256
hostnameOverride: ""
iptables:
  masqueradeAll: false
  masqueradeBit: 14
  minSyncPeriod: 0s
  syncPeriod: 30s
ipvs:
  masqueradeAll: true
  minSyncPeriod: 5s
  scheduler: "rr"
  syncPeriod: 30s
kind: KubeProxyConfiguration
metricsBindAddress: 127.0.0.1:10249
mode: "ipvs"
nodePortAddresses: null
oomScoreAdj: -999
portRange: ""
udpIdleTimeout: 250ms
EOF


systemctl daemon-reload
systemctl enable --now kube-proxy.service
systemctl status kube-proxy.service

2、安装网络插件

wget https://mirrors.chenby.cn/https://github.com/projectcalico/calico/blob/master/manifests/calico-typha.yaml

cp calico-typha.yaml calico.yaml

vim calico.yaml
# calico-config ConfigMap处
  "ipam": {
    "type": "calico-ipam",
  },
  - name: IP
    value: "autodetect"

  - name: CALICO_IPV4POOL_CIDR
    value: "172.16.0.0/12"


# 若docker镜像拉不下来，可以使用国内的仓库
sed -i "s#docker.io/calico/#m.daocloud.io/docker.io/calico/#g" calico.yaml 
sed -i "s#docker.io/calico/#m.daocloud.io/docker.io/calico/#g" calico-ipv6.yaml

sed -i "s#m.daocloud.io/docker.io/calico/#docker.io/calico/#g" calico.yaml 
sed -i "s#m.daocloud.io/docker.io/calico/#docker.io/calico/#g" calico-ipv6.yaml

# 本地没有公网 IPv6 使用 calico.yaml
kubectl apply -f calico.yaml

3、安装CoreDNS（只在 master01 操作）

# 安装 helm，官方脚本，保存执行
#!/usr/bin/env bash

# Copyright The Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# The install script is based off of the MIT-licensed script from glide,
# the package manager for Go: https://github.com/Masterminds/glide.sh/blob/master/get

: ${BINARY_NAME:="helm"}
: ${USE_SUDO:="true"}
: ${DEBUG:="false"}
: ${VERIFY_CHECKSUM:="true"}
: ${VERIFY_SIGNATURES:="false"}
: ${HELM_INSTALL_DIR:="/usr/local/bin"}
: ${GPG_PUBRING:="pubring.kbx"}

HAS_CURL="$(type "curl" &> /dev/null && echo true || echo false)"
HAS_WGET="$(type "wget" &> /dev/null && echo true || echo false)"
HAS_OPENSSL="$(type "openssl" &> /dev/null && echo true || echo false)"
HAS_GPG="$(type "gpg" &> /dev/null && echo true || echo false)"
HAS_GIT="$(type "git" &> /dev/null && echo true || echo false)"

# initArch discovers the architecture for this system.
initArch() {
 ARCH=$(uname -m)
 case $ARCH in
  armv5*) ARCH="armv5";;
  armv6*) ARCH="armv6";;
  armv7*) ARCH="arm";;
  aarch64) ARCH="arm64";;
  x86) ARCH="386";;
  x86_64) ARCH="amd64";;
  i686) ARCH="386";;
  i386) ARCH="386";;
 esac
}

# initOS discovers the operating system for this system.
initOS() {
 OS=$(echo `uname`|tr '[:upper:]' '[:lower:]')

 case "$OS" in
  # Minimalist GNU for Windows
  mingw*|cygwin*) OS='windows';;
 esac
}

# runs the given command as root (detects if we are root already)
runAsRoot() {
 if [ $EUID -ne 0 -a "$USE_SUDO" = "true" ]; then
  sudo "${@}"
 else
  "${@}"
 fi
}

# verifySupported checks that the os/arch combination is supported for
# binary builds, as well whether or not necessary tools are present.
verifySupported() {
 local supported="darwin-amd64\ndarwin-arm64\nlinux-386\nlinux-amd64\nlinux-arm\nlinux-arm64\nlinux-ppc64le\nlinux-s390x\nlinux-riscv64\nwindows-amd64"
 if ! echo "${supported}" | grep -q "${OS}-${ARCH}"; then
  echo "No prebuilt binary for ${OS}-${ARCH}."
  echo "To build from source, go to https://github.com/helm/helm"
  exit 1
 fi

 if [ "${HAS_CURL}" != "true" ] && [ "${HAS_WGET}" != "true" ]; then
  echo "Either curl or wget is required"
  exit 1
 fi

 if [ "${VERIFY_CHECKSUM}" == "true" ] && [ "${HAS_OPENSSL}" != "true" ]; then
  echo "In order to verify checksum, openssl must first be installed."
  echo "Please install openssl or set VERIFY_CHECKSUM=false in your environment."
  exit 1
 fi

 if [ "${VERIFY_SIGNATURES}" == "true" ]; then
  if [ "${HAS_GPG}" != "true" ]; then
   echo "In order to verify signatures, gpg must first be installed."
   echo "Please install gpg or set VERIFY_SIGNATURES=false in your environment."
   exit 1
  fi
  if [ "${OS}" != "linux" ]; then
   echo "Signature verification is currently only supported on Linux."
   echo "Please set VERIFY_SIGNATURES=false or verify the signatures manually."
   exit 1
  fi
 fi

 if [ "${HAS_GIT}" != "true" ]; then
  echo "[WARNING] Could not find git. It is required for plugin installation."
 fi
}

# checkDesiredVersion checks if the desired version is available.
checkDesiredVersion() {
 if [ "x$DESIRED_VERSION" == "x" ]; then
  # Get tag from release URL
  local latest_release_url="https://get.helm.sh/helm-latest-version"
  local latest_release_response=""
  if [ "${HAS_CURL}" == "true" ]; then
   latest_release_response=$( curl -L --silent --show-error --fail "$latest_release_url" 2>&1 || true )
  elif [ "${HAS_WGET}" == "true" ]; then
   latest_release_response=$( wget "$latest_release_url" -q -O - 2>&1 || true )
  fi
  TAG=$( echo "$latest_release_response" | grep '^v[0-9]' )
  if [ "x$TAG" == "x" ]; then
   printf "Could not retrieve the latest release tag information from %s: %s\n" "${latest_release_url}" "${latest_release_response}"
   exit 1
  fi
 else
  TAG=$DESIRED_VERSION
 fi
}

# checkHelmInstalledVersion checks which version of helm is installed and
# if it needs to be changed.
checkHelmInstalledVersion() {
 if [[ -f "${HELM_INSTALL_DIR}/${BINARY_NAME}" ]]; then
  local version=$("${HELM_INSTALL_DIR}/${BINARY_NAME}" version --template="{{ .Version }}")
  if [[ "$version" == "$TAG" ]]; then
   echo "Helm ${version} is already ${DESIRED_VERSION:-latest}"
   return 0
  else
   echo "Helm ${TAG} is available. Changing from version ${version}."
   return 1
  fi
 else
  return 1
 fi
}

# downloadFile downloads the latest binary package and also the checksum
# for that binary.
downloadFile() {
 HELM_DIST="helm-$TAG-$OS-$ARCH.tar.gz"
 DOWNLOAD_URL="https://get.helm.sh/$HELM_DIST"
 CHECKSUM_URL="$DOWNLOAD_URL.sha256"
 HELM_TMP_ROOT="$(mktemp -dt helm-installer-XXXXXX)"
 HELM_TMP_FILE="$HELM_TMP_ROOT/$HELM_DIST"
 HELM_SUM_FILE="$HELM_TMP_ROOT/$HELM_DIST.sha256"
 echo "Downloading $DOWNLOAD_URL"
 if [ "${HAS_CURL}" == "true" ]; then
  curl -SsL "$CHECKSUM_URL" -o "$HELM_SUM_FILE"
  curl -SsL "$DOWNLOAD_URL" -o "$HELM_TMP_FILE"
 elif [ "${HAS_WGET}" == "true" ]; then
  wget -q -O "$HELM_SUM_FILE" "$CHECKSUM_URL"
  wget -q -O "$HELM_TMP_FILE" "$DOWNLOAD_URL"
 fi
}

# verifyFile verifies the SHA256 checksum of the binary package
# and the GPG signatures for both the package and checksum file
# (depending on settings in environment).
verifyFile() {
 if [ "${VERIFY_CHECKSUM}" == "true" ]; then
  verifyChecksum
 fi
 if [ "${VERIFY_SIGNATURES}" == "true" ]; then
  verifySignatures
 fi
}

# installFile installs the Helm binary.
installFile() {
 HELM_TMP="$HELM_TMP_ROOT/$BINARY_NAME"
 mkdir -p "$HELM_TMP"
 tar xf "$HELM_TMP_FILE" -C "$HELM_TMP"
 HELM_TMP_BIN="$HELM_TMP/$OS-$ARCH/helm"
 echo "Preparing to install $BINARY_NAME into ${HELM_INSTALL_DIR}"
 runAsRoot cp "$HELM_TMP_BIN" "$HELM_INSTALL_DIR/$BINARY_NAME"
 echo "$BINARY_NAME installed into $HELM_INSTALL_DIR/$BINARY_NAME"
}

# verifyChecksum verifies the SHA256 checksum of the binary package.
verifyChecksum() {
 printf "Verifying checksum... "
 local sum=$(openssl sha1 -sha256 ${HELM_TMP_FILE} | awk '{print $2}')
 local expected_sum=$(cat ${HELM_SUM_FILE})
 if [ "$sum" != "$expected_sum" ]; then
  echo "SHA sum of ${HELM_TMP_FILE} does not match. Aborting."
  exit 1
 fi
 echo "Done."
}

# verifySignatures obtains the latest KEYS file from GitHub main branch
# as well as the signature .asc files from the specific GitHub release,
# then verifies that the release artifacts were signed by a maintainer's key.
verifySignatures() {
 printf "Verifying signatures... "
 local keys_filename="KEYS"
 local github_keys_url="https://raw.githubusercontent.com/helm/helm/main/${keys_filename}"
 if [ "${HAS_CURL}" == "true" ]; then
  curl -SsL "${github_keys_url}" -o "${HELM_TMP_ROOT}/${keys_filename}"
 elif [ "${HAS_WGET}" == "true" ]; then
  wget -q -O "${HELM_TMP_ROOT}/${keys_filename}" "${github_keys_url}"
 fi
 local gpg_keyring="${HELM_TMP_ROOT}/keyring.gpg"
 local gpg_homedir="${HELM_TMP_ROOT}/gnupg"
 mkdir -p -m 0700 "${gpg_homedir}"
 local gpg_stderr_device="/dev/null"
 if [ "${DEBUG}" == "true" ]; then
  gpg_stderr_device="/dev/stderr"
 fi
 gpg --batch --quiet --homedir="${gpg_homedir}" --import "${HELM_TMP_ROOT}/${keys_filename}" 2> "${gpg_stderr_device}"
 gpg --batch --no-default-keyring --keyring "${gpg_homedir}/${GPG_PUBRING}" --export > "${gpg_keyring}"
 local github_release_url="https://github.com/helm/helm/releases/download/${TAG}"
 if [ "${HAS_CURL}" == "true" ]; then
  curl -SsL "${github_release_url}/helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256.asc" -o "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256.asc"
  curl -SsL "${github_release_url}/helm-${TAG}-${OS}-${ARCH}.tar.gz.asc" -o "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.asc"
 elif [ "${HAS_WGET}" == "true" ]; then
  wget -q -O "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256.asc" "${github_release_url}/helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256.asc"
  wget -q -O "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.asc" "${github_release_url}/helm-${TAG}-${OS}-${ARCH}.tar.gz.asc"
 fi
 local error_text="If you think this might be a potential security issue,"
 error_text="${error_text}\nplease see here: https://github.com/helm/community/blob/master/SECURITY.md"
 local num_goodlines_sha=$(gpg --verify --keyring="${gpg_keyring}" --status-fd=1 "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256.asc" 2> "${gpg_stderr_device}" | grep -c -E '^\[GNUPG:\] (GOODSIG|VALIDSIG)')
 if [[ ${num_goodlines_sha} -lt 2 ]]; then
  echo "Unable to verify the signature of helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256!"
  echo -e "${error_text}"
  exit 1
 fi
 local num_goodlines_tar=$(gpg --verify --keyring="${gpg_keyring}" --status-fd=1 "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.asc" 2> "${gpg_stderr_device}" | grep -c -E '^\[GNUPG:\] (GOODSIG|VALIDSIG)')
 if [[ ${num_goodlines_tar} -lt 2 ]]; then
  echo "Unable to verify the signature of helm-${TAG}-${OS}-${ARCH}.tar.gz!"
  echo -e "${error_text}"
  exit 1
 fi
 echo "Done."
}

# fail_trap is executed if an error occurs.
fail_trap() {
 result=$?
 if [ "$result" != "0" ]; then
  if [[ -n "$INPUT_ARGUMENTS" ]]; then
   echo "Failed to install $BINARY_NAME with the arguments provided: $INPUT_ARGUMENTS"
   help
  else
   echo "Failed to install $BINARY_NAME"
  fi
  echo -e "\tFor support, go to https://github.com/helm/helm."
 fi
 cleanup
 exit $result
}

# testVersion tests the installed client to make sure it is working.
testVersion() {
 set +e
 HELM="$(command -v $BINARY_NAME)"
 if [ "$?" = "1" ]; then
  echo "$BINARY_NAME not found. Is $HELM_INSTALL_DIR on your "'$PATH?'
  exit 1
 fi
 set -e
}

# help provides possible cli installation arguments
help () {
 echo "Accepted cli arguments are:"
 echo -e "\t[--help|-h ] ->> prints this help"
 echo -e "\t[--version|-v <desired_version>] . When not defined it fetches the latest release from GitHub"
 echo -e "\te.g. --version v3.0.0 or -v canary"
 echo -e "\t[--no-sudo]  ->> install without sudo"
}

# cleanup temporary files to avoid https://github.com/helm/helm/issues/2977
cleanup() {
 if [[ -d "${HELM_TMP_ROOT:-}" ]]; then
  rm -rf "$HELM_TMP_ROOT"
 fi
}

# Execution

#Stop execution on any error
trap "fail_trap" EXIT
set -e

# Set debug if desired
if [ "${DEBUG}" == "true" ]; then
 set -x
fi

# Parsing input arguments (if any)
export INPUT_ARGUMENTS="${@}"
set -u
while [[ $# -gt 0 ]]; do
 case $1 in
  '--version'|-v)
    shift
    if [[ $# -ne 0 ]]; then
      export DESIRED_VERSION="${1}"
      if [[ "$1" != "v"* ]]; then
        echo "Expected version arg ('${DESIRED_VERSION}') to begin with 'v', fixing..."
        export DESIRED_VERSION="v${1}"
      fi
    else
      echo -e "Please provide the desired version. e.g. --version v3.0.0 or -v canary"
      exit 0
    fi
    ;;
  '--no-sudo')
    USE_SUDO="false"
    ;;
  '--help'|-h)
    help
    exit 0
    ;;
  *) exit 1
    ;;
 esac
 shift
done
set +u

initArch
initOS
verifySupported
checkDesiredVersion
if ! checkHelmInstalledVersion; then
 downloadFile
 verifyFile
 installFile
fi
testVersion
cleanup

# 下载tgz包
helm repo add coredns https://coredns.github.io/helm
helm pull coredns/coredns
tar xvf coredns-*.tgz
cd coredns/

# 修改IP地址
vim values.yaml
cat values.yaml | grep clusterIP:
clusterIP: "10.96.0.10"

# 示例
---
service:
# clusterIP: ""
# clusterIPs: []
# loadBalancerIP: ""
# externalIPs: []
# externalTrafficPolicy: ""
# ipFamilyPolicy: ""
 # The name of the Service
 # If not set, a name is generated using the fullname template
 clusterIP: "10.96.0.10"
 name: ""
 annotations: {}
---

# 修改为国内源 docker源可选
sed -i "s#coredns/#m.daocloud.io/docker.io/coredns/#g" values.yaml
sed -i "s#registry.k8s.io/#m.daocloud.io/registry.k8s.io/#g" values.yaml

# 默认参数安装
helm install  coredns ./coredns/ -n kube-system

4、安装 Metrics Server

# 下载 
wget https://mirrors.chenby.cn/https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml

# 修改配置
vim components.yaml

---
# 1
   - args:
    - --cert-dir=/tmp
    - --secure-port=10250
    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
    - --kubelet-use-node-status-port
    - --metric-resolution=15s
    - --kubelet-insecure-tls
    - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem
    - --requestheader-username-headers=X-Remote-User
    - --requestheader-group-headers=X-Remote-Group
    - --requestheader-extra-headers-prefix=X-Remote-Extra-


# 2
    volumeMounts:
    - mountPath: /tmp
     name: tmp-dir
    - name: ca-ssl
     mountPath: /etc/kubernetes/pki

# 3
   volumes:
   - emptyDir: {}
    name: tmp-dir
   - name: ca-ssl
    hostPath:
     path: /etc/kubernetes/pki
---


# 修改为国内源 docker源可选
sed -i "s#registry.k8s.io/#m.daocloud.io/registry.k8s.io/#g" *.yaml

# 执行部署
kubectl apply -f components.yaml