# --- 10.10.10.121 ---
sudo hostnamectl set-hostname iaas-1

# --- 10.10.10.122 ---
sudo hostnamectl set-hostname iaas-2

# --- 10.10.10.123 ---
sudo hostnamectl set-hostname iaas-3

cat << EOF | sudo tee -a /etc/hosts
10.10.10.198 k8s-vip
10.10.10.121 iaas-1
10.10.10.122 iaas-2
10.10.10.123 iaas-3
EOF

# 关闭 swap
sudo swapoff -a
sudo sed -i '/swap/d' /etc/fstab

# 加载内核模块
cat << EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF
sudo modprobe overlay
sudo modprobe br_netfilter

# KubeVirt 虚拟化模块
sudo modprobe vhost_net
echo "vhost_net" | sudo tee -a /etc/modules-load.d/kubevirt.conf

# 内核参数
cat << EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF
sudo sysctl --system

egrep -c '(vmx|svm)' /proc/cpuinfo
# 大于 0 说明支持

ls /dev/kvm
# 如果不存在：
sudo apt-get install -y qemu-kvm
sudo modprobe kvm
sudo modprobe kvm_intel  # Intel CPU

sudo apt-get update
sudo apt-get install -y containerd

sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml

sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' \
  /etc/containerd/config.toml

sudo systemctl restart containerd
sudo systemctl enable containerd

sudo apt-get install -y apt-transport-https ca-certificates curl gpg

curl -fsSL https://mirrors.aliyun.com/kubernetes-new/core/\
stable/v1.28/deb/Release.key | \
  sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg

echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] \
  https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.28/deb/ \
  /" | sudo tee /etc/apt/sources.list.d/kubernetes.list

sudo apt-get update
sudo apt-get install -y kubelet=1.28.2-1.1 kubeadm=1.28.2-1.1 \
  kubectl=1.28.2-1.1
sudo apt-mark hold kubelet kubeadm kubectl

ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa
ssh-copy-id iaas-2
ssh-copy-id iaas-3

sudo apt-get install -y haproxy keepalived psmisc

cat << 'EOF' | sudo tee /etc/haproxy/haproxy.cfg
global
    log /dev/log local0
    maxconn 4096
    daemon

defaults
    mode tcp
    timeout connect 5s
    timeout client  30s
    timeout server  30s

frontend k8s-api
    bind *:8443
    default_backend k8s-masters

backend k8s-masters
    balance roundrobin
    option tcp-check
    server iaas-1 10.10.10.121:6443 check inter 3s fall 3 rise 2
    server iaas-2 10.10.10.122:6443 check inter 3s fall 3 rise 2
    server iaas-3 10.10.10.123:6443 check inter 3s fall 3 rise 2

listen stats
    bind *:9090
    mode http
    stats enable
    stats uri /stats
EOF

sudo systemctl restart haproxy
sudo systemctl enable haproxy

cat << 'EOF' | sudo tee /etc/keepalived/keepalived.conf
global_defs {
    router_id LVS_IAAS
}

vrrp_script check_haproxy {
    script "/usr/bin/killall -0 haproxy"
    interval 3
    weight -2
    fall 10
    rise 2
}

vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 52
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass IaaS2026
    }
    virtual_ipaddress {
        10.10.10.198/24
    }
    track_script {
        check_haproxy
    }
}
EOF

sudo systemctl restart keepalived
sudo systemctl enable keepalived

ping -c 3 10.10.10.198
ip addr show | grep 10.10.10.198

sudo kubeadm init \
  --kubernetes-version=v1.28.2 \
  --control-plane-endpoint='k8s-vip:8443' \
  --apiserver-advertise-address=10.10.10.121 \
  --upload-certs \
  --pod-network-cidr=10.244.0.0/16 \
  --service-cidr=10.96.0.0/12 \
  --image-repository=registry.aliyuncs.com/google_containers

mkdir -p $HOME/.kube
sudo cp /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# iaas-2
sudo kubeadm join k8s-vip:8443 --token <token> \
  --discovery-token-ca-cert-hash sha256:<hash> \
  --control-plane --certificate-key <cert-key> \
  --apiserver-advertise-address=10.10.10.122

# iaas-3
sudo kubeadm join k8s-vip:8443 --token <token> \
  --discovery-token-ca-cert-hash sha256:<hash> \
  --control-plane --certificate-key <cert-key> \
  --apiserver-advertise-address=10.10.10.123

# 两台都配置 kubectl
mkdir -p $HOME/.kube
sudo cp /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

kubectl taint nodes --all node-role.kubernetes.io/control-plane-

kubectl apply -f https://raw.githubusercontent.com/projectcalico/\
  calico/v3.26.4/manifests/calico.yaml

kubectl set env daemonset/calico-node -n kube-system \
  IP_AUTODETECTION_METHOD=cidr=10.10.10.0/24

watch kubectl get nodes

curl https://mirrors.huaweicloud.com/helm/v3.13.3/\
helm-v3.13.3-linux-amd64.tar.gz -o helm.tar.gz
tar -zxvf helm.tar.gz
sudo mv linux-amd64/helm /usr/local/bin/helm
helm version

for node in iaas-1 iaas-2 iaas-3; do
  ssh $node 'sudo apt-get install -y open-iscsi nfs-common'
  ssh $node 'sudo systemctl enable iscsid && sudo systemctl start iscsid'
done

helm repo add longhorn https://charts.longhorn.io
helm repo update

helm install longhorn longhorn/longhorn \
  --namespace longhorn-system --create-namespace \
  --set defaultSettings.defaultReplicaCount=2 \
  --set defaultSettings.defaultDataLocality=best-effort \
  --set longhornUI.replicas=1

kubectl patch storageclass longhorn -p \
  '{"metadata":{"annotations":{\
  "storageclass.kubernetes.io/is-default-class":"true"}}}'

kubectl -n longhorn-system get pods -w

kubectl -n longhorn-system patch svc longhorn-frontend -p \
  '{"spec":{"type":"NodePort","ports":[{\
  "port":80,"targetPort":8000,"nodePort":30800}]}}'

export KUBEVIRT_VERSION=v1.3.1

kubectl create -f https://github.com/kubevirt/kubevirt/releases/\
download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml

kubectl create -f https://github.com/kubevirt/kubevirt/releases/\
download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml

# 如果没有硬件虚拟化（嵌套虚拟化未开启），启用软件模拟
kubectl -n kubevirt patch kubevirt kubevirt --type=merge -p \
  '{"spec":{"configuration":{"developerConfiguration":{\
  "useEmulation":true}}}}'

kubectl -n kubevirt wait kv kubevirt \
  --for condition=Available --timeout=300s

curl -L -o /usr/local/bin/virtctl \
  https://github.com/kubevirt/kubevirt/releases/download/\
${KUBEVIRT_VERSION}/virtctl-${KUBEVIRT_VERSION}-linux-amd64

chmod +x /usr/local/bin/virtctl

export CDI_VERSION=v1.59.0

kubectl create -f https://github.com/kubevirt/\
containerized-data-importer/releases/download/\
${CDI_VERSION}/cdi-operator.yaml

kubectl create -f https://github.com/kubevirt/\
containerized-data-importer/releases/download/\
${CDI_VERSION}/cdi-cr.yaml

# 验证虚拟机
kubectl -n tenant-a get vm
kubectl -n tenant-a get vmi

# 进入控制台
virtctl -n tenant-a console vm-01

# 生命周期管理
virtctl -n tenant-a start vm-01
virtctl -n tenant-a stop vm-01
virtctl -n tenant-a restart vm-01
virtctl -n tenant-a migrate vm-01

kubectl delete -f https://raw.githubusercontent.com/projectcalico/\
  calico/v3.26.4/manifests/calico.yaml

wget https://raw.githubusercontent.com/kubeovn/kube-ovn/\
release-1.12/dist/images/install.sh

sed -i 's/MASTER_NODES=""/MASTER_NODES="iaas-1,iaas-2,iaas-3"/' install.sh
sed -i 's/POD_CIDR="10.16.0.0\/16"/POD_CIDR="10.244.0.0\/16"/' install.sh
sed -i 's/SVC_CIDR="10.96.0.0\/12"/SVC_CIDR="10.96.0.0\/12"/' install.sh

bash install.sh

kubectl -n kube-system get pods | grep kube-ovn

kubectl get vpc
kubectl get subnet

curl -sL https://raw.githubusercontent.com/karmada-io/karmada/\
master/hack/install-cli.sh | bash

karmadactl version

# 方式1：karmadactl init（推荐）
sudo karmadactl init \
  --karmada-apiserver-advertise-address=10.10.10.121 \
  --etcd-storage-mode=hostPath \
  --karmada-data=/var/lib/karmada

# 方式2：Helm 安装
helm repo add karmada https://raw.githubusercontent.com/\
karmada-io/karmada/master/charts
helm repo update

helm install karmada karmada/karmada \
  --namespace karmada-system --create-namespace \
  --set apiServer.hostNetwork=true \
  --set apiServer.serviceType=NodePort

# 验证
kubectl --kubeconfig /etc/karmada/karmada-apiserver.config \
  get clusters

# 设置 alias 方便操作
echo 'alias kctl="kubectl --kubeconfig /etc/karmada/karmada-apiserver.config"' \
  >> ~/.bashrc
source ~/.bashrc

karmadactl --kubeconfig /etc/karmada/karmada-apiserver.config \
  join iaas-cluster \
  --cluster-kubeconfig=$HOME/.kube/config \
  --cluster-context=kubernetes-admin@kubernetes

# 验证
kctl get clusters
# 预期看到：iaas-cluster   Ready

# 前提：已在阿里云创建 ACK 集群，下载 kubeconfig 到本地
# 保存为 ~/.kube/aliyun-ack.config

karmadactl --kubeconfig /etc/karmada/karmada-apiserver.config \
  join aliyun-cluster \
  --cluster-kubeconfig=~/.kube/aliyun-ack.config

kctl get clusters
# 预期：iaas-cluster   Ready
#       aliyun-cluster Ready

# 前提：已在华为云创建 CCE 集群，下载 kubeconfig
# 保存为 ~/.kube/huawei-cce.config

karmadactl --kubeconfig /etc/karmada/karmada-apiserver.config \
  join huawei-cluster \
  --cluster-kubeconfig=~/.kube/huawei-cce.config

kctl get clusters
# 预期：iaas-cluster    Ready
#       aliyun-cluster  Ready
#       huawei-cluster  Ready

curl -L https://github.com/kubernetes-sigs/cluster-api/releases/\
latest/download/clusterctl-linux-amd64 -o /usr/local/bin/clusterctl

chmod +x /usr/local/bin/clusterctl
clusterctl version

# 初始化核心组件
clusterctl init

# 安装各云 Provider（按需选择）
# OpenStack
clusterctl init --infrastructure=openstack

# AWS
clusterctl init --infrastructure=aws

# Azure
clusterctl init --infrastructure=azure

# 查看集群状态
kubectl get clusters
clusterctl describe cluster openstack-prod-cluster

# 创建完成后自动注册到 Karmada
karmadactl join openstack-cluster \
  --kubeconfig /etc/karmada/karmada-apiserver.config \
  --cluster-kubeconfig=<generated-kubeconfig>

curl -Ls https://get.submariner.io | VERSION=v0.17.0 bash

subctl version

subctl deploy-broker --kubeconfig $HOME/.kube/config

# 会生成 broker-info.subm 文件
ls broker-info.subm

# 自建集群加入
subctl join --kubeconfig $HOME/.kube/config \
  broker-info.subm \
  --clusterid iaas-cluster \
  --natt=false

# 阿里云集群加入
subctl join --kubeconfig ~/.kube/aliyun-ack.config \
  broker-info.subm \
  --clusterid aliyun-cluster

# 华为云集群加入
subctl join --kubeconfig ~/.kube/huawei-cce.config \
  broker-info.subm \
  --clusterid huawei-cluster

# 查看所有连接
subctl show all

# 验证跨集群 DNS
# 在自建集群的 Pod 里可以直接访问阿里云集群的 Service：
# curl http://my-service.my-namespace.svc.clusterset.local

# 跨集群连通性测试
subctl verify --kubeconfig $HOME/.kube/config \
  --toconfig ~/.kube/aliyun-ack.config \
  --only connectivity

helm repo add harbor https://helm.goharbor.io
helm repo update

helm install harbor harbor/harbor \
  --namespace harbor --create-namespace \
  --set expose.type=nodePort \
  --set expose.nodePort.ports.http.nodePort=30003 \
  --set externalURL=http://10.10.10.121:30003 \
  --set persistence.persistentVolumeClaim.registry.size=50Gi \
  --set persistence.persistentVolumeClaim.database.size=5Gi \
  --set persistence.persistentVolumeClaim.redis.size=2Gi \
  --set harborAdminPassword=Harbor@2026

kubectl -n harbor get pods -w

helm install gatekeeper gatekeeper/gatekeeper \
  --namespace gatekeeper-system --create-namespace \
  --set replicas=1

helm repo add falcosecurity https://falcosecurity.github.io/charts
helm install falco falcosecurity/falco \
  --namespace falco --create-namespace \
  --set falcosidekick.enabled=true

helm repo add jetstack https://charts.jetstack.io
helm install cert-manager jetstack/cert-manager \
  --namespace cert-manager --create-namespace \
  --set installCRDs=true \
  --set replicas=1

helm repo add prometheus-community \
  https://prometheus-community.github.io/helm-charts
helm repo update

helm install monitoring \
  prometheus-community/kube-prometheus-stack \
  --namespace monitoring --create-namespace \
  --set grafana.adminPassword=admin2026 \
  --set grafana.service.type=NodePort \
  --set grafana.service.nodePort=31301 \
  --set prometheus.service.type=NodePort \
  --set prometheus.service.nodePort=31090 \
  --set prometheus.prometheusSpec.retention=3d \
  --set prometheus.prometheusSpec.resources.requests.memory=512Mi

helm repo add bitnami https://charts.bitnami.com/bitnami

# 在 iaas-1 部署 Thanos Query（聚合多个集群的 Prometheus）
helm install thanos bitnami/thanos \
  --namespace monitoring \
  --set query.enabled=true \
  --set query.service.type=NodePort \
  --set query.service.nodePort=31191 \
  --set query.stores[0]=prometheus-kube-prometheus-prometheus.monitoring.svc:10901 \
  --set queryFrontend.enabled=true \
  --set compactor.enabled=false \
  --set storegateway.enabled=false

# 每个成员集群的 Prometheus 开启 Sidecar
# 在 Helm values 中添加：
# prometheus.prometheusSpec.thanos:
#   objectStorageConfig:
#     name: thanos-objstore-secret
#     key: objstore.yml

# Thanos Query 自动聚合所有集群的指标
# 在 Thanos Query UI 中可以：
# - 查看所有集群的统一视图
# - 跨集群 PromQL 查询

helm install emissary datawire/emissary-ingress \
  --namespace emissary --create-namespace \
  --set replicaCount=1 \
  --set service.type=NodePort \
  --set service.nodePorts.http=30080

# 模板
sudo ctr -n k8s.io image pull \
  swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/<镜像名>:<tag>
sudo ctr -n k8s.io image tag \
  swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/<镜像名>:<tag> \
  docker.io/<镜像名>:<tag>

# Mac 上（有 VPN）
docker pull <镜像名>:<tag>
docker save <镜像名>:<tag> -o image.tar
scp image.tar root@10.10.10.121:~/

# 节点上导入
sudo ctr -n k8s.io image import ~/image.tar

#!/bin/bash
# sync-images.sh — 在 Mac 上执行
IMAGES=(
  "longhornio/longhorn-manager:v1.6.2"
  "longhornio/longhorn-engine:v1.6.2"
  "quay.io/kubevirt/virt-operator:v1.3.1"
  "quay.io/kubevirt/virt-api:v1.3.1"
  "quay.io/kubevirt/virt-controller:v1.3.1"
  "quay.io/kubevirt/virt-handler:v1.3.1"
  "quay.io/kubevirt/virt-launcher:v1.3.1"
)

for img in "${IMAGES[@]}"; do
  docker pull $img
done

docker save ${IMAGES[@]} -o all-images.tar

for node in 10.10.10.121 10.10.10.122 10.10.10.123; do
  scp all-images.tar root@$node:~/
  ssh root@$node 'sudo ctr -n k8s.io image import ~/all-images.tar'
done