Setup K8S Cluster on KVM Host using Bash

#!/bin/bash

# k8s-cluster-deploy-automated.sh
# Deploys Kubernetes cluster with automatic password authentication

set -e

# Configuration
CLUSTER_NAME="k8s-cluster"
MASTER_NAME="${CLUSTER_NAME}-master"
WORKER1_NAME="${CLUSTER_NAME}-worker1"
WORKER2_NAME="${CLUSTER_NAME}-worker2"
POD_CIDR="10.244.0.0/16"

# VM Configuration
MASTER_MEMORY=2048
WORKER_MEMORY=1536
MASTER_CPUS=2
WORKER_CPUS=1
DISK_SIZE=20

# IP Configuration
MASTER_IP="192.168.122.10"
WORKER1_IP="192.168.122.11"
WORKER2_IP="192.168.122.12"

# Credentials
SSH_USER="k8suser"
SSH_PASSWORD="k8suser123"

# Create working directory
WORK_DIR="/tmp/k8s-deploy-$(date +%s)"
mkdir -p ${WORK_DIR}
cd ${WORK_DIR}

echo "=== Creating Kubernetes Cluster with 1 Master + 2 Workers ==="
echo "Working directory: ${WORK_DIR}"

# Install sshpass if not installed
if ! command -v sshpass &> /dev/null; then
    echo "Installing sshpass for automatic SSH authentication..."
    sudo apt-get update
    sudo apt-get install -y sshpass
fi

# Start libvirtd if needed
if ! systemctl is-active --quiet libvirtd; then
    echo "Starting libvirtd service..."
    sudo systemctl start libvirtd
    sudo systemctl enable libvirtd
    sleep 3
fi

# Ensure default network is active
if ! virsh net-list --all | grep -q default; then
    sudo virsh net-define /usr/share/libvirt/networks/default.xml
    sudo virsh net-start default
    sudo virsh net-autostart default
elif ! virsh net-list --all | grep -q "default.*active"; then
    sudo virsh net-start default
    sudo virsh net-autostart default
fi

# Download Ubuntu cloud image if not exists
IMAGE_NAME="noble-server-cloudimg-amd64.img"
IMAGE_PATH="/var/lib/libvirt/images/${IMAGE_NAME}"
if [ ! -f ${IMAGE_PATH} ]; then
    echo "Downloading Ubuntu Noble cloud image..."
    sudo wget -O ${IMAGE_PATH} \
        https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
    sudo chmod 644 ${IMAGE_PATH}
fi

# Function to create VM
create_vm() {
    local VM_NAME=$1
    local MEMORY=$2
    local CPUS=$3
    local IP_ADDR=$4

    echo "Creating VM: ${VM_NAME} with IP: ${IP_ADDR}"

    # Remove existing VM if present
    if virsh list --all | grep -q ${VM_NAME}; then
        echo "Removing existing VM: ${VM_NAME}"
        virsh destroy ${VM_NAME} 2>/dev/null || true
        virsh undefine ${VM_NAME} --remove-all-storage --nvram 2>/dev/null || true
        sleep 2
    fi

    # Create cloud-init config files
    cat >${WORK_DIR}/${VM_NAME}-network-config <<EOF
version: 2
ethernets:
  enp1s0:
    dhcp4: false
    addresses:
      - ${IP_ADDR}/24
    gateway4: 192.168.122.1
    nameservers:
      addresses: [8.8.8.8, 8.8.4.4]
EOF

    cat >${WORK_DIR}/${VM_NAME}-meta-data <<EOF
instance-id: ${VM_NAME}
local-hostname: ${VM_NAME}
EOF

    cat >${WORK_DIR}/${VM_NAME}-user-data <<EOF
#cloud-config
hostname: ${VM_NAME}
fqdn: ${VM_NAME}.local
manage_etc_hosts: true
password: ${SSH_PASSWORD}
chpasswd:
  expire: False
ssh_pwauth: True
users:
  - name: ${SSH_USER}
    sudo: ALL=(ALL) NOPASSWD:ALL
    shell: /bin/bash
    lock_passwd: false
    passwd: $(openssl passwd -6 ${SSH_PASSWORD})
packages:
  - qemu-guest-agent
  - curl
  - wget
  - vim
  - net-tools
  - ca-certificates
  - openssh-server
runcmd:
  - systemctl enable qemu-guest-agent
  - systemctl start qemu-guest-agent
  - systemctl enable ssh
  - systemctl start ssh
  - echo "${SSH_USER} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
EOF

    # Create backing disk
    sudo qemu-img create -f qcow2 -b ${IMAGE_PATH} -F qcow2 ${WORK_DIR}/${VM_NAME}.qcow2 ${DISK_SIZE}G

    # Create VM
    sudo virt-install \
        --name ${VM_NAME} \
        --memory ${MEMORY} \
        --vcpus ${CPUS} \
        --disk path=${WORK_DIR}/${VM_NAME}.qcow2,format=qcow2,bus=virtio \
        --os-variant detect=on,name=ubuntunoble \
        --network network=default,model=virtio \
        --cloud-init user-data=${WORK_DIR}/${VM_NAME}-user-data,meta-data=${WORK_DIR}/${VM_NAME}-meta-data,network-config=${WORK_DIR}/${VM_NAME}-network-config \
        --import \
        --noautoconsole \
        --wait=0 \
        --graphics none

    echo "VM ${VM_NAME} creation initiated"
}

# Create all VMs
echo "Creating Master node..."
create_vm ${MASTER_NAME} ${MASTER_MEMORY} ${MASTER_CPUS} ${MASTER_IP}

echo "Creating Worker1 node..."
create_vm ${WORKER1_NAME} ${WORKER_MEMORY} ${WORKER_CPUS} ${WORKER1_IP}

echo "Creating Worker2 node..."
create_vm ${WORKER2_NAME} ${WORKER_MEMORY} ${WORKER_CPUS} ${WORKER2_IP}

# Wait for VMs to be created and boot
echo "Waiting for VMs to be created and boot (90 seconds)..."
sleep 90

# Function to check if VM is ready
check_vm_ready() {
    local VM_NAME=$1
    if virsh list --all | grep -q "${VM_NAME}.*running"; then
        return 0
    else
        return 1
    fi
}

# Wait for each VM to be running
for VM in ${MASTER_NAME} ${WORKER1_NAME} ${WORKER2_NAME}; do
    echo "Checking ${VM} status..."
    for i in {1..30}; do
        if check_vm_ready ${VM}; then
            echo "${VM} is running"
            break
        fi
        echo "Waiting for ${VM} to start... (${i}/30)"
        sleep 5
    done
done

# Function to wait for SSH using sshpass
wait_for_ssh() {
    local IP=$1
    local NAME=$2
    local MAX_RETRIES=30
    local RETRY=0

    echo "Waiting for SSH on ${NAME} (${IP})..."
    while [ $RETRY -lt $MAX_RETRIES ]; do
        if sshpass -p "${SSH_PASSWORD}" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
            -o PasswordAuthentication=yes -o BatchMode=no ${SSH_USER}@${IP} "exit" 2>/dev/null; then
            echo "SSH ready on ${NAME} (${IP})"
            return 0
        fi
        echo "Attempt $((RETRY+1))/$MAX_RETRIES - SSH not ready yet on ${IP}..."
        sleep 10
        RETRY=$((RETRY + 1))
    done
    echo "Failed to connect to ${NAME} (${IP})"
    return 1
}

# Wait for SSH on all VMs using IPs
echo "Waiting for SSH to become available on all VMs..."
wait_for_ssh ${MASTER_IP} "Master"
wait_for_ssh ${WORKER1_IP} "Worker1"
wait_for_ssh ${WORKER2_IP} "Worker2"

# Setup Kubernetes Master
echo "=== Setting up Kubernetes Master ==="
cat >${WORK_DIR}/setup-master.sh <<'EOF'
#!/bin/bash

# Disable swap
sudo swapoff -a
sudo sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

# Enable kernel modules
cat <<MODULES | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
MODULES

sudo modprobe overlay
sudo modprobe br_netfilter

# Configure sysctl
cat <<SYSCTL | sudo tee /etc/sysctl.d/99-kubernetes.conf
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
SYSCTL

sudo sysctl --system

# Install containerd
sudo apt-get update
sudo apt-get install -y containerd
sudo mkdir -p /etc/containerd
sudo containerd config default | sudo tee /etc/containerd/config.toml
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
sudo systemctl restart containerd
sudo systemctl enable containerd

# Install Kubernetes components
sudo apt-get update
sudo apt-get install -y apt-transport-https ca-certificates curl gnupg
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.30/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.30/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list

sudo apt-get update
sudo apt-get install -y kubelet=1.30.0-1.1 kubeadm=1.30.0-1.1 kubectl=1.30.0-1.1
sudo apt-mark hold kubelet kubeadm kubectl

# Initialize cluster with specific IP
sudo kubeadm init --pod-network-cidr=10.244.0.0/16 --apiserver-advertise-address=${MASTER_IP}

# Configure kubectl for regular user
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# Install Flannel CNI
kubectl apply -f https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml

# Get join command
sudo kubeadm token create --print-join-command > /tmp/join-command.txt

echo "Master setup complete!"
EOF

# Replace MASTER_IP in setup script
sed -i "s/\${MASTER_IP}/${MASTER_IP}/g" ${WORK_DIR}/setup-master.sh

# Copy and execute master setup with sshpass
echo "Copying setup script to master..."
sshpass -p "${SSH_PASSWORD}" scp -o StrictHostKeyChecking=no ${WORK_DIR}/setup-master.sh ${SSH_USER}@${MASTER_IP}:~/
echo "Running Kubernetes setup on master (this may take 5-10 minutes)..."
sshpass -p "${SSH_PASSWORD}" ssh -o StrictHostKeyChecking=no ${SSH_USER}@${MASTER_IP} "chmod +x ~/setup-master.sh && ~/setup-master.sh"

# Get join command from master
echo "Retrieving join command..."
JOIN_COMMAND=$(sshpass -p "${SSH_PASSWORD}" ssh -o StrictHostKeyChecking=no ${SSH_USER}@${MASTER_IP} "sudo cat /tmp/join-command.txt")
echo "Join command retrieved"

# Setup Workers
echo "=== Setting up Worker nodes ==="
cat >${WORK_DIR}/setup-worker.sh <<'EOF'
#!/bin/bash

# Disable swap
sudo swapoff -a
sudo sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

# Enable kernel modules
cat <<MODULES | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
MODULES

sudo modprobe overlay
sudo modprobe br_netfilter

# Configure sysctl
cat <<SYSCTL | sudo tee /etc/sysctl.d/99-kubernetes.conf
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
SYSCTL

sudo sysctl --system

# Install containerd
sudo apt-get update
sudo apt-get install -y containerd
sudo mkdir -p /etc/containerd
sudo containerd config default | sudo tee /etc/containerd/config.toml
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
sudo systemctl restart containerd
sudo systemctl enable containerd

# Install Kubernetes components
sudo apt-get update
sudo apt-get install -y apt-transport-https ca-certificates curl gnupg
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.30/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.30/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list

sudo apt-get update
sudo apt-get install -y kubelet=1.30.0-1.1 kubeadm=1.30.0-1.1
sudo apt-mark hold kubelet kubeadm

echo "Worker setup complete!"
EOF

# Setup worker1
echo "Setting up worker1..."
sshpass -p "${SSH_PASSWORD}" scp -o StrictHostKeyChecking=no ${WORK_DIR}/setup-worker.sh ${SSH_USER}@${WORKER1_IP}:~/
sshpass -p "${SSH_PASSWORD}" ssh -o StrictHostKeyChecking=no ${SSH_USER}@${WORKER1_IP} "chmod +x ~/setup-worker.sh && ~/setup-worker.sh"
echo "Joining worker1 to cluster..."
sshpass -p "${SSH_PASSWORD}" ssh -o StrictHostKeyChecking=no ${SSH_USER}@${WORKER1_IP} "sudo ${JOIN_COMMAND}"

# Setup worker2
echo "Setting up worker2..."
sshpass -p "${SSH_PASSWORD}" scp -o StrictHostKeyChecking=no ${WORK_DIR}/setup-worker.sh ${SSH_USER}@${WORKER2_IP}:~/
sshpass -p "${SSH_PASSWORD}" ssh -o StrictHostKeyChecking=no ${SSH_USER}@${WORKER2_IP} "chmod +x ~/setup-worker.sh && ~/setup-worker.sh"
echo "Joining worker2 to cluster..."
sshpass -p "${SSH_PASSWORD}" ssh -o StrictHostKeyChecking=no ${SSH_USER}@${WORKER2_IP} "sudo ${JOIN_COMMAND}"

# Verify cluster
echo "=== Verifying Kubernetes Cluster ==="
echo "Waiting for nodes to be ready (90 seconds)..."
sleep 90

echo ""
echo "Cluster Nodes:"
sshpass -p "${SSH_PASSWORD}" ssh -o StrictHostKeyChecking=no ${SSH_USER}@${MASTER_IP} "kubectl get nodes -o wide"
echo ""
echo "All Pods:"
sshpass -p "${SSH_PASSWORD}" ssh -o StrictHostKeyChecking=no ${SSH_USER}@${MASTER_IP} "kubectl get pods --all-namespaces"

echo ""
echo "=== Deployment Complete! ==="
echo ""
echo "Master IP: ${MASTER_IP}"
echo "Worker1 IP: ${WORKER1_IP}"
echo "Worker2 IP: ${WORKER2_IP}"
echo ""
echo "To access the cluster:"
echo "  sshpass -p '${SSH_PASSWORD}' ssh ${SSH_USER}@${MASTER_IP}"
echo "  kubectl get nodes"
echo ""
echo "Credentials:"
echo "  User: ${SSH_USER}"
echo "  Password: ${SSH_PASSWORD}"
echo ""
echo "To check VMs:"
echo "  virsh list --all"
echo ""
echo "To SSH into VMs (without password prompt):"
echo "  sshpass -p '${SSH_PASSWORD}' ssh ${SSH_USER}@${MASTER_IP}"
echo "  sshpass -p '${SSH_PASSWORD}' ssh ${SSH_USER}@${WORKER1_IP}"
echo "  sshpass -p '${SSH_PASSWORD}' ssh ${SSH_USER}@${WORKER2_IP}"