使用Terraform和Ansible创建K8s集群

- name: Download Ubuntu Cloud Image for Proxmox
  hosts: proxmox # Target Proxmox hosts
  become: true

  vars:
    image_url: "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
    image_name: "ubuntu-24.04-noble-cloudimg.img"
    # Proxmox 存储路径，根据实际情况修改
    image_dest_path: "/mnt/pve/iso-templates/template/iso/"

  tasks:
    - name: Ensure destination directory exists
      ansible.builtin.file:
        path: "{{ image_dest_path }}"
        state: directory
        mode: '0755'

    - name: Download Ubuntu 24.04 cloud image
      ansible.builtin.get_url:
        url: "{{ image_url }}"
        dest: "{{ image_dest_path }}{{ image_name }}"
        mode: '0644'

- name: Create Ubuntu 24.04 Cloud-Init Template from existing image
  hosts: proxmox
  become: true

  vars:
    # --- Template Configuration ---
    template_vmid: 9000 # 使用一个高的、固定的ID作为模板ID
    template_name: "ubuntu-2404-cloud-template"
    template_memory: 2048
    template_cores: 2
    template_storage: "local-lvm" # 模板的虚拟磁盘将存放在此

    # --- Image Source ---
    image_name: "ubuntu-24.04-noble-cloudimg.img"
    # !! 重要: 这个路径必须与 'download-cloud-image.yaml' 中的 'image_dest_path' 完全匹配
    image_source_path: "/mnt/pve/iso-templates/template/iso/{{ image_name }}"

  tasks:
    - name: Check if template already exists
      ansible.builtin.command: "qm status {{ template_vmid }}"
      register: template_check
      failed_when: false # 即使命令失败（模板不存在）也不要报错
      changed_when: false

    - name: Stop if template already exists
      ansible.builtin.meta: end_play
      when: template_check.rc == 0

    - name: Check if the source image exists
      ansible.builtin.stat:
        path: "{{ image_source_path }}"
      register: image_file

    - name: Fail if source image is not found
      ansible.builtin.fail:
        msg: >
          Source image not found at {{ image_source_path }}.
          Please run the 'download-cloud-image' task first.          
      when: not image_file.stat.exists

    # --- Block for creating the template ---
    - name: Create a new template from the existing image
      block:
        - name: Create a new temporary VM
          ansible.builtin.command: >
            qm create {{ template_vmid }} --name {{ template_name }}-builder --memory {{ template_memory }}
            --cores {{ template_cores }} --net0 virtio,bridge=vmbr0 --scsihw virtio-scsi-pci            
          changed_when: true

        - name: Import the disk from the existing image
          ansible.builtin.command: >
            qm importdisk {{ template_vmid }} {{ image_source_path }} {{ template_storage }}            
          changed_when: true

        - name: Attach the new disk to the VM as scsi0
          ansible.builtin.command: >
            qm set {{ template_vmid }} --scsi0 {{ template_storage }}:vm-{{ template_vmid }}-disk-0,discard=on            
          changed_when: true

        - name: Add Cloud-Init CD-ROM drive
          ansible.builtin.command: >
            qm set {{ template_vmid }} --ide2 {{ template_storage }}:cloudinit            
          changed_when: true

        - name: Set boot order to boot from the new disk
          ansible.builtin.command: >
            qm set {{ template_vmid }} --boot c --bootdisk scsi0            
          changed_when: true

        - name: Add a serial console for debugging
          ansible.builtin.command: >
            qm set {{ template_vmid }} --serial0 socket --vga serial0            
          changed_when: true

        - name: Convert the VM to a template
          ansible.builtin.command: "qm template {{ template_vmid }}"
          changed_when: true
      when: template_check.rc != 0

# --- Proxmox Provider Configuration ---
variable "proxmox_endpoint" {
  description = "The endpoint URL of the Proxmox API (e.g., https://pve.example.com:8006)."
  type        = string
}

variable "proxmox_username" {
  description = "The username for the Proxmox API (e.g., root@pam)."
  type        = string
}

variable "proxmox_password" {
  description = "The password for the Proxmox API."
  type        = string
  sensitive   = true
}

# --- VM Configuration ---
variable "vms" {
  description = "A map of virtual machines to create."
  type = map(object({
    cores    = number
    memory   = number
    disk_size = number
    user     = string
    ip       = string
    gateway  = string
  }))
  default   = {}
}

variable "proxmox_node" {
  description = "The Proxmox node where the VMs will be created."
  type        = string
}

variable "template_vmid" {
  description = "The VM ID of the Proxmox template to clone from."
  type        = number
  default     = 9000 # 匹配 Ansible playbook 中创建的模板 ID
}

variable "ssh_public_key" {
  description = "The SSH public key to add to the 'ubuntu' user."
  type        = string
  sensitive   = true
}

#  Create Ubuntu VM
resource "proxmox_virtual_environment_vm" "ubuntu_vm" {
  for_each = var.vms

  name      = each.key
  node_name = var.proxmox_node

  # Clone from the specified template
  clone {
    vm_id = var.template_vmid
    full          = true
  }

  # should be true if qemu agent is not installed / enabled on the VM
  stop_on_destroy = true

  initialization {
    ip_config {
      ipv4 {
        address = each.value.ip
        gateway = each.value.gateway
      }
    }
    user_account {
      username = each.value.user
      keys     = [var.ssh_public_key]
    }
  }

  network_device {
    bridge = "vmbr0"
  }

  cpu {
    cores = each.value.cores
  }

  memory {
    dedicated = each.value.memory
  }

  serial_device {}

  disk {
    # The disk interface must match the template's disk (scsi0 in this case)
    interface    = "scsi0"
    datastore_id = "local-lvm"
    size         = each.value.disk_size
    discard      = "on"
  }
}

- name: Install MicroK8s on all nodes
  hosts: k8s_cluster
  become: true
  pre_tasks:
    - name: Wait for snapd to be fully ready by polling 'snap list'
      ansible.builtin.command: snap list
      register: snap_list_result
      until: snap_list_result.rc == 0
      retries: 15
      delay: 10
      changed_when: false

  tasks:
    - name: Check if MicroK8s is already installed
      ansible.builtin.command: snap list microk8s
      register: microk8s_check
      failed_when: false
      changed_when: false

    - name: Install MicroK8s using snap command
      ansible.builtin.command: snap install microk8s --classic
      when: microk8s_check.rc != 0
      # Only install if the previous check failed (meaning microk8s is not installed)

    - name: Add ubuntu user to microk8s group
      ansible.builtin.user:
        name: ubuntu
        groups: microk8s
        append: true

- name: Configure Firewall on all nodes
  hosts: k8s_cluster
  become: true
  tasks:
    - name: Allow SSH connections
      community.general.ufw:
        rule: allow
        name: OpenSSH

    - name: Allow all traffic from other k8s nodes
      community.general.ufw:
        rule: allow
        src: "{{ hostvars[item]['ansible_host'] }}"
      loop: "{{ groups['k8s_cluster'] }}"

    - name: Allow traffic on CNI pods' CIDR (Calico)
      community.general.ufw:
        rule: allow
        src: 10.1.0.0/16

    # --- NEW TASK: Allow Kubernetes API Server access ---
    - name: Allow Kubernetes API Server access (port 16443)
      community.general.ufw:
        rule: allow
        port: '16443'
        proto: tcp
      when: inventory_hostname in groups['k8s_masters']
      # Only apply this rule to master nodes

    - name: Enable ufw
      community.general.ufw:
        state: enabled

- name: Initialize Master Node and Generate Join Tokens
  hosts: k8s_masters
  become: true
  tasks:
    - name: Wait for MicroK8s to be ready
      ansible.builtin.command: microk8s status --wait-ready
      changed_when: false

    - name: Generate a unique join token for each worker node
      ansible.builtin.command: microk8s add-node
      register: join_command_result
      loop: "{{ groups['k8s_workers'] }}"
      loop_control:
        loop_var: worker_host
      changed_when: false

    - name: Set the unique join command as a fact for each worker
      ansible.builtin.set_fact:
        join_command: "{{ item.stdout_lines | select('search', 'microk8s join') | first }}"
      loop: "{{ join_command_result.results }}"
      loop_control:
        index_var: idx
      delegate_to: "{{ groups['k8s_workers'][idx] }}"
      delegate_facts: true

- name: Join Worker Nodes to the Cluster
  hosts: k8s_workers
  become: true
  tasks:
    - name: Join the cluster using the unique token
      ansible.builtin.command: "{{ join_command }}"
      register: join_result
      changed_when: "'Contacting cluster' in join_result.stdout"

- name: Enable Addons on Master
  hosts: k8s_masters
  become: true
  tasks:
    - name: Enable core addons (DNS, default storage, dashboard)
      ansible.builtin.command: "microk8s enable {{ item }}"
      loop:
        - dns
        - hostpath-storage
        - dashboard
      changed_when: true