#在裸金属上创建集群

apiVersion: elemental.cattle.io/v1beta1
kind: MachineRegistration
metadata:
  name: <cluster-name>-registration
  namespace: cpaas-system
  labels:
    app.kubernetes.io/part-of: cluster-api-provider-baremetal
spec:
  # ${...} placeholders are expanded by elemental-register from SMBIOS data.
  # System UUID is preferred over Serial Number because it is more reliable
  # across virtualised test environments.
  machineName: "<cluster-name>-${System Information/UUID}"
  machineInventoryLabels:
    app.kubernetes.io/part-of: cluster-api-provider-baremetal
    pool.baremetal.alauda.io/eligible: "true"
    elemental.cattle.io/serial-number: "${System Information/Serial Number}"
  machineInventoryAnnotations:
    elemental.cattle.io/machine-uuid: "${System Information/UUID}"
  config:
    elemental:
      install:
        device: <install-device>
        eject-cd: true
        reboot: true
        snapshotter:
          type: btrfs
          maxSnaps: 4
      registration:
        # false only when the host has a real hardware TPM (/dev/tpm0). Set true
        # (with emulated-tpm-seed: -1) on any host without one, including physical
        # servers that ship without a TPM module.
        emulate-tpm: false
---
apiVersion: elemental.cattle.io/v1beta1
kind: SeedImage
metadata:
  name: <cluster-name>-registration-iso
  namespace: cpaas-system
  labels:
    app.kubernetes.io/part-of: cluster-api-provider-baremetal
spec:
  type: iso
  baseImage: <base-image-iso>          # Use the imported target-platform registry reference.
  registrationRef:
    apiVersion: elemental.cattle.io/v1beta1
    kind: MachineRegistration
    name: <cluster-name>-registration
    namespace: cpaas-system
  targetPlatform: linux/amd64
  size: 20Gi
  cleanupAfterMinutes: 120
  cloud-config:
    stages:
      boot:
        - name: "Size COS_STATE for reprovisioning"
          files:
            - path: /etc/elemental/config.d/partitions.yaml
              permissions: 0644
              content: |
                install:
                  partitions:
                    state:
                      size: 20480

apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: MachineInventoryPool
metadata:
  name: <cluster-name>-control-plane-pool
  namespace: cpaas-system
spec:
  clusterName: <cluster-name>
  machineInventories:
    - name: <control-plane-inventory-1>
      hostname: <control-plane-host-1>    # Optional; falls back to the inventory name.
    - name: <control-plane-inventory-2>
      hostname: <control-plane-host-2>
    - name: <control-plane-inventory-3>
      hostname: <control-plane-host-3>
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: MachineInventoryPool
metadata:
  name: <cluster-name>-worker-pool
  namespace: cpaas-system
spec:
  clusterName: <cluster-name>
  machineInventories:
    - name: <worker-inventory-1>
      hostname: <worker-host-1>
    - name: <worker-inventory-2>
    - name: <worker-inventory-3>

apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: BaremetalCluster
metadata:
  name: <cluster-name>
  namespace: cpaas-system
spec:
  controlPlaneLoadBalancer:
    type: <control-plane-load-balancer-type>
    host: <control-plane-vip>
    port: <control-plane-port>
    # Required only for Internal. Remove this field for External.
    vrid: <vrid>
  networkType: kube-ovn      # Enables provider-managed Kube-OVN AppRelease reconciliation.
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: BaremetalMachineTemplate
metadata:
  name: <cluster-name>-control-plane-template
  namespace: cpaas-system
spec:
  template:
    spec:
      machineInventoryPoolRef:
        name: <cluster-name>-control-plane-pool
---
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlane
metadata:
  name: <cluster-name>-control-plane
  namespace: cpaas-system
spec:
  replicas: 3
  version: <kubernetes-version>
  rolloutStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 0            # Bare-metal does not over-provision; replace one at a time.
  machineTemplate:
    nodeDrainTimeout: 5m
    nodeVolumeDetachTimeout: 5m
    infrastructureRef:
      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
      kind: BaremetalMachineTemplate
      name: <cluster-name>-control-plane-template
  kubeadmConfigSpec:
    users:
      - name: boot
        sudo: ALL=(ALL) NOPASSWD:ALL
        shell: /bin/bash
        sshAuthorizedKeys:
          - "<ssh-authorized-keys>"
    clusterConfiguration:
      imageRepository: cloud.alauda.io/alauda
      dns:
        imageTag: <dns-image-tag>
      etcd:
        local:
          imageTag: <etcd-image-tag>
      apiServer:
        extraArgs:
          profiling: "false"
          tls-min-version: VersionTLS12
      controllerManager:
        extraArgs:
          profiling: "false"
          tls-min-version: VersionTLS12
      scheduler:
        extraArgs:
          profiling: "false"
          tls-min-version: VersionTLS12
    initConfiguration:
      nodeRegistration:
        kubeletExtraArgs:
          node-labels: "kube-ovn/role=master"
    joinConfiguration:
      nodeRegistration:
        kubeletExtraArgs:
          node-labels: "kube-ovn/role=master"
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
  name: <cluster-name>
  namespace: cpaas-system
  annotations:
    capi.cpaas.io/resource-group-version: infrastructure.cluster.x-k8s.io/v1beta1
    capi.cpaas.io/resource-kind: BaremetalCluster
    cpaas.io/kube-ovn-join-cidr: <kube-ovn-join-cidr>
    cpaas.io/kube-ovn-version: <kube-ovn-version>
    cpaas.io/sentry-deploy-type: Baremetal
    cpaas.io/alb-address-type: ClusterAddress
  labels:
    cluster-type: ProviderBaremetal
spec:
  clusterNetwork:
    pods:
      cidrBlocks:
        - <pods-cidr>
    services:
      cidrBlocks:
        - <services-cidr>
  controlPlaneRef:
    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    kind: KubeadmControlPlane
    name: <cluster-name>-control-plane
  infrastructureRef:
    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    kind: BaremetalCluster
    name: <cluster-name>

apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: BaremetalMachineTemplate
metadata:
  name: <cluster-name>-worker-template
  namespace: cpaas-system
spec:
  template:
    spec:
      machineInventoryPoolRef:
        name: <cluster-name>-worker-pool

apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
metadata:
  name: <cluster-name>-worker-bootstrap
  namespace: cpaas-system
spec:
  template:
    spec:
      users:
        - name: boot
          sudo: ALL=(ALL) NOPASSWD:ALL
          shell: /bin/bash
          sshAuthorizedKeys:
            - "<ssh-authorized-keys>"
      files:
        - path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.json
          owner: "root:root"
          permissions: "0644"
          content: |
            {
              "apiVersion": "kubelet.config.k8s.io/v1beta1",
              "kind": "KubeletConfiguration",
              "imagePullCredentialsVerificationPolicy": "NeverVerify",
              "protectKernelDefaults": true,
              "staticPodPath": null,
              "tlsCertFile": "/etc/kubernetes/pki/kubelet.crt",
              "tlsPrivateKeyFile": "/etc/kubernetes/pki/kubelet.key",
              "streamingConnectionIdleTimeout": "5m",
              "clientCAFile": "/etc/kubernetes/pki/ca.crt"
            }
      joinConfiguration:
        patches:
          directory: /etc/kubernetes/patches

apiVersion: cluster.x-k8s.io/v1beta1
kind: MachineDeployment
metadata:
  name: <cluster-name>-workers
  namespace: cpaas-system
spec:
  clusterName: <cluster-name>
  replicas: 3
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 0        # Bare-metal pools cannot over-provision.
      maxUnavailable: 1
  selector:
    matchLabels: {}
  template:
    metadata:
      labels:
        cluster.x-k8s.io/cluster-name: <cluster-name>
        pool.name: <cluster-name>-workers
    spec:
      clusterName: <cluster-name>
      version: <kubernetes-version>
      nodeDrainTimeout: 5m
      nodeVolumeDetachTimeout: 5m
      bootstrap:
        configRef:
          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
          kind: KubeadmConfigTemplate
          name: <cluster-name>-worker-bootstrap
          namespace: cpaas-system
      infrastructureRef:
        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
        kind: BaremetalMachineTemplate
        name: <cluster-name>-worker-template
        namespace: cpaas-system

apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlane
metadata:
  name: <cluster-name>-control-plane
  namespace: cpaas-system
spec:
  replicas: 3
  version: <kubernetes-version>
  rolloutStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 0
  machineTemplate:
    nodeDrainTimeout: 5m
    nodeVolumeDetachTimeout: 5m
    infrastructureRef:
      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
      kind: BaremetalMachineTemplate
      name: <cluster-name>-control-plane-template
  kubeadmConfigSpec:
    users:
      - name: boot
        sudo: ALL=(ALL) NOPASSWD:ALL
        shell: /bin/bash
        sshAuthorizedKeys:
          - "<ssh-authorized-keys>"
    files:
      - path: /etc/kubernetes/admission/psa-config.yaml
        owner: "root:root"
        permissions: "0644"
        content: |
          apiVersion: apiserver.config.k8s.io/v1
          kind: AdmissionConfiguration
          plugins:
          - name: PodSecurity
            configuration:
              apiVersion: pod-security.admission.config.k8s.io/v1
              kind: PodSecurityConfiguration
              defaults:
                enforce: "privileged"
                enforce-version: "latest"
                audit: "baseline"
                audit-version: "latest"
                warn: "baseline"
                warn-version: "latest"
              exemptions:
                usernames: []
                runtimeClasses: []
                namespaces:
                - kube-system
                - cpaas-system
      - path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.json
        owner: "root:root"
        permissions: "0644"
        content: |
          {
            "apiVersion": "kubelet.config.k8s.io/v1beta1",
            "kind": "KubeletConfiguration",
            "imagePullCredentialsVerificationPolicy": "NeverVerify",
            "protectKernelDefaults": true,
            "tlsCertFile": "/etc/kubernetes/pki/kubelet.crt",
            "tlsPrivateKeyFile": "/etc/kubernetes/pki/kubelet.key",
            "streamingConnectionIdleTimeout": "5m",
            "clientCAFile": "/etc/kubernetes/pki/ca.crt"
          }
      - path: /etc/kubernetes/audit/policy.yaml
        owner: "root:root"
        permissions: "0644"
        content: |
          apiVersion: audit.k8s.io/v1
          kind: Policy
          omitStages:
          - "RequestReceived"
          rules:
          - level: None
            users:
            - system:kube-controller-manager
            - system:kube-scheduler
            - system:serviceaccount:kube-system:endpoint-controller
            verbs: ["get", "update"]
            namespaces: ["kube-system"]
            resources:
            - group: ""
              resources: ["endpoints"]
          - level: None
            nonResourceURLs:
            - /healthz*
            - /version
            - /swagger*
          - level: None
            resources:
            - group: ""
              resources: ["events"]
          - level: None
            verbs: ["get", "list", "watch"]
          - level: None
            resources:
            - group: "coordination.k8s.io"
              resources: ["leases"]
          - level: None
            resources:
            - group: "authorization.k8s.io"
              resources: ["subjectaccessreviews", "selfsubjectaccessreviews"]
            - group: "authentication.k8s.io"
              resources: ["tokenreviews"]
          - level: Metadata
            resources:
            - group: ""
              resources: ["secrets", "configmaps"]
          - level: RequestResponse
            resources:
            - group: ""
            - group: "apps"
            - group: "rbac.authorization.k8s.io"
            - group: "storage.k8s.io"
            - group: "networking.k8s.io"
          - level: Metadata
    postKubeadmCommands:
      - chmod 600 /var/lib/kubelet/config.yaml
    clusterConfiguration:
      imageRepository: cloud.alauda.io/alauda
      dns:
        imageTag: <dns-image-tag>
      etcd:
        local:
          imageTag: <etcd-image-tag>
      apiServer:
        extraArgs:
          audit-log-format: json
          audit-log-maxage: "30"
          audit-log-maxbackup: "10"
          audit-log-maxsize: "200"
          profiling: "false"
          audit-log-mode: batch
          audit-log-path: /etc/kubernetes/audit/audit.log
          audit-policy-file: /etc/kubernetes/audit/policy.yaml
          tls-cipher-suites: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"
          admission-control-config-file: /etc/kubernetes/admission/psa-config.yaml
          tls-min-version: VersionTLS12
          kubelet-certificate-authority: /etc/kubernetes/pki/ca.crt
        extraVolumes:
        - name: vol-dir-0
          hostPath: /etc/kubernetes
          mountPath: /etc/kubernetes
          pathType: Directory
      controllerManager:
        extraArgs:
          bind-address: "::"
          profiling: "false"
          tls-min-version: VersionTLS12
      scheduler:
        extraArgs:
          bind-address: "::"
          tls-min-version: VersionTLS12
          profiling: "false"
    initConfiguration:
      patches:
        directory: /etc/kubernetes/patches
      nodeRegistration:
        kubeletExtraArgs:
          node-labels: "kube-ovn/role=master"
    joinConfiguration:
      patches:
        directory: /etc/kubernetes/patches
      nodeRegistration:
        kubeletExtraArgs:
          node-labels: "kube-ovn/role=master"