HAQM EKS 存储应用：实例存储本地盘的应用实践方案

# pv-nvme-nodegroup.yaml
apiVersion: eksctl.io/v1alpha5
kind: ClusterConfig

metadata:
  # Replace with your EKS Cluster's name
  name: eksworkshop-eksctl
  # Replace with the AWS Region your cluster is deployed in
  region: us-west-2

managedNodeGroups:
  # Name to give the managed node-group
  - name: eks-pv-nvme-ng
    # Label the nodes that they contain fast-disks 
    labels: { fast-disk-node: "pv-nvme" }
    instanceType: i3.8xlarge
    desiredCapacity: 2
    volumeSize: 100 # EBS Boot Volume size
    privateNetworking: true
    preBootstrapCommands:
      - |
        # Install NVMe CLI
        yum install nvme-cli -y
        
        # Get a list of instance-store NVMe drives
        nvme_drives=$(nvme list | grep "HAQM EC2 NVMe Instance Storage" | cut -d " " -f 1 || true)
        readarray -t nvme_drives <<< "$nvme_drives"

        for disk in "${nvme_drives[@]}"
        do
          # Format the disk to ext4
          mkfs.ext4 -F $disk
          
          # Get disk UUID
          uuid=$(blkid -o value -s UUID $disk)
   
          # Create a filesystem path to mount the disk
          mount_location="/mnt/fast-disks/${uuid}"
          mkdir -p $mount_location
   
          # Mount the disk
          mount $disk $mount_location
          
          # Mount the disk during a reboot
          echo $disk $mount_location ext4 defaults,noatime 0 2 >> /etc/fstab 
        done

eksctl create nodegroup -f pv-nvme-nodegroup.yaml

apiVersion: eksctl.io/v1alpha5
kind: ClusterConfig

metadata:
  # Replace with your EKS Cluster's name
  name: eksworkshop-eksctl
  # Replace with the AWS Region your cluster is deployed in
  region: us-west-2

managedNodeGroups:
  # Name to give the managed node-group
  - name: eks-pv-raid-ng
    # Label the nodes that they contain fast-disks 
    labels: { fast-disk-node: "pv-raid" }
    instanceType: i3.8xlarge
    desiredCapacity: 2
    volumeSize: 100 # EBS Boot Volume size
    privateNetworking: true
    preBootstrapCommands:
      - |
        # Install NVMe CLI
        yum install nvme-cli -y
        
        # Get list of NVMe Drives
        nvme_drives=$(nvme list | grep "HAQM EC2 NVMe Instance Storage" | cut -d " " -f 1 || true)
        readarray -t nvme_drives <<< "$nvme_drives"
        num_drives=${#nvme_drives[@]}
        
        # Install software RAID utility
        yum install mdadm -y
        
        # Create RAID-0 array across the instance store NVMe SSDs
        mdadm --create /dev/md0 --level=0 --name=md0 --raid-devices=$num_drives "${nvme_drives[@]}"

        # Format drive with Ext4
        mkfs.ext4 /dev/md0

        # Get RAID array's UUID
        uuid=$(blkid -o value -s UUID /dev/md0)
   
        # Create a filesystem path to mount the disk
        mount_location="/mnt/fast-disks/${uuid}"
        mkdir -p $mount_location
        
        # Mount RAID device
        mount /dev/md0 $mount_location
        
        # Have disk be mounted on reboot
        mdadm --detail --scan >> /etc/mdadm.conf 
        echo /dev/md0 $mount_location ext4 defaults,noatime 0 2 >> /etc/fstab

eksctl create nodegroup --config-file=pv-raid-nodegroup.yaml

# K8s service account for CSI Driver
apiVersion: v1
kind: ServiceAccount
metadata:
  name: local-volume-provisioner
  namespace: kube-system
---
# List of Permissions 
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: local-storage-provisioner-node-clusterrole
rules:
- apiGroups: [""]
  resources: ["persistentvolumes"]
  verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: ["storage.k8s.io"]
  resources: ["storageclasses"]
  verbs: ["get", "list", "watch"]
- apiGroups: [""]
  resources: ["events"]
  verbs: ["watch"]
- apiGroups: ["", "events.k8s.io"]
  resources: ["events"]
  verbs: ["create", "update", "patch"]
- apiGroups: [""]
  resources: ["nodes"]
  verbs: ["get"]
---
# Attach the K8s ClusterRole to our K8s ServiceAccount
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: local-storage-provisioner-node-binding
  namespace: kube-system
subjects:
- kind: ServiceAccount
  name: local-volume-provisioner
  namespace: kube-system
roleRef:
  kind: ClusterRole
  name: local-storage-provisioner-node-clusterrole
  apiGroup: rbac.authorization.k8s.io

kubectl apply -f service-account.yaml

apiVersion: storage.k8s.io/v1kind: StorageClassmetadata:
name: fast-disksprovisioner: kubernetes.io/no-provisionervolumeBindingMode: WaitForFirstConsumer# Supported policies: Delete, Retain
reclaimPolicy: Retain---
# Configuration for our Local Persistent Volume CSI Driver
apiVersion: v1kind: ConfigMapmetadata:
name: local-volume-provisioner-config
  namespace: kube-system
data:
# Adds the node's hostname as a label to each PV created
nodeLabelsForPV: |
    - kubernetes.io/hostname
storageClassMap: |
    fast-disks:
      # Path on the host where local volumes of this storage class
      # are mounted under.
      hostDir: /mnt/fast-disks
# Optionally specify mount path of local volumes.
# By default, we use same path as hostDir in container.
mountDir: /mnt/fast-disks
 
      # The /scripts/shred.sh is contained in the CSI drivers container
# http://github.com/kubernetes-sigs/sig-storage-local-static-provisioner/blob/master/deployment/docker/scripts/shred.sh
blockCleanerCommand:
- "/scripts/shred.sh"
- "2"
# The volume mode of PV defines whether a device volume is
# intended to use as a formatted filesystem volume or to remain in block
# state. Value of Filesystem is implied when omitted.
volumeMode: Filesystem
      fsType: ext4
      
      # name pattern check
# only discover local disk mounted to path matching pattern("*" by default).
namePattern: "*"

kubectl apply -f config-map.yaml

# The Local Persistent Volume CSI DaemonSet
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: local-volume-provisioner
  namespace: kube-system
  labels:
    app.kubernetes.io/name: local-volume-provisioner
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: local-volume-provisioner 
  template:
    metadata:
      labels:
        app.kubernetes.io/name: local-volume-provisioner
    spec:
      serviceAccountName: local-volume-provisioner
      containers:
          # The latest version can be found in the changelog.
          # In production, one might want to use the container digest hash 
          # over version for improved security.
          # http://github.com/kubernetes-sigs/sig-storage-local-static-provisioner/blob/master/CHANGELOG.md
        - image: "registry.k8s.io/sig-storage/local-volume-provisioner:v2.5.0"
          # In production you might want to set this to use a locally cached 
          # image by setting this to: IfNotPresent
          imagePullPolicy: "Always"
          name: provisioner 
          securityContext:
            privileged: true
          env:
          - name: MY_NODE_NAME
            valueFrom:
              fieldRef:
                fieldPath: spec.nodeName
          - name: MY_NAMESPACE
            valueFrom:
              fieldRef:
                fieldPath: metadata.namespace
          ports:
            # List of metrics at
            # http://github.com/kubernetes-sigs/sig-storage-local-static-provisioner/blob/cee9e228dc28a4355f664b4fe2236b1857fe4eca/pkg/metrics/metrics.go
            - name: metrics
              containerPort: 8080
          volumeMounts:
            - name: provisioner-config
              mountPath: /etc/provisioner/config
              readOnly: true             
            - mountPath:  /mnt/fast-disks 
              name: fast-disks
              mountPropagation: "HostToContainer" 
      volumes:
        - name: provisioner-config
          configMap:
            name: local-volume-provisioner-config
        - name: fast-disks
          hostPath:
            path: /mnt/fast-disks
      # Only run CSI Driver on the `fast-disk` tagged nodegroup 
      affinity:
        nodeAffinity: 
          requiredDuringSchedulingIgnoredDuringExecution: 
            nodeSelectorTerms: 
            - matchExpressions:
              - key: fast-disk-node
                operator: In
                values:
                - "pv-raid"
                - "pv-nvme"

kubectl apply -f daemonset.yaml

apiVersion: eksctl.io/v1alpha5
kind: ClusterConfig
metadata:
  name: eksworkshop-eksctl
  region: us-west-2
managedNodeGroups:
  - name: storage-nvme
    desiredCapacity: 4
    instanceType: i4g.large
    preBootstrapCommands:
      - mkdir -p /mnt/fast-disks
      - |
          cat <<EOF > /etc/udev/rules.d/90-kubernetes-nvme-symlink.rules
          # Discover Instance Storage disks so kubernetes local provisioner can pick them up from
          KERNEL=="nvme[0-9]*n[0-9]*", ENV{DEVTYPE}=="disk", ATTRS{model}=="HAQM EC2 NVMe Instance Storage", ATTRS{serial}=="?*", SYMLINK+="/mnt/fast-disks/nvme-\\\$attr{model}_\\\$attr{serial}", OPTIONS="string_escape=replace"
          EOF
      - udevadm control --reload && udevadm trigger
      

eksctl create nodegroup --config-file=pv-nodegroup.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: fast-pvc
spec:
  storageClassName: local-fast
  accessModes: [ReadWriteOnce]
  resources:
    requests:
      storage: 500Gi  # 容量仅用于匹配PV

---
apiVersion: v1
kind: Pod
metadata:
  name: app-with-pvc
spec:
  containers:
  - name: app
    image: nginx
    volumeMounts:
    - name: data
      mountPath: /data
  volumes:
  - name: data
    persistentVolumeClaim:
      claimName: fast-pvc