WIP: configure rook

2023-12-29 21:02:38 +01:00
10 changed files with 387 additions and 34 deletions
--- a/badhouseplants/helmfile.yaml
+++ b/badhouseplants/helmfile.yaml
@ -5,42 +5,42 @@ releases:
  - <<: *drone
    installed: true
    namespace: drone-service
-    createNamespace: false
+    createNamespace: true

  - <<: *drone-runner-docker
    installed: true
    namespace: drone-service
-    createNamespace: false
+    createNamespace: true

  - <<: *longhorn
    installed: true
    namespace: longhorn-system
-    createNamespace: false
+    createNamespace: true

  - <<: *argocd
    installed: true
    namespace: argo-system
-    createNamespace: false
+    createNamespace: true

  - <<: *nrodionov
    installed: true
    namespace: nrodionov-application
-    createNamespace: false
+    createNamespace: true

  - <<: *minecraft
    installed: true
    namespace: minecraft-application
-    createNamespace: false
+    createNamespace: true

  - <<: *gitea
    installed: true
    namespace: gitea-service
-    createNamespace: false
+    createNamespace: true

  - <<: *funkwhale
    installed: true
    namespace: funkwhale-application
-    createNamespace: false
+    createNamespace: true

  - <<: *prometheus
    installed: true
@ -50,16 +50,11 @@ releases:
  - <<: *loki
    installed: true
    namespace: monitoring-system
-    createNamespace: false
+    createNamespace: true

  - <<: *promtail
    installed: true
    namespace: monitoring-system
-    createNamespace: false
-
-  - <<: *bitwarden
-    installed: false
-    namespace: bitwarden-application
    createNamespace: true

  - <<: *redis
@ -82,11 +77,6 @@ releases:
    namespace: database-service
    createNamespace: true

-  - <<: *mysql
-    installed: false
-    namespace: database-service
-    createNamespace: true
-
  - <<: *docker-mailserver
    installed: true
    namespace: mail-service
@ -95,7 +85,7 @@ releases:
  - <<: *istio-gateway-resources
    installed: true
    namespace: istio-system
-    createNamespace: false
+    createNamespace: true

  - <<: *vaultwarden
    createNamespace: true
@ -107,6 +97,16 @@ releases:
    namespace: woodpecker-ci
    createNamespace: true

+  - <<: *rook-ceph
+    installed: true
+    namespace: rook-ceph
+    createNamespace: true
+
+  - <<: *rook-ceph-cluster
+    installed: true
+    namespace: rook-ceph-cluster
+    createNamespace: true
+
 bases:
  - ../environments.yaml
  - ../repositories.yaml
--- a/badhouseplants/values/values.istio-gateway-resources.yaml
+++ b/badhouseplants/values/values.istio-gateway-resources.yaml
@ -76,7 +76,7 @@ istio-gateway:
          - '*'
          port:
            name: ssh
-            number: 22
+            number: 2022
            protocol: TCP
    - name: badhouseplants-minecraft
      servers:
--- a/badhouseplants/values/values.istio-ingressgateway.yaml
+++ b/badhouseplants/values/values.istio-ingressgateway.yaml
@ -6,7 +6,7 @@ service:
      protocol: TCP
      targetPort: 25565
    - name: ssh-gitea
-      port: 22
+      port: 2222
      protocol: TCP
      targetPort: 22
    - name: http2
@ -21,10 +21,6 @@ service:
      port: 1194
      protocol: TCP
      targetPort: 1194
-    - name: tcp
-      port: 25
-      protocol: TCP
-      targetPort: 25
    # -----------
    # -- Email
    # -----------
--- a/badhouseplants/values/values.rook-ceph-cluster.yaml
+++ b/badhouseplants/values/values.rook-ceph-cluster.yaml
@ -0,0 +1,96 @@
+cephFileSystems:
+  - name: ceph-filesystem
+    spec:
+      metadataPool:
+        replicated:
+          size: 3
+      dataPools:
+        - failureDomain: host
+          replicated:
+            size: 3
+          name: data0
+      metadataServer:
+        activeCount: 1
+        activeStandby: true
+        resources:
+          limits:
+            cpu: "200m"
+            memory: "256Mi"
+          requests:
+            cpu: "50m"
+            memory: "128Mi"
+        priorityClassName: system-cluster-critical
+    storageClass:
+      enabled: true
+      isDefault: false
+      name: ceph-filesystem
+      pool: data0
+      reclaimPolicy: Delete
+      allowVolumeExpansion: true
+      volumeBindingMode: "Immediate"
+      mountOptions: []
+      parameters:
+        csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
+        csi.storage.k8s.io/provisioner-secret-namespace: "{{ .Release.Namespace }}"
+        csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
+        csi.storage.k8s.io/controller-expand-secret-namespace: "{{ .Release.Namespace }}"
+        csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
+        csi.storage.k8s.io/node-stage-secret-namespace: "{{ .Release.Namespace }}"
+        csi.storage.k8s.io/fstype: ext4
+
+cephObjectStores: []
+  #  - name: ceph-objectstore
+  #    spec:
+  #      metadataPool:
+  #        failureDomain: host
+  #        replicated:
+  #          size: 3
+  #      dataPool:
+  #        failureDomain: host
+  #        erasureCoded:
+  #          dataChunks: 2
+  #          codingChunks: 1
+  #      preservePoolsOnDelete: true
+  #      gateway:
+  #        port: 80
+  #        resources:
+  #          limits:
+  #            cpu: "150m"
+  #            memory: "256Mi"
+  #          requests:
+  #            cpu: "50m"
+  #            memory: "128Mi"
+  #        instances: 1
+  #        priorityClassName: system-cluster-critical
+  #    storageClass:
+  #      enabled: true
+  #      name: ceph-bucket
+  #      reclaimPolicy: Delete
+  #      volumeBindingMode: "Immediate"
+  #      parameters:
+  #        region: us-east-1
+  #    ingress:
+  #      enabled: false
+cephClusterSpec:
+  resources:
+    mgr:
+      limits:
+        cpu: "200m"
+        memory: "512Mi"
+      requests:
+        cpu: "100m"
+        memory: "128Mi"
+    mon:
+      limits:
+        cpu: "200m"
+        memory: "512Mi"
+      requests:
+        cpu: "100m"
+        memory: "128Mi"
+    osd:
+      limits:
+        cpu: "200m"
+        memory: "2Gi"
+      requests:
+        cpu: "100m"
+        memory: "256Mi"
--- a/badhouseplants/values/values.rook-ceph.yaml
+++ b/badhouseplants/values/values.rook-ceph.yaml
@ -0,0 +1,215 @@
+---
+csi:
+  csiRBDProvisionerResource: |
+    - name : csi-provisioner
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-resizer
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-attacher
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-snapshotter
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-rbdplugin
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-omap-generator
+      resource:
+        requests:
+          memory: 12Mi
+          cpu: 250m
+        limits:
+          memory: 1Gi
+          cpu: 500m
+    - name : liveness-prometheus
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+
+  # -- CEPH CSI RBD plugin resource requirement list
+  # @default -- see values.yaml
+  csiRBDPluginResource: |
+    - name : driver-registrar
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+    - name : csi-rbdplugin
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+    - name : liveness-prometheus
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+
+  # -- CEPH CSI CephFS provisioner resource requirement list
+  # @default -- see values.yaml
+  csiCephFSProvisionerResource: |
+    - name : csi-provisioner
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-resizer
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-attacher
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-snapshotter
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-cephfsplugin
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+    - name : liveness-prometheus
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+
+  # -- CEPH CSI CephFS plugin resource requirement list
+  # @default -- see values.yaml
+  csiCephFSPluginResource: |
+    - name : driver-registrar
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+    - name : csi-cephfsplugin
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+    - name : liveness-prometheus
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+
+  # -- CEPH CSI NFS provisioner resource requirement list
+  # @default -- see values.yaml
+  csiNFSProvisionerResource: |
+    - name : csi-provisioner
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-nfsplugin
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+    - name : csi-attacher
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+
+  # -- CEPH CSI NFS plugin resource requirement list
+  # @default -- see values.yaml
+  csiNFSPluginResource: |
+    - name : driver-registrar
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+    - name : csi-nfsplugin
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
--- a/environments.yaml
+++ b/environments.yaml
@ -1,5 +1,5 @@
 environments:
  badhouseplants:
-    kubeContext: badhouseplants
+    kubeContext: badhouseplants-arm
  etersoft:
    kubeContext: etersoft
--- a/helmfile.yaml
+++ b/helmfile.yaml
@ -9,37 +9,37 @@ releases:
  - <<: *metrics-server
    installed: true
    namespace: kube-system
-    createNamespace: false
+    createNamespace: true

  - <<: *istio-base
    installed: true
    namespace: istio-system
-    createNamespace: false
+    createNamespace: true
  
  - <<: *istio-gateway
    installed: true
    namespace: istio-system
-    createNamespace: false
+    createNamespace: true

  - <<: *istiod
    installed: true
    namespace: istio-system
-    createNamespace: false
+    createNamespace: true

  - <<: *cert-manager
    installed: true
    namespace: cert-manager
-    createNamespace: false
+    createNamespace: true

  - <<: *minio
    installed: true
    namespace: minio-service
-    createNamespace: false
+    createNamespace: true

  - <<: *openvpn
    installed: true
    namespace: openvpn-service
-    createNamespace: false
+    createNamespace: true
  
  - <<: *metallb
    installed: true
--- a/releases.yaml
+++ b/releases.yaml
@ -357,3 +357,17 @@ templates:
    name: reflector
    chart: emberstack/reflector
    version: 7.1.216
+
+  rook-ceph: &rook-ceph
+    name: rook-ceph
+    chart: rook/rook-ceph
+    version: v1.13.1
+    inherit:
+      - template: default-env-values
+
+  rook-ceph-cluster: &rook-ceph-cluster
+    name: rook-ceph-cluster
+    chart: rook/rook-ceph-cluster
+    version: v1.13.1
+    inherit:
+      - template: default-env-values
--- a/repositories.yaml
+++ b/repositories.yaml
@ -43,3 +43,5 @@ repositories:
    url: https://firefly-iii.github.io/kubernetes/
  - name: emberstack
    url: https://emberstack.github.io/helm-charts
+  - name: rook
+    url: https://charts.rook.io/release
--- a/test/test.yaml
+++ b/test/test.yaml
@ -0,0 +1,30 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pvc-test
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pvc-test
+spec:
+  restartPolicy: Never
+  volumes:
+  - name: vol
+    persistentVolumeClaim:
+      claimName: pvc-test
+  containers:
+  - name: pv-recycler
+    image: ubuntu
+    command: ["/bin/sh", "-c", "sleep 10000"]
+    volumeMounts:
+    - name: vol
+      mountPath: /data