Tenant TLS plus KES TLS all with cert manager together - cniackz/public GitHub Wiki

Steps:

• Window 1:

createcluster

echo "apiVersion: v1
kind: Namespace
metadata:
  name: tenant-certmanager" > namespace.yaml

k apply -f namespace.yaml

kubectl apply -f ~/operator/examples/vault/deployment.yaml

kubectl wait --namespace tenant-certmanager --for=condition=ready pod --selector=app=vault --timeout=120s
echo " "
echo " "
echo " "
installoperator
echo " "
echo " "
echo " "

• Window 2:

VAULT_ROOT_TOKEN=$(kubectl logs -l app=vault -n tenant-certmanager | grep "Root Token: " | sed -e "s/Root Token: //g");

kubectl exec $(kubectl get pods -l app=vault -n tenant-certmanager  | grep -v NAME | awk '{print $1}') -n tenant-certmanager -- sh -c 'VAULT_TOKEN='$VAULT_ROOT_TOKEN' VAULT_ADDR="http://127.0.0.1:8200" vault auth enable approle'

kubectl exec $(kubectl get pods -l app=vault -n tenant-certmanager | grep -v NAME | awk '{print $1}') -n tenant-certmanager -- sh -c 'VAULT_TOKEN='$VAULT_ROOT_TOKEN' VAULT_ADDR="http://127.0.0.1:8200" vault secrets enable kv'

kubectl cp ~/operator/examples/vault/kes-policy.hcl $(kubectl get pods -l app=vault -n tenant-certmanager | grep -v NAME | awk '{print $1}'):/kes-policy.hcl -n tenant-certmanager

kubectl exec $(kubectl get pods -l app=vault -n tenant-certmanager | grep -v NAME | awk '{print $1}') -n tenant-certmanager -- sh -c 'VAULT_TOKEN='$VAULT_ROOT_TOKEN' VAULT_ADDR="http://127.0.0.1:8200" vault policy write kes-policy /kes-policy.hcl'

kubectl exec $(kubectl get pods -l app=vault -n tenant-certmanager | grep -v NAME | awk '{print $1}') -n tenant-certmanager -- sh -c 'VAULT_TOKEN='$VAULT_ROOT_TOKEN' VAULT_ADDR="http://127.0.0.1:8200" vault write auth/approle/role/kes-role token_num_uses=0 secret_id_num_uses=0 period=5m policies=kes-policy'

ROLE_ID=$(kubectl exec $(kubectl get pods -l app=vault -n tenant-certmanager | grep -v NAME | awk '{print $1}') -n tenant-certmanager -- sh -c 'VAULT_TOKEN='$VAULT_ROOT_TOKEN' VAULT_ADDR="http://127.0.0.1:8200" vault read auth/approle/role/kes-role/role-id' | grep "role_id    " | sed -e "s/role_id    //g")

SECRET_ID=$(kubectl exec $(kubectl get pods -l app=vault -n tenant-certmanager | grep -v NAME | awk '{print $1}') -n tenant-certmanager -- sh -c 'VAULT_TOKEN='$VAULT_ROOT_TOKEN' VAULT_ADDR="http://127.0.0.1:8200" vault write -f auth/approle/role/kes-role/secret-id' | grep "secret_id             " | sed -e "s/secret_id             //g")

echo " "
echo " "
echo " "
echo " "
echo "ROLE_ID = ${ROLE_ID}"
echo "SECRET_ID = ${SECRET_ID}"

• apply

echo "apiVersion: v1
kind: Secret
metadata:
  name: storage-certmanager-secret-kes-configuration
  namespace: tenant-certmanager
  labels:
    v1.min.io/tenant: storage-certmanager
  managedFields:
    - manager: console
      operation: Update
      apiVersion: v1
      time: '2022-11-29T14:29:22Z'
      fieldsType: FieldsV1
      fieldsV1:
        f:data:
          .: {}
          f:server-config.yaml: {}
        f:immutable: {}
        f:metadata:
          f:labels:
            .: {}
            f:v1.min.io/tenant: {}
        f:type: {}
  selfLink: /api/v1/namespaces/tenant-certmanager/secrets/storage-certmanager-secret-kes-configuration
immutable: true
type: Opaque
stringData:
  server-config.yaml: |-
    address: 0.0.0.0:7373
    root: disabled
    tls:
      key: /tmp/kes/server.key
      cert: /tmp/kes/server.crt
    policy:
      default-policy:
        paths:
        - /v1/key/create/my-minio-key
        - /v1/key/generate/my-minio-key
        - /v1/key/decrypt/my-minio-key
        identities:
        - \${MINIO_KES_IDENTITY}
    cache:
      expiry:
        any: 5m0s
        unused: 20s
    log:
      error: "on"
      audit: "off"
    keys:
      vault:
        endpoint: http://vault.tenant-certmanager.svc.cluster.local:8200
        prefix: my-minio
        approle:
          id: ${ROLE_ID}
          secret: ${SECRET_ID}
        status: {}" > kes-configuration.yaml
k apply -f kes-configuration.yaml

• next

echo "apiVersion: v1
kind: Secret
metadata:
  name: storage-certmanager-env-configuration
  namespace: tenant-certmanager
  uid: c89ddc64-a723-41cf-8a8d-8ecd0d8c88d8
  resourceVersion: '1085'
  creationTimestamp: '2022-11-29T14:29:22Z'
  labels:
    v1.min.io/tenant: storage-certmanager
  managedFields:
    - manager: console
      operation: Update
      apiVersion: v1
      time: '2022-11-29T14:29:22Z'
      fieldsType: FieldsV1
      fieldsV1:
        f:data:
          .: {}
          f:config.env: {}
        f:metadata:
          f:labels:
            .: {}
            f:v1.min.io/tenant: {}
        f:type: {}
  selfLink: /api/v1/namespaces/tenant-certmanager/secrets/storage-certmanager-env-configuration
type: Opaque
stringData:
  config.env: |-
    export MINIO_BROWSER=\"on\"
    export MINIO_ROOT_USER=\"WHNXZOGKN5IQIPQD\"
    export MINIO_ROOT_PASSWORD=\"4IJJVSFGVCJV4X2HGRK1K04KCLRIWT1Q\"
    export MINIO_STORAGE_CLASS_STANDARD=\"EC:2\"" > kes-env.yaml
k apply -f kes-env.yaml

• next

echo "apiVersion: v1
kind: Secret
metadata:
  name: storage-certmanager-user-0
  namespace: tenant-certmanager
  uid: 08f62e3e-3ac1-4199-af46-9ee7a0cc4e15
  resourceVersion: '1083'
  creationTimestamp: '2022-11-29T14:29:22Z'
  labels:
    v1.min.io/tenant: storage-certmanager
  managedFields:
    - manager: console
      operation: Update
      apiVersion: v1
      time: '2022-11-29T14:29:22Z'
      fieldsType: FieldsV1
      fieldsV1:
        f:data:
          .: {}
          f:CONSOLE_ACCESS_KEY: {}
          f:CONSOLE_SECRET_KEY: {}
        f:immutable: {}
        f:metadata:
          f:labels:
            .: {}
            f:v1.min.io/tenant: {}
        f:type: {}
  selfLink: /api/v1/namespaces/tenant-certmanager/secrets/storage-certmanager-user-0
immutable: true
type: Opaque
data:
  CONSOLE_ACCESS_KEY: Y29uc29sZQ==
  CONSOLE_SECRET_KEY: Y29uc29sZTEyMw==" > kes-console.yaml
k apply -f kes-console.yaml

• issuer

kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml

echo "apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
  name: tenant-certmanager-issuer
  namespace: tenant-certmanager
spec:
  selfSigned: {}" > issuer.yaml

k apply -f issuer.yaml

• next Certificate

echo "apiVersion: cert-manager.io/v1" > certificate222.yaml
echo "kind: Certificate" >> certificate222.yaml
echo "metadata:" >> certificate222.yaml
echo "  name: tenant-certmanager-cert" >> certificate222.yaml
echo "  namespace: tenant-certmanager" >> certificate222.yaml
echo "spec:" >> certificate222.yaml
echo "  dnsNames:" >> certificate222.yaml
echo "    - \"*.tenant-certmanager.svc.cluster.local\"" >> certificate222.yaml
echo "    - \"*.storage-certmanager.tenant-certmanager.svc.cluster.local\"" >> certificate222.yaml
echo "    - \"*.storage-certmanager-hl.tenant-certmanager.svc.cluster.local\"" >> certificate222.yaml
echo "  secretName: tenant-certmanager-tls" >> certificate222.yaml
echo "  issuerRef:" >> certificate222.yaml
echo "    name: tenant-certmanager-issuer" >> certificate222.yaml

k apply -f certificate222.yaml

• next

echo "apiVersion: minio.min.io/v2
kind: Tenant
metadata:
  generation: 2
  name: storage-certmanager
  namespace: tenant-certmanager
spec:
  ## Use certificates generated by cert-manager.
  externalCertSecret:
    - name: tenant-certmanager-tls
      type: cert-manager.io/v1
  configuration:
    name: storage-certmanager-env-configuration
  credsSecret:
    name: storage-certmanager-secret
  exposeServices:
    console: true
    minio: true
  image: minio/minio:RELEASE.2022-11-26T22-43-32Z
  imagePullPolicy: IfNotPresent
  imagePullSecret: {}
  kes:
    externalCertSecret:
      name: tenant-certmanager-tls
      type: cert-manager.io/v1
    image: minio/kes:v0.17.6
    imagePullPolicy: IfNotPresent
    kesSecret:
      name: storage-certmanager-secret-kes-configuration
    keyName: my-minio-key
    replicas: 1
    resources: {}
    securityContext:
      fsGroup: 1000
      fsGroupChangePolicy: Always
      runAsGroup: 1000
      runAsNonRoot: true
      runAsUser: 1000
  mountPath: /export
  podManagementPolicy: Parallel
  pools:
  - affinity:
      podAntiAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchExpressions:
            - key: v1.min.io/tenant
              operator: In
              values:
              - storage-certmanager
            - key: v1.min.io/pool
              operator: In
              values:
              - pool-0
          topologyKey: kubernetes.io/hostname
    name: pool-0
    resources: {}
    servers: 4
    volumeClaimTemplate:
      metadata:
        name: data
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: \"26843545600\"
        storageClassName: standard
      status: {}
    volumesPerServer: 1
  requestAutoCert: false
  users:
  - name: storage-certmanager-user-0" > storage-certmanager.yaml
k apply -f storage-certmanager.yaml