create_workspace -name ws-gke

switch_workspace ws-gke

echo $PADOGRID_WORKSPACE 
/Users/dpark/Padogrid/workspaces/myrwe/ws-gke

create_k8s -k8s gke -cluster kustomize-test

kustomize-test
├── README-GKE.md
├── bin_sh
│   ├── create_certs
│   └── setenv.sh
└── etc
    ├── custom-metrics
    │   ├── base
    │   ├── custom-metrics-api
    │   ├── metrics-server
    │   ├── overlay-base
    │   └── prometheus
    └── hazelcast
        ├── base
        ├── init
        ├── overlay-base
        ├── overlay-nfs
        └── storage

# Edit $PADOGRID_WORKSPACE/k8s/kustomize-test/bin_sh/setenv.sh
vi $PADOGRID_WORKSPACE/k8s/kustomize-test/bin_sh/setenv.sh

# Set the following environment variables with your values.
export HAZELCAST_KUSTOM_DIR=$PADOGRID_WORKSPACE/k8s/kustomize-test
export CLUSTER_NAME=kustomize-test
export GCR_HOSTNAME=gcr.io
export PROJECT_ID=hazelcast-33
export REGION=us-east1
export ZONE=us-east1-b
export INSTANCE_NAME=gke-kustomize-test-default-pool-d2bd95b0-3jb5

# Source in setenv.sh
. $PADOGRID_WORKSPACE/k8s/kustomize-test/bin_sh/setenv.sh

gcloud config set project $PROJECT_ID
gcloud config set compute/region $REGION
gcloud config set compute/zone $ZONE

# Check the setttings
gcloud info

[GCR_HOSTNAME]/[PROJECT_ID]/[IMAGE]:[TAG]

# Authenticate to GCR. If this step is not ideal then follow the steps below.
gcloud auth configure-docker

# If the above step is not ideal then execute the following two commands instead:
gcloud components install docker-credential-gcr
docker-credential-gcr configure-docker

# Once you have docker credentials configured, you can 
# push Docker images to GCR.

# Push Hazelcast Enterprise image to GCR
docker pull hazelcast/hazelcast-enterprise:3.12.1
docker tag hazelcast/hazelcast-enterprise:3.12.1 $GCR_HOSTNAME/$PROJECT_ID/hazelcast/hazelcast-enterprise:3.12.1
docker push $GCR_HOSTNAME/$PROJECT_ID/hazelcast/hazelcast-enterprise:3.12.1

# To pull (not required, just an example)
# docker pull $GCR_HOSTNAME/$PROJECT_ID/hazelcast/hazelcast-enterprise:3.12.1

# Man Center to GCR
docker pull hazelcast/management-center:latest
docker tag hazelcast/management-center:latest $GCR_HOSTNAME/$PROJECT_ID/hazelcast/management-center:latest
docker push $GCR_HOSTNAME/$PROJECT_ID/hazelcast/management-center:latest

# To pull (not required, just an example)
# docker pull $GCR_HOSTNAME/$PROJECT_ID/hazelcast/management-center:latest

# To delete Hazelcast Enterprise image
gcloud container images delete $GCR_HOSTNAME/$PROJECT_ID/hazelcast/hazelcast-enterprise:3.12.1 --force-delete-tags

# To delete Man Center image
gcloud container images delete $GCR_HOSTNAME/$PROJECT_ID/hazelcast/management-center:latest --force-delete-tags

# Create cluster
gcloud container clusters create --zone=$ZONE --disk-type=pd-standard --machine-type=n1-standard-2 $CLUSTER_NAME

# Get authentication credentials
gcloud container clusters get-credentials $CLUSTER_NAME

# Create disk to store application specific library files
# gcloud compute disks create --size=[SIZE] --zone=[ZONE] [DISK_NAME]
gcloud compute disks create --size=1GB --zone=$ZONE gce-nfs-disk

# Verify the disk
gcloud compute disks list | grep gce-nfs-disk
gcloud compute disks describe gce-nfs-disk --zone=$ZONE

# List the VM instances for the cluster you just created (kustomize-test for this demo)
# and set the INSTANCE_NAME environment variable in setenv.sh.
# Make sure to source in setenv.sh afterwards.
gcloud compute instances list | grep $CLUSTER_NAME
vi $HAZELCAST_KUSTOM_DIR/bin_sh/setenv.sh
export INSTANCE_NAME=<a-node-name>
. $HAZELCAST_KUSTOM_DIR/bin_sh/setenv.sh

# Select one of the instances and attach the disk to it
# gcloud compute instances attach-disk --disk [DISK_NAME] [INSTANCE_NAME]
gcloud compute instances attach-disk --disk gce-nfs-disk $INSTANCE_NAME

# Login to the VM instance (node) that you attached the disk
# gcloud compute ssh --project [PROJECT_ID] --zone [ZONE] [INSTANCE_NAME]
gcloud compute ssh $INSTANCE_NAME

# Display disk information from the $INSTANCE_NAME (node) shell
sudo df -h
sudo lsblk

# Format the disk to ext4 
# sudo mkfs.ext4 -m 0 -F -E lazy_itable_init=0,lazy_journal_init=0,discard /dev/[DEVICE_ID]
sudo mkfs.ext4 -m 0 -F -E lazy_itable_init=0,lazy_journal_init=0,discard /dev/sdb

# Create a directory that serves as the mount point
# sudo mkdir -p /mnt/disks/[MNT_DIR]
sudo mkdir -p /mnt/disks/hazelcast

# Mount the disk
# sudo mount -o discard,defaults /dev/[DEVICE_ID] /mnt/disks/[MNT_DIR]
sudo mount -o discard,defaults /dev/sdb /mnt/disks/hazelcast

# Configure read/write permissions
sudo chmod a+w /mnt/disks/hazelcast

# Create the plugins directory where the application specific library (jar) 
# files will be stored.
mkdir -p /mnt/disks/hazelcast/data/custom/plugins/v1

# Unmount disk
sudo umount /mnt/disks/hazelcast

# Exit from the node shell
exit

cd $HAZELCAST_KUSTOM_DIR/etc

# Start NFS Server
kubectl apply -k hazelcast/storage/gke/init-nfs

# NFS server address is set to "nfs-server.default.svc.cluster.local".
# This demo uses the "default" namespace and therefore no changes are required.
# Change this value in nfs-pv.yaml only if your namespace is not "default".
vi hazelcast/overlay-nfs/nfs-pv.yaml

# Create the persistent volume and claim for the NFS disk
kubectl apply -k hazelcast/overlay-nfs

kubectl get pv
kubectl get pvc
kubectl get pod

# First, create a pod to which you will mount the disk
kubectl apply -k hazelcast/storage/gke/nfs-pod

# Copy the PADOGRID jar files 
# (Note: copy to /var/nfs/plugins/v1, NOT /data/custom/plugins/v1)
kubectl cp $PADOGRID_HOME/lib/PADOGRID-core-0.2.0-SNAPSHOT.jar nfs-pod:/var/nfs/plugins/v1/
kubectl cp $PADOGRID_HOME/plugins/PADOGRID-core-0.2.0-SNAPSHOT-tests.jar nfs-pod:/var/nfs/plugins/v1/

# Delete the pod
kubectl delete -k hazelcast/storage/gke/nfs-pod

# IMPORTANT: First, create TLS certificates for the Prometheus custom metrics API adapter
cd $HAZELCAST_KUSTOM_DIR/bin_sh
./create_certs

[GCR_HOSTNAME]/[PROJECT_ID]/[IMAGE]:[TAG]

cd $HAZELCAST_KUSTOM_DIR/etc

# Set license key and change Hazelcast Enterprise image name
vi hazelcast/overlay-base/statefulset.yaml

# In statefulset.yaml make the following changes
- name: HZ_LICENSE_KEY
  value: <License key goes here -- done by overlay-base>
image: gcr.io/hazelcast-33/hazelcast/hazelcast-enterprise:3.12.1

# Change Management Center image name
vi hazelcast/overlay-base/mc-statefulset.yaml

# In mc-statefulset.yaml make the following changes
- name: MC_LICENSE_KEY
  value: <License key goes here -- done by overlay-base>
image: gcr.io/hazelcast-33/hazelcast/management-center:latest

# Initialize Kubernetes cluster. This command configures a service account and RBAC.
kubectl apply -k hazelcast/init

# Deploy Hazelcast Enterprise
kubectl apply -k hazelcast/overlay-base

# Deploy custom metrics API and start Prometheus/HPA.
kubectl apply -k custom-metrics/overlay-base

# default namespace
watch kubectl get statefulsets
watch kubectl get pods

# monitoring namespace
watch kubectl get deployments --namespace=monitoring

kustomize-test
├── bin_sh
└── etc
    ├── custom-metrics
    │   ├── base
    │   ├── custom-metrics-api
    │   ├── metrics-server
    │   ├── overlay-base
    │   └── prometheus
    └── hazelcast
        ├── base
        ├── init
        ├── overlay-base
        ├── overlay-nfs
        └── storage
            ├── gke
                ├── init-nfs
                └── nfs

# Get external addresses of kustomize-test cluster nodes
gcloud compute instances list | grep kustomize-test

# Watch HPA
watch kubectl describe hpa my-release-hazelcast 

# Watch the on_heap-ratio metric
watch -d 'kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1/namespaces/default/pods/*/on_heap_ratio" |jq'

# Get the Management Center loadbalancer service external IP
kubectl get svc my-release-hazelcast-enterprise-mancenter

# Get the master URI
kubectl cluster-info

# List the screts
kubectl get secrets
NAME                     TYPE                                  DATA   AGE
default-token-hd5w2      kubernetes.io/service-account-token   3      43m
enterprise-token-2qdzz   kubernetes.io/service-account-token   3      41m

# Use the token name that starts with the prefix "enterprise-token-" to get api-token
# and ca-certificate
# Get api-token
kubectl get secret enterprise-token-2qdzz  -o jsonpath={.data.token} | base64 --decode
# Get ca-certificate
kubectl get secret enterprise-token-2qdzz  -o jsonpath={.data.ca\\.crt} | base64 --decode

# Monitor HPA.
watch kubectl describe hpa my-release-hazelcast

Name:               my-release-hazelcast
Namespace:          default
Labels:             <none>
Annotations:        autoscaling.alpha.kubernetes.io/conditions:
                      [{"type":"AbleToScale","status":"True","lastTransitionTime":"2019-09-28T13:31:32Z","reason":"Re
adyForNewScale","message":"recommended size...
                    autoscaling.alpha.kubernetes.io/current-metrics: [{"type":"Pods","pods":{"metricName":"on_heap_ra
tio","currentAverageValue":"99m"}}]
                    autoscaling.alpha.kubernetes.io/metrics: [{"type":"Pods","pods":{"metricName":"on_heap_ratio","ta
rgetAverageValue":"850m"}}]
                    kubectl.kubernetes.io/last-applied-configuration:
                      {"apiVersion":"autoscaling/v2beta1","kind":"HorizontalPodAutoscaler","metadata":{"annotations":
{},"name":"my-release-hazelcast","namespace...
CreationTimestamp:  Sat, 28 Sep 2019 09:31:17 -0400
Reference:          StatefulSet/my-release-hazelcast-enterprise
Min replicas:       2
Max replicas:       10
StatefulSet pods:   2 current / 2 desired

"pods":{"metricName":"on_heap_ratio","currentAverageValue":"99m"}
"pods":{"metricName":"on_heap_ratio","targetAverageValue":"850m"}

cd $PADOGRID_WORKSPACE/apps/perf_test/bin_sh
vi ../etc/hazelcast-client.xml

# After hazelcast-client.xml has been configured, run the following command to ingest data
./test_ingestion -run

StatefulSet pods:   3 current / 3 desired

# Uninstall custom metrics and Hazelcast
kubectl delete -k custom-metrics/overlay-base
kubectl delete -k hazelcast/overlay-base
kubectl delete -k hazelcast/storage/gke/nfs-pod
kubectl delete -k hazelcast/overlay-nfs
kubectl delete -k hazelcast/init
kubectl delete -k hazelcast/storage/gke/init-nfs

# Delete disk
gcloud compute disks delete gce-nfs-disk

# Delete Kuternetes cluster
gcloud container clusters delete kustomize-test