$ sudo wget https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
$ sudo chmod +x minikube-linux-amd64
$ sudo mv minikube-linux-amd64 /usr/local/bin/minikube
$ minikube start

$ sudo curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl 
$ sudo chmod +x kubectl
$ sudo mv kubectl /usr/local/bin/

$ kubectl cluster-info
Kubernetes control plane is running at https://192.168.49.2:8443

$ minikube start --nodes 3 -p multinode-demo
$ kubectl get nodes
NAME                 STATUS   ROLES           AGE     VERSION
multinode-demo       Ready    control-plane   2m42s   v1.27.4
multinode-demo-m02   Ready    <none>          88s     v1.27.4
multinode-demo-m03   Ready    <none>          27s     v1.27.4

$ kubectl create secret docker-registry [secret 이름] --docker-server=[사설 레지스트리 주소 (포트 포함) 도메인의 경우 도메인만 입력] --docker-username=[docker ID] --docker-password=[docker Password]

[예제]
$ kubectl create secret docker-registry dongtest --docker-server=10.0.2.5:5000 --docker-username=iksoon --docker-password=qwer1234

[명령어]
$ kubectl get secret
$ kubectl describe secret
$ kubectl get secret dongtest --output=yaml

apiVersion: v1
data:
  .dockerconfigjson: eyJhdXRocyI6eyJkb25nZGV2cmVzaXN0cnkuYXp1cmVjci5pbyI6eyJ1c2VybmFtZSI6ImRvbmdkZXZyZXNpc3RyeSIsInBhc3N3b3JkIjoiQitvQVUyUE8wZnRxS3pINnlwSldVL0Q3Q1FHaUIwU1JEZ0dtK0tBQk1QK0FDUkRRWmFNWiIsImF1dGgiOiJaRzl1WjJSbGRuSmxjMmx6ZEhKNU9rSXJiMEZWTWxCUE1HWjBjVXQ2U0RaNWNFcFhWUzlFTjBOUlIybENNRk5TUkdkSGJTdExRVUpOVUN0QlExSkVVVnBoVFZvPSJ9fX0=
kind: Secret
metadata:
  creationTimestamp: "2023-09-20T01:58:04Z"
  name: dongtest
  namespace: default
  uid: 07bbab38-ec35-43e5-8d2b-c063cf0d43d9
type: kubernetes.io/dockerconfigjson

---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: dong-service
spec:
  replicas: 2
  selector:
    matchLabels:
      app: dong-service
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
      maxSurge: 1
  template:
    metadata:
      labels:
        app: dong-service
    spec:
      imagePullSecrets:
        - name: dongtest
      containers:
        - name: dong-service
          image: dongdevresistry.azurecr.io/dong-service:${build_number}_${CI_PIPELINE_ID}_${CI_COMMIT_SHORT_SHA} # Azure Container Registry에서 가져올 이미지의 경로와 태그를 지정.
          ports:
            - containerPort: 8080

stages:
  - build         
  - deploy
build:     
  stage: build
  before_script:
    - source /home/gitlab-runner/jdk/OpenJDK17.0.2.sh
    - source /home/gitlab-runner/gradle/Gradle-8.0.sh
    - bash /home/gitlab-runner/repo/ACR.sh
  script:
    - echo [Variable 선언]
    - docker_registry=dongdevresistry.azurecr.io
    - artifactId=dong-service
    - export build_number="$(date +"%Y%m%d")"
    - gradle clean build
    - sleep 3
    - docker build -t $docker_registry/$artifactId:"$build_number"_"$CI_PIPELINE_ID"_"$CI_COMMIT_SHORT_SHA" .
    - docker tag $docker_registry/$artifactId:"$build_number"_"$CI_PIPELINE_ID"_"$CI_COMMIT_SHORT_SHA" $docker_registry/$artifactId:latest 
    - sleep 3
    - docker push $docker_registry/$artifactId:"$build_number"_"$CI_PIPELINE_ID"_"$CI_COMMIT_SHORT_SHA"
    - docker push $docker_registry/$artifactId:latest
    - docker rmi -f $(docker images --format "{{.ID}} {{.Repository}}" | grep "$artifactId" | awk '{print $1}') ### docker 모든 이미지 삭제
    - cd kubeyaml
    - envsubst < $artifactId.yaml > "$artifactId"_deploy.yaml ### gitlab 변수 kubernetes yaml에 적용
  parallel:
    matrix:
      - RUNNER: Azurevm_k8s
  tags:
    - ${RUNNER}
  only:
    - main
  when: manual

deploy:
  stage: deploy
  variables:
    GIT_STRATEGY: none
  before_script:
    - bash /home/gitlab-runner/repo/ACR.sh
  script:
    - echo [Variable 선언]
    - artifactId=dong-service
    - echo [Azure Kubernetes 배포]
    - cd kubeyaml
    - kubectl apply -f "$artifactId"_deploy.yaml
    - sleep 3
    - kubectl rollout status deployment $artifactId
  parallel:
    matrix:
      - RUNNER: Azurevm_k8s
  tags:
    - ${RUNNER}
  only:
    - main
  when: manual  

deploy:
  stage: deploy
  cache:
    paths:
      - yml/  
  before_script:
    - az login --service-principal -u "28bf8f6c-fc25-4b48-96e7-7ec3f394a78b" -p "8qJ8Q~6Yyliz~kX4oh0JJBLFROqoZRaQsTGRnbuL" --tenant "5643e260-846d-40a2-9efb-a9a19f5c6907"
    - az aks get-credentials --resource-group Develop --name aks-sumitsplatform-dev-001
  script:
    - echo [Variable 선언]
    - artifactId=dong-jdk17-gradle8
    - kuberArtifactId=dong-jdk17-gradle8
    - echo [Azure Kubernetes 배포]
    - cd kubeyml
    - kubectl apply -f 2."$artifactId"_deploy.yaml
    - sleep 3
    - kubectl rollout status deployment $kuberArtifactId
  after_script:
    - az logout
    - if [ -f ~/.kube/config ]; then  rm -rf ~/.kube/config; fi
  parallel:
    matrix:
      - RUNNER: Azurevm_k8s
  tags:
    - ${RUNNER}
  only:
    - main
  when: manual