kubectl get namespaces monitor > /dev/null 2>&1 || kubectl create namespace monitor
kubectl -n monitor create secret generic prometheus-stack-credentials \
  --from-literal=grafana-username=admin \
  --from-literal=grafana-password=$(tr -dc A-Za-z0-9 </dev/urandom | head -c 16)

  apiVersion: argoproj.io/v1alpha1
  kind: Application
  metadata:
    name: prometheus-stack
  spec:
    syncPolicy:
      syncOptions:
        - CreateNamespace=true
        - ServerSideApply=true
    project: default
    source:
      repoURL: https://aaronyang0628.github.io/helm-chart-mirror/charts
      chart: kube-prometheus-stack
      targetRevision: 72.6.2
      helm:
        releaseName: prometheus-stack
        values: |
          crds:
            enabled: true
          global:
            rbac:
              create: true
            imageRegistry: ""
            imagePullSecrets: []
          alertmanager:
            enabled: true
            ingress:
              enabled: false
            serviceMonitor:
              selfMonitor: true
              interval: ""
            alertmanagerSpec:
              image:
                registry: m.daocloud.io/quay.io
                repository: prometheus/alertmanager
                tag: v0.28.1
              replicas: 1
              resources: {}
              storage:
                volumeClaimTemplate:
                  spec:
                    storageClassName: ""
                    accessModes: ["ReadWriteOnce"]
                    resources:
                      requests:
                        storage: 2Gi
          grafana:
            enabled: true
            ingress:
              enabled: true
              annotations:
                cert-manager.io/clusterissuer: self-signed-issuer
                kubernetes.io/ingress.class: nginx
              hosts:
                - grafana.dev.tech
              path: /
              pathtype: ImplementationSpecific
              tls:
              - secretName: grafana.dev.tech-tls
                hosts:
                - grafana.dev.tech
          prometheusOperator:
            admissionWebhooks:
              patch:
                resources: {}
                image:
                  registry: m.daocloud.io/registry.k8s.io
                  repository: ingress-nginx/kube-webhook-certgen
                  tag: v1.5.3  
            image:
              registry: m.daocloud.io/quay.io
              repository: prometheus-operator/prometheus-operator
            prometheusConfigReloader:
              image:
                registry: m.daocloud.io/quay.io
                repository: prometheus-operator/prometheus-config-reloader
              resources: {}
            thanosImage:
              registry: m.daocloud.io/quay.io
              repository: thanos/thanos
              tag: v0.38.0
          prometheus:
            enabled: true
            ingress:
              enabled: true
              annotations:
                cert-manager.io/clusterissuer: self-signed-issuer
                kubernetes.io/ingress.class: nginx
              hosts:
                - prometheus.dev.tech
              path: /
              pathtype: ImplementationSpecific
              tls:
              - secretName: prometheus.dev.tech-tls
                hosts:
                - prometheus.dev.tech
            prometheusSpec:
              image:
                registry: m.daocloud.io/quay.io
                repository: prometheus/prometheus
                tag: v3.4.0
              replicas: 1
              shards: 1
              resources: {}
              storageSpec: 
                volumeClaimTemplate:
                  spec:
                    storageClassName: ""
                    accessModes: ["ReadWriteOnce"]
                    resources:
                      requests:
                        storage: 2Gi
          thanosRuler:
            enabled: false
            ingress:
              enabled: false
            thanosRulerSpec:
              replicas: 1
              storage: {}
              resources: {}
              image:
                registry: m.daocloud.io/quay.io
                repository: thanos/thanos
                tag: v0.38.0
    destination:
      server: https://kubernetes.default.svc
      namespace: monitor

  kubectl -n argocd apply -f prometheus-stack.yaml

  argocd app sync argocd/prometheus-stack

  kubectl -n monitor get secret prometheus-stack-credentials -o jsonpath='{.data.grafana-password}' | base64 -d

  > add `$K8S_MASTER_IP grafana.dev.tech` to **/etc/hosts**

  > add `$K8S_MASTER_IP prometheus.dev.tech` to **/etc/hosts**