BuckUp

ES [Local Disk]

Preliminary

ElasticSearch has installed, if not check link

The elasticsearch.yml has configed path.repo, which should be set the same value of settings.location (this will be handled by helm chart, dont worry)

ES argocd-app yaml

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: elastic-search
spec:
  syncPolicy:
    syncOptions:
    - CreateNamespace=true
  project: default
  source:
    repoURL: https://charts.bitnami.com/bitnami
    chart: elasticsearch
    targetRevision: 19.11.3
    helm:
      releaseName: elastic-search
      values: |
        global:
          kibanaEnabled: true
        clusterName: elastic
        image:
          registry: m.zjvis.net/docker.io
          pullPolicy: IfNotPresent
        security:
          enabled: false
        service:
          type: ClusterIP
        extraConfig:
          path:
            repo: /tmp
        ingress:
          enabled: true
          annotations:
            cert-manager.io/cluster-issuer: self-signed-ca-issuer
            nginx.ingress.kubernetes.io/rewrite-target: /$1
          hostname: elastic-search.dev.tech
          ingressClassName: nginx
          path: /?(.*)
          tls: true
        master:
          masterOnly: false
          replicaCount: 1
          persistence:
            enabled: false
          resources:
            requests:
              cpu: 2
              memory: 1024Mi
            limits:
              cpu: 4
              memory: 4096Mi
          heapSize: 2g
        data:
          replicaCount: 0
          persistence:
            enabled: false
        coordinating:
          replicaCount: 0
        ingest:
          enabled: true
          replicaCount: 0
          service:
            enabled: false
            type: ClusterIP
          ingress:
            enabled: false
        metrics:
          enabled: false
          image:
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
        volumePermissions:
          enabled: false
          image:
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
        sysctlImage:
          enabled: true
          registry: m.zjvis.net/docker.io
          pullPolicy: IfNotPresent
        kibana:
          elasticsearch:
            hosts:
              - '{{ include "elasticsearch.service.name" . }}'
            port: '{{ include "elasticsearch.service.ports.restAPI" . }}'
        esJavaOpts: "-Xmx2g -Xms2g"        
  destination:
    server: https://kubernetes.default.svc
    namespace: application

diff from oirginal file :

extraConfig:
    path:
      repo: /tmp

Methods

Elasticsearch 做备份有两种方式，

是将数据导出成文本文件，比如通过elasticdump、esm等工具将存储在 Elasticsearch 中的数据导出到文件中。
是使用snapshot接口实现快照功能，增量备份文件

第一种方式相对简单，在数据量小的时候比较实用，但当应对大数据量场景时，更推荐使用snapshot api 的方式。

Steps

buckup

asdadas

创建快照仓库repo -> my_fs_repository

curl -k -X PUT "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository?pretty" -H 'Content-Type: application/json' -d'
{
  "type": "fs",
  "settings": {
    "location": "/tmp"
  }
}
'

你也能使用storage-class 挂载一个路径在pod中，将snapshot文件存放在外挂路径上

验证集群各个节点是否可以使用这个快照仓库repo

curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository/_verify?pretty"

查看快照仓库repo

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/_all?pretty"

查看某一个快照仓库repo的具体setting

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository?pretty"

分析一个快照仓库repo

curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository/_analyze?blob_count=10&max_blob_size=1mb&timeout=120s&pretty"

手动打快照

curl -k -X PUT "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository/ay_snap_02?pretty"

查看指定快照仓库repo 可用的快照

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository/*?verbose=false&pretty"

测试恢复

# Delete an index
curl -k -X DELETE "https://elastic-search.dev.tech:32443/books?pretty"

# restore that index
curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository/ay_snap_02/_restore?pretty" -H 'Content-Type: application/json' -d'
{
  "indices": "books"
}
'

# query
curl -k -X GET "https://elastic-search.dev.tech:32443/books/_search?pretty" -H 'Content-Type: application/json' -d'
{
  "query": {
    "match_all": {}
  }
}
'

ES [S3 Compatible]

Preliminary

ElasticSearch has installed, if not check link

ES argocd-app yaml

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: elastic-search
spec:
  syncPolicy:
    syncOptions:
    - CreateNamespace=true
  project: default
  source:
    repoURL: https://charts.bitnami.com/bitnami
    chart: elasticsearch
    targetRevision: 19.11.3
    helm:
      releaseName: elastic-search
      values: |
        global:
          kibanaEnabled: true
        clusterName: elastic
        image:
          registry: m.zjvis.net/docker.io
          pullPolicy: IfNotPresent
        security:
          enabled: true
        service:
          type: ClusterIP
        extraEnvVars:
        - name: S3_ACCESSKEY
          value: admin
        - name: S3_SECRETKEY
          value: ZrwpsezF1Lt85dxl
        extraConfig:
          s3:
            client:
              default:
                protocol: http
                endpoint: "http://192.168.31.111:9090"
                path_style_access: true
        initScripts:
          configure-s3-client.sh: |
            elasticsearch_set_key_value "s3.client.default.access_key" "${S3_ACCESSKEY}"
            elasticsearch_set_key_value "s3.client.default.secret_key" "${S3_SECRETKEY}"
        hostAliases:
        - ip: 192.168.31.111
          hostnames:
          - minio-api.dev.tech
        ingress:
          enabled: true
          annotations:
            cert-manager.io/cluster-issuer: self-signed-ca-issuer
            nginx.ingress.kubernetes.io/rewrite-target: /$1
          hostname: elastic-search.dev.tech
          ingressClassName: nginx
          path: /?(.*)
          tls: true
        master:
          masterOnly: false
          replicaCount: 1
          persistence:
            enabled: false
          resources:
            requests:
              cpu: 2
              memory: 1024Mi
            limits:
              cpu: 4
              memory: 4096Mi
          heapSize: 2g
        data:
          replicaCount: 0
          persistence:
            enabled: false
        coordinating:
          replicaCount: 0
        ingest:
          enabled: true
          replicaCount: 0
          service:
            enabled: false
            type: ClusterIP
          ingress:
            enabled: false
        metrics:
          enabled: false
          image:
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
        volumePermissions:
          enabled: false
          image:
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
        sysctlImage:
          enabled: true
          registry: m.zjvis.net/docker.io
          pullPolicy: IfNotPresent
        kibana:
          elasticsearch:
            hosts:
              - '{{ include "elasticsearch.service.name" . }}'
            port: '{{ include "elasticsearch.service.ports.restAPI" . }}'
        esJavaOpts: "-Xmx2g -Xms2g"        
  destination:
    server: https://kubernetes.default.svc
    namespace: application

diff from oirginal file :

extraEnvVars:
- name: S3_ACCESSKEY
  value: admin
- name: S3_SECRETKEY
  value: ZrwpsezF1Lt85dxl
extraConfig:
  s3:
    client:
      default:
        protocol: http
        endpoint: "http://192.168.31.111:9090"
        path_style_access: true
initScripts:
  configure-s3-client.sh: |
    elasticsearch_set_key_value "s3.client.default.access_key" "${S3_ACCESSKEY}"
    elasticsearch_set_key_value "s3.client.default.secret_key" "${S3_SECRETKEY}"
hostAliases:
- ip: 192.168.31.111
  hostnames:
  - minio-api.dev.tech

Methods

Elasticsearch 做备份有两种方式，

是将数据导出成文本文件，比如通过elasticdump、esm等工具将存储在 Elasticsearch 中的数据导出到文件中。
是使用snapshot接口实现快照功能，增量备份文件

第一种方式相对简单，在数据量小的时候比较实用，但当应对大数据量场景时，更推荐使用snapshot api 的方式。

Steps

buckup

asdadas

创建快照仓库repo -> my_s3_repository

curl -k -X PUT "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository?pretty" -H 'Content-Type: application/json' -d'
{
  "type": "s3",
  "settings": {
    "bucket": "local-test",
    "client": "default",
    "endpoint": "http://192.168.31.111:9000"
  }
}
'

你也能使用storage-class 挂载一个路径在pod中，将snapshot文件存放在外挂路径上

验证集群各个节点是否可以使用这个快照仓库repo

curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository/_verify?pretty"

查看快照仓库repo

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/_all?pretty"

查看某一个快照仓库repo的具体setting

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository?pretty"

分析一个快照仓库repo

curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository/_analyze?blob_count=10&max_blob_size=1mb&timeout=120s&pretty"

手动打快照

curl -k -X PUT "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository/ay_s3_snap_02?pretty"

查看指定快照仓库repo 可用的快照

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository/*?verbose=false&pretty"

测试恢复

# Delete an index
curl -k -X DELETE "https://elastic-search.dev.tech:32443/books?pretty"

# restore that index
curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository/ay_s3_snap_02/_restore?pretty" -H 'Content-Type: application/json' -d'
{
  "indices": "books"
}
'

# query
curl -k -X GET "https://elastic-search.dev.tech:32443/books/_search?pretty" -H 'Content-Type: application/json' -d'
{
  "query": {
    "match_all": {}
  }
}
'

ES Auto BackUp

Preliminary

ElasticSearch has installed, if not check link
We use local disk to save the snapshots, more deatils check link

And the security is enabled.

ES argocd-app yaml

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: elastic-search
spec:
  syncPolicy:
    syncOptions:
    - CreateNamespace=true
  project: default
  source:
    repoURL: https://charts.bitnami.com/bitnami
    chart: elasticsearch
    targetRevision: 19.11.3
    helm:
      releaseName: elastic-search
      values: |
        global:
          kibanaEnabled: true
        clusterName: elastic
        image:
          registry: m.zjvis.net/docker.io
          pullPolicy: IfNotPresent
        security:
          enabled: true
          tls:
            autoGenerated: true
        service:
          type: ClusterIP
        extraConfig:
          path:
            repo: /tmp
        ingress:
          enabled: true
          annotations:
            cert-manager.io/cluster-issuer: self-signed-ca-issuer
            nginx.ingress.kubernetes.io/rewrite-target: /$1
          hostname: elastic-search.dev.tech
          ingressClassName: nginx
          path: /?(.*)
          tls: true
        master:
          masterOnly: false
          replicaCount: 1
          persistence:
            enabled: false
          resources:
            requests:
              cpu: 2
              memory: 1024Mi
            limits:
              cpu: 4
              memory: 4096Mi
          heapSize: 2g
        data:
          replicaCount: 0
          persistence:
            enabled: false
        coordinating:
          replicaCount: 0
        ingest:
          enabled: true
          replicaCount: 0
          service:
            enabled: false
            type: ClusterIP
          ingress:
            enabled: false
        metrics:
          enabled: false
          image:
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
        volumePermissions:
          enabled: false
          image:
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
        sysctlImage:
          enabled: true
          registry: m.zjvis.net/docker.io
          pullPolicy: IfNotPresent
        kibana:
          elasticsearch:
            hosts:
              - '{{ include "elasticsearch.service.name" . }}'
            port: '{{ include "elasticsearch.service.ports.restAPI" . }}'
        esJavaOpts: "-Xmx2g -Xms2g"        
  destination:
    server: https://kubernetes.default.svc
    namespace: application

diff from oirginal file :

security:
  enabled: true
extraConfig:
    path:
      repo: /tmp

Methods

Steps

auto buckup

创建快照仓库repo -> slm_fs_repository

curl --user elastic:L9shjg6csBmPZgCZ -k -X PUT "https://10.88.0.143:30294/_snapshot/slm_fs_repository?pretty" -H 'Content-Type: application/json' -d'
{
  "type": "fs",
  "settings": {
    "location": "/tmp"
  }
}
'

你也能使用storage-class 挂载一个路径在pod中，将snapshot文件存放在外挂路径上

验证集群各个节点是否可以使用这个快照仓库repo

curl --user elastic:L9shjg6csBmPZgCZ  -k -X POST "https://10.88.0.143:30294/_snapshot/slm_fs_repository/_verify?pretty"

查看快照仓库repo

curl --user elastic:L9shjg6csBmPZgCZ  -k -X GET "https://10.88.0.143:30294/_snapshot/_all?pretty"

查看某一个快照仓库repo的具体setting

curl --user elastic:L9shjg6csBmPZgCZ  -k -X GET "https://10.88.0.143:30294/_snapshot/slm_fs_repository?pretty"

分析一个快照仓库repo

curl --user elastic:L9shjg6csBmPZgCZ  -k -X POST "https://10.88.0.143:30294/_snapshot/slm_fs_repository/_analyze?blob_count=10&max_blob_size=1mb&timeout=120s&pretty"

查看指定快照仓库repo 可用的快照

curl --user elastic:L9shjg6csBmPZgCZ  -k -X GET "https://10.88.0.143:30294/_snapshot/slm_fs_repository/*?verbose=false&pretty"

创建SLM admin 角色

curl --user elastic:L9shjg6csBmPZgCZ -k -X POST "https://10.88.0.143:30294/_security/role/slm-admin?pretty" -H 'Content-Type: application/json' -d'
{
  "cluster": [ "manage_slm", "cluster:admin/snapshot/*" ],
  "indices": [
    {
      "names": [ ".slm-history-*" ],
      "privileges": [ "all" ]
    }
  ]
}
'

创建自动备份cornjob

curl --user elastic:L9shjg6csBmPZgCZ -k -X PUT "https://10.88.0.143:30294/_slm/policy/nightly-snapshots?pretty" -H 'Content-Type: application/json' -d'
{
  "schedule": "0 30 1 * * ?",       
  "name": "<nightly-snap-{now/d}>", 
  "repository": "slm_fs_repository",    
  "config": {
    "indices": "*",                 
    "include_global_state": true    
  },
  "retention": {                    
    "expire_after": "30d",
    "min_count": 5,
    "max_count": 50
  }
}
'

启动自动备份

curl --user elastic:L9shjg6csBmPZgCZ -k -X POST "https://10.88.0.143:30294/_slm/policy/nightly-snapshots/_execute?pretty"

查看SLM备份历史

curl --user elastic:L9shjg6csBmPZgCZ -k -X GET "https://10.88.0.143:30294/_slm/stats?pretty"

测试恢复

# Delete an index
curl --user elastic:L9shjg6csBmPZgCZ  -k -X DELETE "https://10.88.0.143:30294/books?pretty"

# restore that index
curl --user elastic:L9shjg6csBmPZgCZ  -k -X POST "https://10.88.0.143:30294/_snapshot/slm_fs_repository/my_snapshot_2099.05.06/_restore?pretty" -H 'Content-Type: application/json' -d'
{
  "indices": "books"
}
'

# query
curl --user elastic:L9shjg6csBmPZgCZ  -k -X GET "https://10.88.0.143:30294/books/_search?pretty" -H 'Content-Type: application/json' -d'
{
  "query": {
    "match_all": {}
  }
}
'

BuckUp

Subsections of BuckUp

ES [Local Disk]

Preliminary

Methods

Steps

ES [S3 Compatible]

Preliminary

Methods

Steps

ES Auto BackUp

Preliminary

Methods

Steps