ES [Local Disk]

Preliminary

• ElasticSearch has installed, if not check link

• The elasticsearch.yml has configed path.repo, which should be set the same value of settings.location (this will be handled by helm chart, dont worry)

  ES argocd-app yaml

  apiVersion: argoproj.io/v1alpha1
  kind: Application
  metadata:
    name: elastic-search
  spec:
    syncPolicy:
      syncOptions:
      - CreateNamespace=true
    project: default
    source:
      repoURL: https://charts.bitnami.com/bitnami
      chart: elasticsearch
      targetRevision: 19.11.3
      helm:
        releaseName: elastic-search
        values: |
          global:
            kibanaEnabled: true
          clusterName: elastic
          image:
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
          security:
            enabled: false
          service:
            type: ClusterIP
          extraConfig:
            path:
              repo: /tmp
          ingress:
            enabled: true
            annotations:
              cert-manager.io/cluster-issuer: self-signed-ca-issuer
              nginx.ingress.kubernetes.io/rewrite-target: /$1
            hostname: elastic-search.dev.tech
            ingressClassName: nginx
            path: /?(.*)
            tls: true
          master:
            masterOnly: false
            replicaCount: 1
            persistence:
              enabled: false
            resources:
              requests:
                cpu: 2
                memory: 1024Mi
              limits:
                cpu: 4
                memory: 4096Mi
            heapSize: 2g
          data:
            replicaCount: 0
            persistence:
              enabled: false
          coordinating:
            replicaCount: 0
          ingest:
            enabled: true
            replicaCount: 0
            service:
              enabled: false
              type: ClusterIP
            ingress:
              enabled: false
          metrics:
            enabled: false
            image:
              registry: m.zjvis.net/docker.io
              pullPolicy: IfNotPresent
          volumePermissions:
            enabled: false
            image:
              registry: m.zjvis.net/docker.io
              pullPolicy: IfNotPresent
          sysctlImage:
            enabled: true
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
          kibana:
            elasticsearch:
              hosts:
                - '{{ include "elasticsearch.service.name" . }}'
              port: '{{ include "elasticsearch.service.ports.restAPI" . }}'
          esJavaOpts: "-Xmx2g -Xms2g"        
    destination:
      server: https://kubernetes.default.svc
      namespace: application

  diff from oirginal file :

  extraConfig:
      path:
        repo: /tmp

Methods

Elasticsearch 做备份有两种方式，

1. 是将数据导出成文本文件，比如通过elasticdump、esm等工具将存储在 Elasticsearch 中的数据导出到文件中。
2. 是使用snapshot接口实现快照功能，增量备份文件

第一种方式相对简单，在数据量小的时候比较实用，但当应对大数据量场景时，更推荐使用snapshot api 的方式。

Steps

curl -k -X PUT "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository?pretty" -H 'Content-Type: application/json' -d'
{
  "type": "fs",
  "settings": {
    "location": "/tmp"
  }
}
'

curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository/_verify?pretty"

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/_all?pretty"

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository?pretty"

curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository/_analyze?blob_count=10&max_blob_size=1mb&timeout=120s&pretty"

curl -k -X PUT "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository/ay_snap_02?pretty"

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository/*?verbose=false&pretty"

# Delete an index
curl -k -X DELETE "https://elastic-search.dev.tech:32443/books?pretty"

# restore that index
curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository/ay_snap_02/_restore?pretty" -H 'Content-Type: application/json' -d'
{
  "indices": "books"
}
'

# query
curl -k -X GET "https://elastic-search.dev.tech:32443/books/_search?pretty" -H 'Content-Type: application/json' -d'
{
  "query": {
    "match_all": {}
  }
}
'

ES [S3 Compatible]

Preliminary

• ElasticSearch has installed, if not check link

  ES argocd-app yaml

  apiVersion: argoproj.io/v1alpha1
  kind: Application
  metadata:
    name: elastic-search
  spec:
    syncPolicy:
      syncOptions:
      - CreateNamespace=true
    project: default
    source:
      repoURL: https://charts.bitnami.com/bitnami
      chart: elasticsearch
      targetRevision: 19.11.3
      helm:
        releaseName: elastic-search
        values: |
          global:
            kibanaEnabled: true
          clusterName: elastic
          image:
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
          security:
            enabled: true
          service:
            type: ClusterIP
          extraEnvVars:
          - name: S3_ACCESSKEY
            value: admin
          - name: S3_SECRETKEY
            value: ZrwpsezF1Lt85dxl
          extraConfig:
            s3:
              client:
                default:
                  protocol: http
                  endpoint: "http://192.168.31.111:9090"
                  path_style_access: true
          initScripts:
            configure-s3-client.sh: |
              elasticsearch_set_key_value "s3.client.default.access_key" "${S3_ACCESSKEY}"
              elasticsearch_set_key_value "s3.client.default.secret_key" "${S3_SECRETKEY}"
          hostAliases:
          - ip: 192.168.31.111
            hostnames:
            - minio-api.dev.tech
          ingress:
            enabled: true
            annotations:
              cert-manager.io/cluster-issuer: self-signed-ca-issuer
              nginx.ingress.kubernetes.io/rewrite-target: /$1
            hostname: elastic-search.dev.tech
            ingressClassName: nginx
            path: /?(.*)
            tls: true
          master:
            masterOnly: false
            replicaCount: 1
            persistence:
              enabled: false
            resources:
              requests:
                cpu: 2
                memory: 1024Mi
              limits:
                cpu: 4
                memory: 4096Mi
            heapSize: 2g
          data:
            replicaCount: 0
            persistence:
              enabled: false
          coordinating:
            replicaCount: 0
          ingest:
            enabled: true
            replicaCount: 0
            service:
              enabled: false
              type: ClusterIP
            ingress:
              enabled: false
          metrics:
            enabled: false
            image:
              registry: m.zjvis.net/docker.io
              pullPolicy: IfNotPresent
          volumePermissions:
            enabled: false
            image:
              registry: m.zjvis.net/docker.io
              pullPolicy: IfNotPresent
          sysctlImage:
            enabled: true
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
          kibana:
            elasticsearch:
              hosts:
                - '{{ include "elasticsearch.service.name" . }}'
              port: '{{ include "elasticsearch.service.ports.restAPI" . }}'
          esJavaOpts: "-Xmx2g -Xms2g"        
    destination:
      server: https://kubernetes.default.svc
      namespace: application

  diff from oirginal file :

  extraEnvVars:
  - name: S3_ACCESSKEY
    value: admin
  - name: S3_SECRETKEY
    value: ZrwpsezF1Lt85dxl
  extraConfig:
    s3:
      client:
        default:
          protocol: http
          endpoint: "http://192.168.31.111:9090"
          path_style_access: true
  initScripts:
    configure-s3-client.sh: |
      elasticsearch_set_key_value "s3.client.default.access_key" "${S3_ACCESSKEY}"
      elasticsearch_set_key_value "s3.client.default.secret_key" "${S3_SECRETKEY}"
  hostAliases:
  - ip: 192.168.31.111
    hostnames:
    - minio-api.dev.tech

Methods

Elasticsearch 做备份有两种方式，

1. 是将数据导出成文本文件，比如通过elasticdump、esm等工具将存储在 Elasticsearch 中的数据导出到文件中。
2. 是使用snapshot接口实现快照功能，增量备份文件

第一种方式相对简单，在数据量小的时候比较实用，但当应对大数据量场景时，更推荐使用snapshot api 的方式。

Steps

curl -k -X PUT "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository?pretty" -H 'Content-Type: application/json' -d'
{
  "type": "s3",
  "settings": {
    "bucket": "local-test",
    "client": "default",
    "endpoint": "http://192.168.31.111:9000"
  }
}
'

curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository/_verify?pretty"

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/_all?pretty"

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository?pretty"

curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository/_analyze?blob_count=10&max_blob_size=1mb&timeout=120s&pretty"

curl -k -X PUT "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository/ay_s3_snap_02?pretty"

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository/*?verbose=false&pretty"

# Delete an index
curl -k -X DELETE "https://elastic-search.dev.tech:32443/books?pretty"

# restore that index
curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository/ay_s3_snap_02/_restore?pretty" -H 'Content-Type: application/json' -d'
{
  "indices": "books"
}
'

# query
curl -k -X GET "https://elastic-search.dev.tech:32443/books/_search?pretty" -H 'Content-Type: application/json' -d'
{
  "query": {
    "match_all": {}
  }
}
'

ES Auto BackUp

Preliminary

• ElasticSearch has installed, if not check link

• We use local disk to save the snapshots, more deatils check link

• And the security is enabled.

  ES argocd-app yaml

  apiVersion: argoproj.io/v1alpha1
  kind: Application
  metadata:
    name: elastic-search
  spec:
    syncPolicy:
      syncOptions:
      - CreateNamespace=true
    project: default
    source:
      repoURL: https://charts.bitnami.com/bitnami
      chart: elasticsearch
      targetRevision: 19.11.3
      helm:
        releaseName: elastic-search
        values: |
          global:
            kibanaEnabled: true
          clusterName: elastic
          image:
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
          security:
            enabled: true
            tls:
              autoGenerated: true
          service:
            type: ClusterIP
          extraConfig:
            path:
              repo: /tmp
          ingress:
            enabled: true
            annotations:
              cert-manager.io/cluster-issuer: self-signed-ca-issuer
              nginx.ingress.kubernetes.io/rewrite-target: /$1
            hostname: elastic-search.dev.tech
            ingressClassName: nginx
            path: /?(.*)
            tls: true
          master:
            masterOnly: false
            replicaCount: 1
            persistence:
              enabled: false
            resources:
              requests:
                cpu: 2
                memory: 1024Mi
              limits:
                cpu: 4
                memory: 4096Mi
            heapSize: 2g
          data:
            replicaCount: 0
            persistence:
              enabled: false
          coordinating:
            replicaCount: 0
          ingest:
            enabled: true
            replicaCount: 0
            service:
              enabled: false
              type: ClusterIP
            ingress:
              enabled: false
          metrics:
            enabled: false
            image:
              registry: m.zjvis.net/docker.io
              pullPolicy: IfNotPresent
          volumePermissions:
            enabled: false
            image:
              registry: m.zjvis.net/docker.io
              pullPolicy: IfNotPresent
          sysctlImage:
            enabled: true
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
          kibana:
            elasticsearch:
              hosts:
                - '{{ include "elasticsearch.service.name" . }}'
              port: '{{ include "elasticsearch.service.ports.restAPI" . }}'
          esJavaOpts: "-Xmx2g -Xms2g"        
    destination:
      server: https://kubernetes.default.svc
      namespace: application

  diff from oirginal file :

  security:
    enabled: true
  extraConfig:
      path:
        repo: /tmp

Methods

Steps

curl --user elastic:L9shjg6csBmPZgCZ -k -X PUT "https://10.88.0.143:30294/_snapshot/slm_fs_repository?pretty" -H 'Content-Type: application/json' -d'
{
  "type": "fs",
  "settings": {
    "location": "/tmp"
  }
}
'

curl --user elastic:L9shjg6csBmPZgCZ  -k -X POST "https://10.88.0.143:30294/_snapshot/slm_fs_repository/_verify?pretty"

curl --user elastic:L9shjg6csBmPZgCZ  -k -X GET "https://10.88.0.143:30294/_snapshot/_all?pretty"

curl --user elastic:L9shjg6csBmPZgCZ  -k -X GET "https://10.88.0.143:30294/_snapshot/slm_fs_repository?pretty"

curl --user elastic:L9shjg6csBmPZgCZ  -k -X POST "https://10.88.0.143:30294/_snapshot/slm_fs_repository/_analyze?blob_count=10&max_blob_size=1mb&timeout=120s&pretty"

curl --user elastic:L9shjg6csBmPZgCZ  -k -X GET "https://10.88.0.143:30294/_snapshot/slm_fs_repository/*?verbose=false&pretty"

curl --user elastic:L9shjg6csBmPZgCZ -k -X POST "https://10.88.0.143:30294/_security/role/slm-admin?pretty" -H 'Content-Type: application/json' -d'
{
  "cluster": [ "manage_slm", "cluster:admin/snapshot/*" ],
  "indices": [
    {
      "names": [ ".slm-history-*" ],
      "privileges": [ "all" ]
    }
  ]
}
'

curl --user elastic:L9shjg6csBmPZgCZ -k -X PUT "https://10.88.0.143:30294/_slm/policy/nightly-snapshots?pretty" -H 'Content-Type: application/json' -d'
{
  "schedule": "0 30 1 * * ?",       
  "name": "<nightly-snap-{now/d}>", 
  "repository": "slm_fs_repository",    
  "config": {
    "indices": "*",                 
    "include_global_state": true    
  },
  "retention": {                    
    "expire_after": "30d",
    "min_count": 5,
    "max_count": 50
  }
}
'

curl --user elastic:L9shjg6csBmPZgCZ -k -X POST "https://10.88.0.143:30294/_slm/policy/nightly-snapshots/_execute?pretty"

curl --user elastic:L9shjg6csBmPZgCZ -k -X GET "https://10.88.0.143:30294/_slm/stats?pretty"

# Delete an index
curl --user elastic:L9shjg6csBmPZgCZ  -k -X DELETE "https://10.88.0.143:30294/books?pretty"

# restore that index
curl --user elastic:L9shjg6csBmPZgCZ  -k -X POST "https://10.88.0.143:30294/_snapshot/slm_fs_repository/my_snapshot_2099.05.06/_restore?pretty" -H 'Content-Type: application/json' -d'
{
  "indices": "books"
}
'

# query
curl --user elastic:L9shjg6csBmPZgCZ  -k -X GET "https://10.88.0.143:30294/books/_search?pretty" -H 'Content-Type: application/json' -d'
{
  "query": {
    "match_all": {}
  }
}
'

Aliyun Related

• OSSutil
• ECS DNS
• OS Mirrors

App Related

• Mirrors [Aliyun, Tsinghua]

Git Related

• Not Allow Push

Linux Related

• Disable Service
• Example Shell Script
• Example Shell Script
• Login Without Pwd
• Set Http Proxy

Storage Related

• User Based Policy

Git CMD

Init global config

git config --list
git config --global user.name "AaronYang"
git config --global user.email aaron19940628@gmail.com
git config --global pager.branch false
git config --global pull.ff only
git --no-pager diff

change user and email (locally)

git config user.name ""
git config user.email ""

list all remote repo

git remote -v

git remote set-url origin git@github.com:<$user>/<$repo>.git

Get specific file from remote

git archive --remote=git@github.com:<$user>/<$repo>.git <$branch>:<$source_file_path> -o <$target_source_path>

git archive --remote=git@github.com:AaronYang2333/LOL_Overlay_Assistant_Tool.git master:paper/2003.11755.pdf -o a.pdf

Clone specific branch

git clone -b slurm-23.02 --single-branch --depth=1 https://github.com/SchedMD/slurm.git

Update submodule

git submodule add –depth 1 https://github.com/xxx/xxxx a/b/c

git submodule update --init --recursive

Save credential

login first and then execute this

git config --global credential.helper store

Delete Branch

• Deleting a remote branch

  git push origin --delete <branch>  # Git version 1.7.0 or newer
  git push origin -d <branch>        # Shorter version (Git 1.7.0 or newer)
  git push origin :<branch>          # Git versions older than 1.7.0

• Deleting a local branch

  git branch --delete <branch>
  git branch -d <branch> # Shorter version
  git branch -D <branch> # Force-delete un-merged branches

Prune remote branches

git remote prune origin

Update remote repo

git remote set-url origin http://xxxxx.git

Linux

useradd

sudo useradd <$name> -m -r -s /bin/bash -p <$password>

echo '<$name> ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers

telnet

a command line interface for communication with a remote device or serve

telnet <$ip> <$port>

telnet 172.27.253.50 9000 #test application connectivity

lsof (list as open files)

everything is a file

lsof <$option:value>

lsof -i:30443 # find port 30443 
lsof -i -P -n # list all connections

awk (Aho, Weinberger, and Kernighan [Names])

awk is a scripting language used for manipulating data and generating reports.

# awk [params] 'script' 
awk <$params> <$string_content>

echo -e "1\n2\n3\n4\n5\n" | awk '$1>3'

ss (socket statistics)

view detailed information about your system’s network connections, including TCP/IP, UDP, and Unix domain sockets

ss [options]

#show all listening TCP connection
ss -tln

#show all established TCP connections
ss -tan

clean files 3 days ago

find /aaa/bbb/ccc/*.gz -mtime +3 -exec rm {} \;

ssh without affect $HOME/.ssh/known_hosts

ssh -o "UserKnownHostsFile /dev/null" root@aaa.domain.com
ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking=no" root@aaa.domain.com

sync clock

[yum|dnf] install -y chrony \
    && systemctl enable chronyd \
    && (systemctl is-active chronyd || systemctl start chronyd) \
    && chronyc sources \
    && chronyc tracking \
    && timedatectl set-timezone 'Asia/Shanghai'

set hostname

hostnamectl set-hostname develop

add remote key to other server

ssh -o "UserKnownHostsFile /dev/null" \
    root@aaa.bbb.ccc \
    "mkdir -p /root/.ssh && chmod 700 /root/.ssh && echo '$SOME_PUBLIC_KEY' \
    >> /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys"

ssh -o "UserKnownHostsFile /dev/null" \
    root@17.27.253.67 \
    "mkdir -p /root/.ssh && chmod 700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC00JLKF/Cd//rJcdIVGCX3ePo89KAgEccvJe4TEHs5pI5FSxs/7/JfQKZ+by2puC3IT88bo/d7nStw9PR3BXgqFXaBCknNBpSLWBIuvfBF+bcL+jGnQYo2kPjrO+2186C5zKGuPRi9sxLI5AkamGB39L5SGqwe5bbKq2x/8OjUP25AlTd99XsNjEY2uxNVClHysExVad/ZAcl0UVzG5xmllusXCsZVz9HlPExqB6K1sfMYWvLVgSCChx6nUfgg/NZrn/kQG26X0WdtXVM2aXpbAtBioML4rWidsByDb131NqYpJF7f+x3+I5pQ66Qpc72FW1G4mUiWWiGhF9tL8V9o1AY96Rqz0AVaxAQrBEuyCWKrXbA97HeC3Xp57Luvlv9TqUd8CIJYq+QTL0hlIDrzK9rJsg34FRAvf9sh8K2w/T/gC9UnRjRXgkPUgKldq35Y6Z9wP6KY45gCXka1PU4nVqb6wicO+RHcZ5E4sreUwqfTypt5nTOgW2/p8iFhdN8= Administrator@AARON-X1-8TH' \
    >> /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys"

set -x

This will print each command to the standard error before executing it, which is useful for debugging scripts.

set -x

set -e

Exit immediately if a command exits with a non-zero status.

set -x

sed (Stream Editor)

sed <$option> <$file_path>

echo "linux is great os. unix is opensource. unix is free os." | sed 's/unix/linux/'

fdisk

list all disk

fdisk -l

create CFS file system

Use mkfs.xfs command to create xfs file system and internal log on the same disk, example is shown below:

mkfs.xfs <$path>

modprobe

program to add and remove modules from the Linux Kernel

modprobe nfs && modprobe nfsd

disown

disown command in Linux is used to remove jobs from the job table.

disown [options] jobID1 jobID2 ... jobIDN

ping google.com > /dev/null &

jobs -l

disown -a

disown %2

generate SSH key

ssh-keygen -t rsa -b 4096 -C "aaron19940628@gmail.com"

create soft link

sudo ln -sf <$install_path>/bin/* /usr/local/bin

append dir into $PATH (temporary)

export PATH="/root/bin:$PATH"

copy public key to ECS

ssh-copy-id -i ~/.ssh/id_rsa.pub root@10.200.60.53

Maven

1. build from submodule

You dont need to build from the head of project.

./mvnw clean package -DskipTests  -rf :<$submodule-name>

you can find the <$submodule-name> from submodule ’s pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
		xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<modelVersion>4.0.0</modelVersion>

	<parent>
		<groupId>org.apache.flink</groupId>
		<artifactId>flink-formats</artifactId>
		<version>1.20-SNAPSHOT</version>
	</parent>

	<artifactId>flink-avro</artifactId>
	<name>Flink : Formats : Avro</name>

Then you can modify the command as

./mvnw clean package -DskipTests  -rf :flink-avro

[WARNING] For this reason, future Maven versions might no longer support building such malformed projects.
[WARNING] 
[INFO] ------------------------------------------------------------------------
[INFO] Detecting the operating system and CPU architecture
[INFO] ------------------------------------------------------------------------
[INFO] os.detected.name: linux
[INFO] os.detected.arch: x86_64
[INFO] os.detected.bitness: 64
[INFO] os.detected.version: 6.7
[INFO] os.detected.version.major: 6
[INFO] os.detected.version.minor: 7
[INFO] os.detected.release: fedora
[INFO] os.detected.release.version: 38
[INFO] os.detected.release.like.fedora: true
[INFO] os.detected.classifier: linux-x86_64
[INFO] ------------------------------------------------------------------------
[INFO] Reactor Build Order:
[INFO] 
[INFO] Flink : Formats : Avro                                             [jar]
[INFO] Flink : Formats : SQL Avro                                         [jar]
[INFO] Flink : Formats : Parquet                                          [jar]
[INFO] Flink : Formats : SQL Parquet                                      [jar]
[INFO] Flink : Formats : Orc                                              [jar]
[INFO] Flink : Formats : SQL Orc                                          [jar]
[INFO] Flink : Python                                                     [jar]
...

2. skip some other test

For example, you can skip RAT test by doing this:

./mvnw clean package -DskipTests '-Drat.skip=true'

Gradle

1. spotless

keep your code spotless, check more detail in https://github.com/diffplug/spotless

plugins {
    id("org.gradle.toolchains.foojay-resolver-convention") version "0.7.0"
}

plugins {
    id("com.diffplug.spotless") version "6.23.3"
}
configure<com.diffplug.gradle.spotless.SpotlessExtension> {
    kotlinGradle {
        target("**/*.kts")
        ktlint()
    }
    java {
        target("**/*.java")
        googleJavaFormat()
            .reflowLongStrings()
            .skipJavadocFormatting()
            .reorderImports(false)
    }
    yaml {
        target("**/*.yaml")
        jackson()
            .feature("ORDER_MAP_ENTRIES_BY_KEYS", true)
    }
    json {
        target("**/*.json")
        targetExclude(".vscode/settings.json")
        jackson()
            .feature("ORDER_MAP_ENTRIES_BY_KEYS", true)
    }
}

And the, you can execute follwoing command to format your code.

./gradlew spotlessApply

./mvnw spotless:apply

2. shadowJar

shadowjar could combine a project’s dependency classes and resources into a single jar. check https://imperceptiblethoughts.com/shadow/

import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar

plugins {
    java // Optional 
    id("com.github.johnrengelman.shadow") version "8.1.1"
}

tasks.named<ShadowJar>("shadowJar") {
    archiveBaseName.set("connector-shadow")
    archiveVersion.set("1.0")
    archiveClassifier.set("")
    manifest {
        attributes(mapOf("Main-Class" to "com.example.xxxxx.Main"))
    }
}

./gradlew shadowJar

3. check dependency

list your project’s dependencies in tree view

configurations {
    compileClasspath
}

./gradlew dependencies --configuration compileClasspath

./gradlew :<$module_name>:dependencies --configuration compileClasspath

compileClasspath - Compile classpath for source set 'main'.
+--- org.projectlombok:lombok:1.18.22
+--- org.apache.flink:flink-hadoop-fs:1.17.1
|    \--- org.apache.flink:flink-core:1.17.1
|         +--- org.apache.flink:flink-annotations:1.17.1
|         |    \--- com.google.code.findbugs:jsr305:1.3.9 -> 3.0.2
|         +--- org.apache.flink:flink-metrics-core:1.17.1
|         |    \--- org.apache.flink:flink-annotations:1.17.1 (*)
|         +--- org.apache.flink:flink-shaded-asm-9:9.3-16.1
|         +--- org.apache.flink:flink-shaded-jackson:2.13.4-16.1
|         +--- org.apache.commons:commons-lang3:3.12.0
|         +--- org.apache.commons:commons-text:1.10.0
|         |    \--- org.apache.commons:commons-lang3:3.12.0
|         +--- commons-collections:commons-collections:3.2.2
|         +--- org.apache.commons:commons-compress:1.21 -> 1.24.0
|         +--- org.apache.flink:flink-shaded-guava:30.1.1-jre-16.1
|         \--- com.google.code.findbugs:jsr305:1.3.9 -> 3.0.2
...

Elastic Search DSL

Basic Query

GET /_search
{
  "query": {
    "exists": {
      "field": "user"
    }
  }
}

GET /_search
{
  "query": {
    "bool": {
      "must_not": {
        "exists": {
          "field": "user.id"
        }
      }
    }
  }
}

GET /_search
{
  "query": {
    "fuzzy": {
      "filed_A": {
        "value": "ki"
      }
    }
  }
}

GET /_search
{
  "query": {
    "fuzzy": {
      "filed_A": {
        "value": "ki",
        "fuzziness": "AUTO",
        "max_expansions": 50,
        "prefix_length": 0,
        "transpositions": true,
        "rewrite": "constant_score_blended"
      }
    }
  }
}

GET /_search
{
  "query": {
    "ids" : {
      "values" : ["2NTC5ZIBNLuBWC6V5_0Y"]
    }
  }
}

GET /_search
{
  "query": {
    "prefix": {
      "filed_A": {
        "value": "ki",
         "rewrite": "constant_score_blended",
         "case_insensitive": true
      }
    }
  }
}

GET /_search
{
  "query": {
    "prefix" : { "filed_A" : "ki" }
  }
}

GET /_search
{
  "query": {
    "range": {
      "filed_number": {
        "gte": 10,
        "lte": 20,
        "boost": 2.0
      }
    }
  }
}

GET /_search
{
  "query": {
    "range": {
      "filed_timestamp": {
        "time_zone": "+01:00",        
        "gte": "2020-01-01T00:00:00", 
        "lte": "now"                  
      }
    }
  }
}

GET /_search
{
  "query": {
    "regexp": {
      "filed_A": {
        "value": "k.*y",
        "flags": "ALL",
        "case_insensitive": true,
        "max_determinized_states": 10000,
        "rewrite": "constant_score_blended"
      }
    }
  }
}

GET /_search
{
  "query": {
    "term": {
      "filed_A": {
        "value": "kimchy",
        "boost": 1.0
      }
    }
  }
}

GET /_search
{
  "query": {
    "wildcard": {
      "filed_A": {
        "value": "ki*y",
        "boost": 1.0,
        "rewrite": "constant_score_blended"
      }
    }
  }
}