Template

• DAG Template

ES [Local Disk]

Preliminary

• ElasticSearch has installed, if not check link

• The elasticsearch.yml has configed path.repo, which should be set the same value of settings.location (this will be handled by helm chart, dont worry)

  ES argocd-app yaml

  apiVersion: argoproj.io/v1alpha1
  kind: Application
  metadata:
    name: elastic-search
  spec:
    syncPolicy:
      syncOptions:
      - CreateNamespace=true
    project: default
    source:
      repoURL: https://charts.bitnami.com/bitnami
      chart: elasticsearch
      targetRevision: 19.11.3
      helm:
        releaseName: elastic-search
        values: |
          global:
            kibanaEnabled: true
          clusterName: elastic
          image:
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
          security:
            enabled: false
          service:
            type: ClusterIP
          extraConfig:
            path:
              repo: /tmp
          ingress:
            enabled: true
            annotations:
              cert-manager.io/cluster-issuer: self-signed-ca-issuer
              nginx.ingress.kubernetes.io/rewrite-target: /$1
            hostname: elastic-search.dev.tech
            ingressClassName: nginx
            path: /?(.*)
            tls: true
          master:
            masterOnly: false
            replicaCount: 1
            persistence:
              enabled: false
            resources:
              requests:
                cpu: 2
                memory: 1024Mi
              limits:
                cpu: 4
                memory: 4096Mi
            heapSize: 2g
          data:
            replicaCount: 0
            persistence:
              enabled: false
          coordinating:
            replicaCount: 0
          ingest:
            enabled: true
            replicaCount: 0
            service:
              enabled: false
              type: ClusterIP
            ingress:
              enabled: false
          metrics:
            enabled: false
            image:
              registry: m.zjvis.net/docker.io
              pullPolicy: IfNotPresent
          volumePermissions:
            enabled: false
            image:
              registry: m.zjvis.net/docker.io
              pullPolicy: IfNotPresent
          sysctlImage:
            enabled: true
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
          kibana:
            elasticsearch:
              hosts:
                - '{{ include "elasticsearch.service.name" . }}'
              port: '{{ include "elasticsearch.service.ports.restAPI" . }}'
          esJavaOpts: "-Xmx2g -Xms2g"        
    destination:
      server: https://kubernetes.default.svc
      namespace: application

  diff from oirginal file :

  extraConfig:
      path:
        repo: /tmp

Methods

Elasticsearch 做备份有两种方式，

1. 是将数据导出成文本文件，比如通过elasticdump、esm等工具将存储在 Elasticsearch 中的数据导出到文件中。
2. 是使用snapshot接口实现快照功能，增量备份文件

第一种方式相对简单，在数据量小的时候比较实用，但当应对大数据量场景时，更推荐使用snapshot api 的方式。

Steps

curl -k -X PUT "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository?pretty" -H 'Content-Type: application/json' -d'
{
  "type": "fs",
  "settings": {
    "location": "/tmp"
  }
}
'

curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository/_verify?pretty"

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/_all?pretty"

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository?pretty"

curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository/_analyze?blob_count=10&max_blob_size=1mb&timeout=120s&pretty"

curl -k -X PUT "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository/ay_snap_02?pretty"

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository/*?verbose=false&pretty"

# Delete an index
curl -k -X DELETE "https://elastic-search.dev.tech:32443/books?pretty"

# restore that index
curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_fs_repository/ay_snap_02/_restore?pretty" -H 'Content-Type: application/json' -d'
{
  "indices": "books"
}
'

# query
curl -k -X GET "https://elastic-search.dev.tech:32443/books/_search?pretty" -H 'Content-Type: application/json' -d'
{
  "query": {
    "match_all": {}
  }
}
'

ES [S3 Compatible]

Preliminary

• ElasticSearch has installed, if not check link

  ES argocd-app yaml

  apiVersion: argoproj.io/v1alpha1
  kind: Application
  metadata:
    name: elastic-search
  spec:
    syncPolicy:
      syncOptions:
      - CreateNamespace=true
    project: default
    source:
      repoURL: https://charts.bitnami.com/bitnami
      chart: elasticsearch
      targetRevision: 19.11.3
      helm:
        releaseName: elastic-search
        values: |
          global:
            kibanaEnabled: true
          clusterName: elastic
          image:
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
          security:
            enabled: true
          service:
            type: ClusterIP
          extraEnvVars:
          - name: S3_ACCESSKEY
            value: admin
          - name: S3_SECRETKEY
            value: ZrwpsezF1Lt85dxl
          extraConfig:
            s3:
              client:
                default:
                  protocol: http
                  endpoint: "http://192.168.31.111:9090"
                  path_style_access: true
          initScripts:
            configure-s3-client.sh: |
              elasticsearch_set_key_value "s3.client.default.access_key" "${S3_ACCESSKEY}"
              elasticsearch_set_key_value "s3.client.default.secret_key" "${S3_SECRETKEY}"
          hostAliases:
          - ip: 192.168.31.111
            hostnames:
            - minio-api.dev.tech
          ingress:
            enabled: true
            annotations:
              cert-manager.io/cluster-issuer: self-signed-ca-issuer
              nginx.ingress.kubernetes.io/rewrite-target: /$1
            hostname: elastic-search.dev.tech
            ingressClassName: nginx
            path: /?(.*)
            tls: true
          master:
            masterOnly: false
            replicaCount: 1
            persistence:
              enabled: false
            resources:
              requests:
                cpu: 2
                memory: 1024Mi
              limits:
                cpu: 4
                memory: 4096Mi
            heapSize: 2g
          data:
            replicaCount: 0
            persistence:
              enabled: false
          coordinating:
            replicaCount: 0
          ingest:
            enabled: true
            replicaCount: 0
            service:
              enabled: false
              type: ClusterIP
            ingress:
              enabled: false
          metrics:
            enabled: false
            image:
              registry: m.zjvis.net/docker.io
              pullPolicy: IfNotPresent
          volumePermissions:
            enabled: false
            image:
              registry: m.zjvis.net/docker.io
              pullPolicy: IfNotPresent
          sysctlImage:
            enabled: true
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
          kibana:
            elasticsearch:
              hosts:
                - '{{ include "elasticsearch.service.name" . }}'
              port: '{{ include "elasticsearch.service.ports.restAPI" . }}'
          esJavaOpts: "-Xmx2g -Xms2g"        
    destination:
      server: https://kubernetes.default.svc
      namespace: application

  diff from oirginal file :

  extraEnvVars:
  - name: S3_ACCESSKEY
    value: admin
  - name: S3_SECRETKEY
    value: ZrwpsezF1Lt85dxl
  extraConfig:
    s3:
      client:
        default:
          protocol: http
          endpoint: "http://192.168.31.111:9090"
          path_style_access: true
  initScripts:
    configure-s3-client.sh: |
      elasticsearch_set_key_value "s3.client.default.access_key" "${S3_ACCESSKEY}"
      elasticsearch_set_key_value "s3.client.default.secret_key" "${S3_SECRETKEY}"
  hostAliases:
  - ip: 192.168.31.111
    hostnames:
    - minio-api.dev.tech

Methods

Elasticsearch 做备份有两种方式，

1. 是将数据导出成文本文件，比如通过elasticdump、esm等工具将存储在 Elasticsearch 中的数据导出到文件中。
2. 是使用snapshot接口实现快照功能，增量备份文件

第一种方式相对简单，在数据量小的时候比较实用，但当应对大数据量场景时，更推荐使用snapshot api 的方式。

Steps

curl -k -X PUT "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository?pretty" -H 'Content-Type: application/json' -d'
{
  "type": "s3",
  "settings": {
    "bucket": "local-test",
    "client": "default",
    "endpoint": "http://192.168.31.111:9000"
  }
}
'

curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository/_verify?pretty"

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/_all?pretty"

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository?pretty"

curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository/_analyze?blob_count=10&max_blob_size=1mb&timeout=120s&pretty"

curl -k -X PUT "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository/ay_s3_snap_02?pretty"

curl -k -X GET "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository/*?verbose=false&pretty"

# Delete an index
curl -k -X DELETE "https://elastic-search.dev.tech:32443/books?pretty"

# restore that index
curl -k -X POST "https://elastic-search.dev.tech:32443/_snapshot/my_s3_repository/ay_s3_snap_02/_restore?pretty" -H 'Content-Type: application/json' -d'
{
  "indices": "books"
}
'

# query
curl -k -X GET "https://elastic-search.dev.tech:32443/books/_search?pretty" -H 'Content-Type: application/json' -d'
{
  "query": {
    "match_all": {}
  }
}
'

ES Auto BackUp

Preliminary

• ElasticSearch has installed, if not check link

• We use local disk to save the snapshots, more deatils check link

• And the security is enabled.

  ES argocd-app yaml

  apiVersion: argoproj.io/v1alpha1
  kind: Application
  metadata:
    name: elastic-search
  spec:
    syncPolicy:
      syncOptions:
      - CreateNamespace=true
    project: default
    source:
      repoURL: https://charts.bitnami.com/bitnami
      chart: elasticsearch
      targetRevision: 19.11.3
      helm:
        releaseName: elastic-search
        values: |
          global:
            kibanaEnabled: true
          clusterName: elastic
          image:
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
          security:
            enabled: true
            tls:
              autoGenerated: true
          service:
            type: ClusterIP
          extraConfig:
            path:
              repo: /tmp
          ingress:
            enabled: true
            annotations:
              cert-manager.io/cluster-issuer: self-signed-ca-issuer
              nginx.ingress.kubernetes.io/rewrite-target: /$1
            hostname: elastic-search.dev.tech
            ingressClassName: nginx
            path: /?(.*)
            tls: true
          master:
            masterOnly: false
            replicaCount: 1
            persistence:
              enabled: false
            resources:
              requests:
                cpu: 2
                memory: 1024Mi
              limits:
                cpu: 4
                memory: 4096Mi
            heapSize: 2g
          data:
            replicaCount: 0
            persistence:
              enabled: false
          coordinating:
            replicaCount: 0
          ingest:
            enabled: true
            replicaCount: 0
            service:
              enabled: false
              type: ClusterIP
            ingress:
              enabled: false
          metrics:
            enabled: false
            image:
              registry: m.zjvis.net/docker.io
              pullPolicy: IfNotPresent
          volumePermissions:
            enabled: false
            image:
              registry: m.zjvis.net/docker.io
              pullPolicy: IfNotPresent
          sysctlImage:
            enabled: true
            registry: m.zjvis.net/docker.io
            pullPolicy: IfNotPresent
          kibana:
            elasticsearch:
              hosts:
                - '{{ include "elasticsearch.service.name" . }}'
              port: '{{ include "elasticsearch.service.ports.restAPI" . }}'
          esJavaOpts: "-Xmx2g -Xms2g"        
    destination:
      server: https://kubernetes.default.svc
      namespace: application

  diff from oirginal file :

  security:
    enabled: true
  extraConfig:
      path:
        repo: /tmp

Methods

Steps

curl --user elastic:L9shjg6csBmPZgCZ -k -X PUT "https://10.88.0.143:30294/_snapshot/slm_fs_repository?pretty" -H 'Content-Type: application/json' -d'
{
  "type": "fs",
  "settings": {
    "location": "/tmp"
  }
}
'

curl --user elastic:L9shjg6csBmPZgCZ  -k -X POST "https://10.88.0.143:30294/_snapshot/slm_fs_repository/_verify?pretty"

curl --user elastic:L9shjg6csBmPZgCZ  -k -X GET "https://10.88.0.143:30294/_snapshot/_all?pretty"

curl --user elastic:L9shjg6csBmPZgCZ  -k -X GET "https://10.88.0.143:30294/_snapshot/slm_fs_repository?pretty"

curl --user elastic:L9shjg6csBmPZgCZ  -k -X POST "https://10.88.0.143:30294/_snapshot/slm_fs_repository/_analyze?blob_count=10&max_blob_size=1mb&timeout=120s&pretty"

curl --user elastic:L9shjg6csBmPZgCZ  -k -X GET "https://10.88.0.143:30294/_snapshot/slm_fs_repository/*?verbose=false&pretty"

curl --user elastic:L9shjg6csBmPZgCZ -k -X POST "https://10.88.0.143:30294/_security/role/slm-admin?pretty" -H 'Content-Type: application/json' -d'
{
  "cluster": [ "manage_slm", "cluster:admin/snapshot/*" ],
  "indices": [
    {
      "names": [ ".slm-history-*" ],
      "privileges": [ "all" ]
    }
  ]
}
'

curl --user elastic:L9shjg6csBmPZgCZ -k -X PUT "https://10.88.0.143:30294/_slm/policy/nightly-snapshots?pretty" -H 'Content-Type: application/json' -d'
{
  "schedule": "0 30 1 * * ?",       
  "name": "<nightly-snap-{now/d}>", 
  "repository": "slm_fs_repository",    
  "config": {
    "indices": "*",                 
    "include_global_state": true    
  },
  "retention": {                    
    "expire_after": "30d",
    "min_count": 5,
    "max_count": 50
  }
}
'

curl --user elastic:L9shjg6csBmPZgCZ -k -X POST "https://10.88.0.143:30294/_slm/policy/nightly-snapshots/_execute?pretty"

curl --user elastic:L9shjg6csBmPZgCZ -k -X GET "https://10.88.0.143:30294/_slm/stats?pretty"

# Delete an index
curl --user elastic:L9shjg6csBmPZgCZ  -k -X DELETE "https://10.88.0.143:30294/books?pretty"

# restore that index
curl --user elastic:L9shjg6csBmPZgCZ  -k -X POST "https://10.88.0.143:30294/_snapshot/slm_fs_repository/my_snapshot_2099.05.06/_restore?pretty" -H 'Content-Type: application/json' -d'
{
  "indices": "books"
}
'

# query
curl --user elastic:L9shjg6csBmPZgCZ  -k -X GET "https://10.88.0.143:30294/books/_search?pretty" -H 'Content-Type: application/json' -d'
{
  "query": {
    "match_all": {}
  }
}
'

Aliyun Related

• OSSutil
• ECS DNS
• OS Mirrors

App Related

• Mirrors [Aliyun, Tsinghua]

Git Related

• Not Allow Push

Linux Related

• Disable Service
• Example Shell Script
• Login Without Pwd
• Set Http Proxy

Storage Related

• User Based Policy

Git CMD

Init global config

git config --list
git config --global user.name "AaronYang"
git config --global user.email aaron19940628@gmail.com
git config --global pager.branch false
git config --global pull.ff only
git --no-pager diff

change user and email (locally)

git config user.name ""
git config user.email ""

list all remote repo

git remote -v

git remote set-url origin git@github.com:<$user>/<$repo>.git

Get specific file from remote

git archive --remote=git@github.com:<$user>/<$repo>.git <$branch>:<$source_file_path> -o <$target_source_path>

git archive --remote=git@github.com:AaronYang2333/LOL_Overlay_Assistant_Tool.git master:paper/2003.11755.pdf -o a.pdf

Clone specific branch

git clone -b slurm-23.02 --single-branch --depth=1 https://github.com/SchedMD/slurm.git

Update submodule

git submodule add –depth 1 https://github.com/xxx/xxxx a/b/c

git submodule update --init --recursive

Save credential

login first and then execute this

git config --global credential.helper store

Delete Branch

• Deleting a remote branch

  git push origin --delete <branch>  # Git version 1.7.0 or newer
  git push origin -d <branch>        # Shorter version (Git 1.7.0 or newer)
  git push origin :<branch>          # Git versions older than 1.7.0

• Deleting a local branch

  git branch --delete <branch>
  git branch -d <branch> # Shorter version
  git branch -D <branch> # Force-delete un-merged branches

Prune remote branches

git remote prune origin

Update remote repo

git remote set-url origin http://xxxxx.git

Linux

useradd

sudo useradd <$name> -m -r -s /bin/bash -p <$password>

echo '<$name> ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers

telnet

a command line interface for communication with a remote device or serve

telnet <$ip> <$port>

telnet 172.27.253.50 9000 #test application connectivity

lsof (list as open files)

everything is a file

lsof <$option:value>

lsof -i:30443 # find port 30443 
lsof -i -P -n # list all connections

awk (Aho, Weinberger, and Kernighan [Names])

awk is a scripting language used for manipulating data and generating reports.

# awk [params] 'script' 
awk <$params> <$string_content>

echo -e "1\n2\n3\n4\n5\n" | awk '$1>3'

ss (socket statistics)

view detailed information about your system’s network connections, including TCP/IP, UDP, and Unix domain sockets

ss [options]

#show all listening TCP connection
ss -tln

#show all established TCP connections
ss -tan

clean files 3 days ago

find /aaa/bbb/ccc/*.gz -mtime +3 -exec rm {} \;

ssh without affect $HOME/.ssh/known_hosts

ssh -o "UserKnownHostsFile /dev/null" root@aaa.domain.com
ssh -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking=no" root@aaa.domain.com

sync clock

[yum|dnf] install -y chrony \
    && systemctl enable chronyd \
    && (systemctl is-active chronyd || systemctl start chronyd) \
    && chronyc sources \
    && chronyc tracking \
    && timedatectl set-timezone 'Asia/Shanghai'

set hostname

hostnamectl set-hostname develop

add remote key to other server

ssh -o "UserKnownHostsFile /dev/null" \
    root@aaa.bbb.ccc \
    "mkdir -p /root/.ssh && chmod 700 /root/.ssh && echo '$SOME_PUBLIC_KEY' \
    >> /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys"

ssh -o "UserKnownHostsFile /dev/null" \
    root@17.27.253.67 \
    "mkdir -p /root/.ssh && chmod 700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC00JLKF/Cd//rJcdIVGCX3ePo89KAgEccvJe4TEHs5pI5FSxs/7/JfQKZ+by2puC3IT88bo/d7nStw9PR3BXgqFXaBCknNBpSLWBIuvfBF+bcL+jGnQYo2kPjrO+2186C5zKGuPRi9sxLI5AkamGB39L5SGqwe5bbKq2x/8OjUP25AlTd99XsNjEY2uxNVClHysExVad/ZAcl0UVzG5xmllusXCsZVz9HlPExqB6K1sfMYWvLVgSCChx6nUfgg/NZrn/kQG26X0WdtXVM2aXpbAtBioML4rWidsByDb131NqYpJF7f+x3+I5pQ66Qpc72FW1G4mUiWWiGhF9tL8V9o1AY96Rqz0AVaxAQrBEuyCWKrXbA97HeC3Xp57Luvlv9TqUd8CIJYq+QTL0hlIDrzK9rJsg34FRAvf9sh8K2w/T/gC9UnRjRXgkPUgKldq35Y6Z9wP6KY45gCXka1PU4nVqb6wicO+RHcZ5E4sreUwqfTypt5nTOgW2/p8iFhdN8= Administrator@AARON-X1-8TH' \
    >> /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys"

set -x

This will print each command to the standard error before executing it, which is useful for debugging scripts.

set -x

set -e

Exit immediately if a command exits with a non-zero status.

set -x

sed (Stream Editor)

sed <$option> <$file_path>

echo "linux is great os. unix is opensource. unix is free os." | sed 's/unix/linux/'

fdisk

list all disk

fdisk -l

create CFS file system

Use mkfs.xfs command to create xfs file system and internal log on the same disk, example is shown below:

mkfs.xfs <$path>

modprobe

program to add and remove modules from the Linux Kernel

modprobe nfs && modprobe nfsd

disown

disown command in Linux is used to remove jobs from the job table.

disown [options] jobID1 jobID2 ... jobIDN

ping google.com > /dev/null &

jobs -l

disown -a

disown %2

generate SSH key

ssh-keygen -t rsa -b 4096 -C "aaron19940628@gmail.com"

create soft link

sudo ln -sf <$install_path>/bin/* /usr/local/bin

append dir into $PATH (temporary)

export PATH="/root/bin:$PATH"

copy public key to ECS

ssh-copy-id -i ~/.ssh/id_rsa.pub root@10.200.60.53

Maven

1. build from submodule

You dont need to build from the head of project.

./mvnw clean package -DskipTests  -rf :<$submodule-name>

you can find the <$submodule-name> from submodule ’s pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
		xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<modelVersion>4.0.0</modelVersion>

	<parent>
		<groupId>org.apache.flink</groupId>
		<artifactId>flink-formats</artifactId>
		<version>1.20-SNAPSHOT</version>
	</parent>

	<artifactId>flink-avro</artifactId>
	<name>Flink : Formats : Avro</name>

Then you can modify the command as

./mvnw clean package -DskipTests  -rf :flink-avro

[WARNING] For this reason, future Maven versions might no longer support building such malformed projects.
[WARNING] 
[INFO] ------------------------------------------------------------------------
[INFO] Detecting the operating system and CPU architecture
[INFO] ------------------------------------------------------------------------
[INFO] os.detected.name: linux
[INFO] os.detected.arch: x86_64
[INFO] os.detected.bitness: 64
[INFO] os.detected.version: 6.7
[INFO] os.detected.version.major: 6
[INFO] os.detected.version.minor: 7
[INFO] os.detected.release: fedora
[INFO] os.detected.release.version: 38
[INFO] os.detected.release.like.fedora: true
[INFO] os.detected.classifier: linux-x86_64
[INFO] ------------------------------------------------------------------------
[INFO] Reactor Build Order:
[INFO] 
[INFO] Flink : Formats : Avro                                             [jar]
[INFO] Flink : Formats : SQL Avro                                         [jar]
[INFO] Flink : Formats : Parquet                                          [jar]
[INFO] Flink : Formats : SQL Parquet                                      [jar]
[INFO] Flink : Formats : Orc                                              [jar]
[INFO] Flink : Formats : SQL Orc                                          [jar]
[INFO] Flink : Python                                                     [jar]
...

2. skip some other test

For example, you can skip RAT test by doing this:

./mvnw clean package -DskipTests '-Drat.skip=true'

Gradle

1. spotless

keep your code spotless, check more detail in https://github.com/diffplug/spotless

plugins {
    id("org.gradle.toolchains.foojay-resolver-convention") version "0.7.0"
}

plugins {
    id("com.diffplug.spotless") version "6.23.3"
}
configure<com.diffplug.gradle.spotless.SpotlessExtension> {
    kotlinGradle {
        target("**/*.kts")
        ktlint()
    }
    java {
        target("**/*.java")
        googleJavaFormat()
            .reflowLongStrings()
            .skipJavadocFormatting()
            .reorderImports(false)
    }
    yaml {
        target("**/*.yaml")
        jackson()
            .feature("ORDER_MAP_ENTRIES_BY_KEYS", true)
    }
    json {
        target("**/*.json")
        targetExclude(".vscode/settings.json")
        jackson()
            .feature("ORDER_MAP_ENTRIES_BY_KEYS", true)
    }
}

And the, you can execute follwoing command to format your code.

./gradlew spotlessApply

./mvnw spotless:apply

2. shadowJar

shadowjar could combine a project’s dependency classes and resources into a single jar. check https://imperceptiblethoughts.com/shadow/

import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar

plugins {
    java // Optional 
    id("com.github.johnrengelman.shadow") version "8.1.1"
}

tasks.named<ShadowJar>("shadowJar") {
    archiveBaseName.set("connector-shadow")
    archiveVersion.set("1.0")
    archiveClassifier.set("")
    manifest {
        attributes(mapOf("Main-Class" to "com.example.xxxxx.Main"))
    }
}

./gradlew shadowJar

3. check dependency

list your project’s dependencies in tree view

configurations {
    compileClasspath
}

./gradlew dependencies --configuration compileClasspath

./gradlew :<$module_name>:dependencies --configuration compileClasspath

compileClasspath - Compile classpath for source set 'main'.
+--- org.projectlombok:lombok:1.18.22
+--- org.apache.flink:flink-hadoop-fs:1.17.1
|    \--- org.apache.flink:flink-core:1.17.1
|         +--- org.apache.flink:flink-annotations:1.17.1
|         |    \--- com.google.code.findbugs:jsr305:1.3.9 -> 3.0.2
|         +--- org.apache.flink:flink-metrics-core:1.17.1
|         |    \--- org.apache.flink:flink-annotations:1.17.1 (*)
|         +--- org.apache.flink:flink-shaded-asm-9:9.3-16.1
|         +--- org.apache.flink:flink-shaded-jackson:2.13.4-16.1
|         +--- org.apache.commons:commons-lang3:3.12.0
|         +--- org.apache.commons:commons-text:1.10.0
|         |    \--- org.apache.commons:commons-lang3:3.12.0
|         +--- commons-collections:commons-collections:3.2.2
|         +--- org.apache.commons:commons-compress:1.21 -> 1.24.0
|         +--- org.apache.flink:flink-shaded-guava:30.1.1-jre-16.1
|         \--- com.google.code.findbugs:jsr305:1.3.9 -> 3.0.2
...

Elastic Search DSL

Basic Query

GET /_search
{
  "query": {
    "exists": {
      "field": "user"
    }
  }
}

GET /_search
{
  "query": {
    "bool": {
      "must_not": {
        "exists": {
          "field": "user.id"
        }
      }
    }
  }
}

GET /_search
{
  "query": {
    "fuzzy": {
      "filed_A": {
        "value": "ki"
      }
    }
  }
}

GET /_search
{
  "query": {
    "fuzzy": {
      "filed_A": {
        "value": "ki",
        "fuzziness": "AUTO",
        "max_expansions": 50,
        "prefix_length": 0,
        "transpositions": true,
        "rewrite": "constant_score_blended"
      }
    }
  }
}

GET /_search
{
  "query": {
    "ids" : {
      "values" : ["2NTC5ZIBNLuBWC6V5_0Y"]
    }
  }
}

GET /_search
{
  "query": {
    "prefix": {
      "filed_A": {
        "value": "ki",
         "rewrite": "constant_score_blended",
         "case_insensitive": true
      }
    }
  }
}

GET /_search
{
  "query": {
    "prefix" : { "filed_A" : "ki" }
  }
}

GET /_search
{
  "query": {
    "range": {
      "filed_number": {
        "gte": 10,
        "lte": 20,
        "boost": 2.0
      }
    }
  }
}

GET /_search
{
  "query": {
    "range": {
      "filed_timestamp": {
        "time_zone": "+01:00",        
        "gte": "2020-01-01T00:00:00", 
        "lte": "now"                  
      }
    }
  }
}

GET /_search
{
  "query": {
    "regexp": {
      "filed_A": {
        "value": "k.*y",
        "flags": "ALL",
        "case_insensitive": true,
        "max_determinized_states": 10000,
        "rewrite": "constant_score_blended"
      }
    }
  }
}

GET /_search
{
  "query": {
    "term": {
      "filed_A": {
        "value": "kimchy",
        "boost": 1.0
      }
    }
  }
}

GET /_search
{
  "query": {
    "wildcard": {
      "filed_A": {
        "value": "ki*y",
        "boost": 1.0,
        "rewrite": "constant_score_blended"
      }
    }
  }
}

LOL Overlay Assistant

Using deep learning techniques to help you to win the game.

State Machine Event Bus Python 3.6 TensorFlow2 Captain InfoNew Awesome

ScreenShots

There are four main funcs in this tool.

1. The first one is to detect your game client thread and recognize which status you are in.

2. The second one is to recommend some champions to play. Based on your enemy’s team banned champion, this tool will provide you three more choices to counter your enemies.

3. The third func will scans the mini-map, and when someone is heading to you, a notification window will pop up.

4. The last func will provides you some gear recommendation based on your enemy’s item list.

Framework

Video Link

Checkout in Bilibili

Checkout in Youtube

Repo

you can get code from github, gitee

Roller Coin Assistant

Using deep learning techniques to help you to mining the cryptos, such as BTC, ETH and DOGE.

ScreenShots

There are two main funcs in this tool.

1. Help you to crack the game

• only support ‘Coin-Flip’ Game for now.

  right, rollercoin.com had decrease the benefit from this game, thats why I make the repo public. update

1. Help you to pass the geetest.

• only support level 1 captcha test for now. there are three levels of geetest captcha test

How to use

0. open a web browser.
1. go to this link https://rollercoin.com, and create an account.(https://rollercoin.com)
2. keep the lang equals to ‘English’ (you can click the bottom button to change it).
3. click the ‘Game’ button.
4. start the application, and enjoy it.

Tips

1. only supprot 1920*1080, 2560*1440 and higher resolution screen.
2. and if you use 1920*1080 screen, strongly recommend you to fullscreen you web browser.

Repo

you can get code from gitee

Slurm On K8S

Trying to run slurm cluster on kubernets

Install

You can directly use helm to manage this slurm chart

1. helm repo add ay-helm-mirror https://aaronyang0628.github.io/helm-chart-mirror/charts
2. helm install slurm ay-helm-mirror/slurm --version 1.0.4

And then, you should see something like this

Also, you can modify the values.yaml by yourself, and reinstall the slurm cluster

helm upgrade --create-namespace -n slurm --install -f ./values.yaml slurm ay-helm-mirror/slurm --version=1.0.4

Slurm Operator

if you wanna change slurm configuration ,please check slurm configuration generator click

• for helm user

  just run for fun!

  1. helm repo add ay-helm-repo https://aaronyang0628.github.io/helm-chart-mirror/charts
  2. helm install slurm ay-helm-repo/slurm --version 1.0.4
• for opertaor user

  pull an image and apply

  1. docker pull aaron666/slurm-operator:latest
  2. kubectl apply -f https://raw.githubusercontent.com/AaronYang0628/helm-chart-mirror/refs/heads/main/templates/slurm/install.yaml
  3. kubectl apply -f https://raw.githubusercontent.com/AaronYang0628/helm-chart-mirror/refs/heads/main/templates/slurm/slurmdeployment.values.yaml

Flink S3 F3 Multiple

Normally, Flink only can access only one S3 endpoint during the runtime. But we need to process some files from multiple minio simultaneously.

So I modified the original flink-s3-fs-hadoop and enable flink to do so.

StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.enableCheckpointing(5000L, CheckpointingMode.EXACTLY_ONCE);
env.setParallelism(1);
env.setStateBackend(new HashMapStateBackend());
env.getCheckpointConfig().setCheckpointStorage("file:///./checkpoints");

final FileSource<String> source =
    FileSource.forRecordStreamFormat(
            new TextLineInputFormat(),
            new Path(
                "s3u://admin:ZrwpsezF1Lt85dxl@10.11.33.132:9000/user-data/home/conti/2024-02-08--10"))
        .build();

final FileSource<String> source2 =
    FileSource.forRecordStreamFormat(
            new TextLineInputFormat(),
            new Path(
                "s3u://minioadmin:minioadmin@10.101.16.72:9000/user-data/home/conti"))
        .build();

env.fromSource(source, WatermarkStrategy.noWatermarks(), "file-source")
    .union(env.fromSource(source2, WatermarkStrategy.noWatermarks(), "file-source2"))
    .print("union-result");
    
env.execute();

Configuration pluginConfiguration = new Configuration();
pluginConfiguration.setString("s3a.access-key", "admin");
pluginConfiguration.setString("s3a.secret-key", "ZrwpsezF1Lt85dxl");
pluginConfiguration.setString("s3a.connection.maximum", "1000");
pluginConfiguration.setString("s3a.endpoint", "http://10.11.33.132:9000");
pluginConfiguration.setBoolean("s3a.path.style.access", Boolean.TRUE);
FileSystem.initialize(
    pluginConfiguration, PluginUtils.createPluginManagerFromRootFolder(pluginConfiguration));

StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.enableCheckpointing(5000L, CheckpointingMode.EXACTLY_ONCE);
env.setParallelism(1);
env.setStateBackend(new HashMapStateBackend());
env.getCheckpointConfig().setCheckpointStorage("file:///./checkpoints");

final FileSource<String> source =
    FileSource.forRecordStreamFormat(
            new TextLineInputFormat(), new Path("s3a://user-data/home/conti/2024-02-08--10"))
        .build();
env.fromSource(source, WatermarkStrategy.noWatermarks(), "file-source").print();

env.execute();

Usage

There

  implementation(files("flink-s3-fs-hadoop-$flinkVersion.jar"))

  <dependency>
      <groupId>org.apache</groupId>
      <artifactId>flink</artifactId>
      <version>$flinkVersion</version>
      <systemPath>${project.basedir}flink-s3-fs-hadoop-$flinkVersion.jar</systemPath>
  </dependency>

Repo

you can get code from github, gitlab

Cosmic Antenna

Design Architecture

• objects

continuously processing antenna signal records and convert them into 3 dimension data matrixes, sending them to different astronomical algorithm endpoints.

• how data flows

Building From Zero

Following these steps, you may build comic-antenna from nothing.

1. install podman

you can check article Install Podman

2. install kind and kubectl

you can check article install kubectl

# create a cluster using podman
curl -o kind.cluster.yaml -L https://gitlab.com/-/snippets/3686427/raw/main/kind-cluster.yaml \
&& export KIND_EXPERIMENTAL_PROVIDER=podman \
&& kind create cluster --name cs-cluster --image m.daocloud.io/docker.io/kindest/node:v1.27.3 --config=./kind.cluster.yaml

3. [Optional] pre-downloaded slow images

DOCKER_IMAGE_PATH=/root/docker-images && mkdir -p $DOCKER_IMAGE_PATH
BASE_URL="https://resource-ops-dev.lab.zjvis.net:32443/docker-images"
for IMAGE in "quay.io_argoproj_argocd_v2.9.3.dim" \
    "ghcr.io_dexidp_dex_v2.37.0.dim" \
    "docker.io_library_redis_7.0.11-alpine.dim" \
    "docker.io_library_flink_1.17.dim"
do
    IMAGE_FILE=$DOCKER_IMAGE_PATH/$IMAGE
    if [ ! -f $IMAGE_FILE ]; then
        TMP_FILE=$IMAGE_FILE.tmp \
        && curl -o "$TMP_FILE" -L "$BASE_URL/$IMAGE" \
        && mv $TMP_FILE $IMAGE_FILE
    fi
    kind -n cs-cluster load image-archive $IMAGE_FILE
done

4. install argocd

you can check article Install ArgoCD

5. install essential app on argocd

# install cert manger    
curl -LO https://gitlab.com/-/snippets/3686424/raw/main/cert-manager.yaml \
&& kubectl -n argocd apply -f cert-manager.yaml \
&& argocd app sync argocd/cert-manager

# install ingress
curl -LO https://gitlab.com/-/snippets/3686426/raw/main/ingress-nginx.yaml \
&& kubectl -n argocd apply -f ingress-nginx.yaml \
&& argocd app sync argocd/ingress-nginx

# install flink-kubernetes-operator
curl -LO https://gitlab.com/-/snippets/3686429/raw/main/flink-operator.yaml \
&& kubectl -n argocd apply -f flink-operator.yaml \
&& argocd app sync argocd/flink-operator

6. install git

sudo dnf install -y git \
&& rm -rf $HOME/cosmic-antenna-demo \
&& mkdir $HOME/cosmic-antenna-demo \
&& git clone --branch pv_pvc_template https://github.com/AaronYang2333/cosmic-antenna-demo.git $HOME/cosmic-antenna-demo

7. prepare application image

# cd into  $HOME/cosmic-antenna-demo
sudo dnf install -y java-11-openjdk.x86_64 \
&& $HOME/cosmic-antenna-demo/gradlew :s3sync:buildImage \
&& $HOME/cosmic-antenna-demo/gradlew :fpga-mock:buildImage

# save and load into cluster
VERSION="1.0.3"
podman save --quiet -o $DOCKER_IMAGE_PATH/fpga-mock_$VERSION.dim localhost/fpga-mock:$VERSION \
&& kind -n cs-cluster load image-archive $DOCKER_IMAGE_PATH/fpga-mock_$VERSION.dim
podman save --quiet -o $DOCKER_IMAGE_PATH/s3sync_$VERSION.dim localhost/s3sync:$VERSION \
&& kind -n cs-cluster load image-archive $DOCKER_IMAGE_PATH/s3sync_$VERSION.dim

kubectl -n flink edit role/flink -o yaml

kubectl -n flink edit role/flink -o yaml

8. prepare k8s resources [pv, pvc, sts]

cp -rf $HOME/cosmic-antenna-demo/flink/*.yaml /tmp \
&& podman exec -d cs-cluster-control-plane mkdir -p /mnt/flink-job
# create persist volume
kubectl -n flink create -f /tmp/pv.template.yaml
# create pv claim
kubectl -n flink create -f /tmp/pvc.template.yaml
# start up flink application
kubectl -n flink create -f /tmp/job.template.yaml
# start up ingress
kubectl -n flink create -f /tmp/ingress.forward.yaml

# start up fpga UDP client, sending data 
cp $HOME/cosmic-antenna-demo/fpga-mock/client.template.yaml /tmp \
&& kubectl -n flink create -f /tmp/client.template.yaml

9. check dashboard in browser

http://job-template-example.flink.lab.zjvis.net

Repo

you can get code from github

Reference

1. https://github.com/ben-wangz/blog/tree/main/docs/content/6.kubernetes/7.installation/ha-cluster
2. xxx

Yaml Crawler

Steps

1. define which web url you wanna crawl, lets say https://www.xxx.com/aaa.apex
2. create a page pojo org.example.business.page.MainPage to describe that page

Then you can create a yaml file named root-pages.yaml and its content is

- '@class': "org.example.business.page.MainPage"
  url: "https://www.xxx.com/aaa.apex"

3. and then define a process flow yaml file, implying how to process web pages the crawler will meet.

processorChain:
  - '@class': "org.example.crawler.core.processor.decorator.ExceptionRecord"
    processor:
      '@class': "org.example.crawler.core.processor.decorator.RetryControl"
      processor:
        '@class': "org.example.crawler.core.processor.decorator.SpeedControl"
        processor:
          '@class': "org.example.business.hs.code.MainPageProcessor"
          application: "app-name"
        time: 100
        unit: "MILLISECONDS"
      retryTimes: 1
  - '@class': "org.example.crawler.core.processor.decorator.ExceptionRecord"
    processor:
      '@class': "org.example.crawler.core.processor.decorator.RetryControl"
      processor:
        '@class': "org.example.crawler.core.processor.decorator.SpeedControl"
        processor:
          '@class': "org.example.crawler.core.processor.download.DownloadProcessor"
          pagePersist:
            '@class': "org.example.business.persist.DownloadPageDatabasePersist"
            downloadPageRepositoryBeanName: "downloadPageRepository"
          downloadPageTransformer:
            '@class': "org.example.crawler.download.DefaultDownloadPageTransformer"
          skipExists:
            '@class': "org.example.crawler.download.SkipExistsById"
        time: 1
        unit: "SECONDS"
      retryTimes: 1
nThreads: 1
pollWaitingTime: 30
pollWaitingTimeUnit: "SECONDS"
waitFinishedTimeout: 180
waitFinishedTimeUnit: "SECONDS" 

ExceptionRecord, RetryControl, SpeedControl are provided by the yaml crawler itself, dont worry. you only need to extend how to process your page MainPage, for example, you defined a MainPageProcessor. each processor will produce a set of other page or DownloadPage. DownloadPage like a ship containing information you need, and this framework will help you process DownloadPage and download or persist.

4. Vola, run your crawler then.

Repo

you can get code from github, gitlab

Job Privilieges

Template

apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  namespace: flink
  name: flink-deployment-manager
rules:
- apiGroups: 
  - flink.apache.org
  resources: 
  - flinkdeployments
  verbs: 
  - 'get'
  - 'list'
  - 'create'
  - 'update'
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: flink-deployment-manager-binding
  namespace: flink
subjects:
- kind: User
  name: "277293711358271379"  
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: Role
  name: flink-deployment-manager
  apiGroup: rbac.authorization.k8s.io

OSS Template

Template

apiVersion: "flink.apache.org/v1beta1"
kind: "FlinkDeployment"
metadata:
  name: "financial-job"
spec:
  image: "cr.registry.res.cloud.wuxi-yqgcy.cn/mirror/financial-topic:1.5-oss"
  flinkVersion: "v1_17"
  flinkConfiguration:
    taskmanager.numberOfTaskSlots: "8"
    fs.oss.endpoint: http://ay-test.oss-cn-jswx-xuelang-d01-a.ops.cloud.wuxi-yqgcy.cn/
    fs.oss.accessKeyId: 4gqOVOfQqCsCUwaC
    fs.oss.accessKeySecret: xxx
  ingress:
    template: "flink.k8s.io/{{namespace}}/{{name}}(/|$)(.*)"
    className: "nginx"
    annotations:
      cert-manager.io/cluster-issuer: "self-signed-ca-issuer"
      nginx.ingress.kubernetes.io/rewrite-target: "/$2"
  serviceAccount: "flink"
  podTemplate:
    apiVersion: "v1"
    kind: "Pod"
    metadata:
      name: "financial-job"
    spec:
      containers:
        - name: "flink-main-container"
          env:
            - name: ENABLE_BUILT_IN_PLUGINS
              value: flink-oss-fs-hadoop-1.17.2.jar
  jobManager:
    resource:
      memory: "2048m"
      cpu: 1
  taskManager:
    resource:
      memory: "2048m"
      cpu: 1
  job:
    jarURI: "local:///app/application.jar"
    parallelism: 1
    upgradeMode: "stateless"