Установка системы мониторинга для кластерного развертывания

sudo tar xvf ./prometheus-2.33.0-rc.0.linux-amd64.tar.gz --directory /opt/

sudo nano /opt/prometheus-2.33.0-rc.0.linux-amd64/prometheus.yml

# my global config
global:
  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.

  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
- static_configs:
- targets:
# - alertmanager:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  # - "first_rules.yml"

  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:

  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: "prometheus"

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
      - targets: ["localhost:9090"]

- job_name: 'java-demo-entaxy-krf1' metrics_path: /metrics static_configs:
- targets: ['X.X.X.X:XXXX']

- job_name: 'java-demo-entaxy-krf2' metrics_path: /metrics static_configs:
- targets: ['X.X.X.X:XXXX']

- job_name: 'java-demo-entaxy-amq1' metrics_path: /metrics static_configs:
- targets: ['X.X.X.X:XXXX']

- job_name: 'demo-entaxy-krf1' metrics_path: /metrics static_configs:
- targets: ['X.X.X.X:XXXX']

- job_name: 'demo-entaxy-krf2' metrics_path: /metrics static_configs:
- targets: ['X.X.X.X:XXXX']

- job_name: 'demo-entaxy-amq1' metrics_path: /metrics static_configs:
- targets: ['X.X.X.X:XXXX']

- job_name: node metrics_path: /metrics static_configs:
- targets: ['X.X.X.X:XXXX']
- targets: ['X.X.X.X:XXXX']
- targets: ['X.X.X.X:XXXX']

sudo nano /etc/systemd/system/prometheus.service

[Unit]
Description=prometheus
After=network.target
StartLimitIntervalSec=0

[Service]
Type=simple
User=root
WorkingDirectory=/opt/prometheus-2.33.0-rc.0.linux-amd64
LimitAS=infinity
LimitRSS=infinity
LimitCORE=infinity
LimitNOFILE=65536
ExecStart=/opt/prometheus-2.33.0-rc.0.linux-amd64/prometheus
Restart=always

[Install]
WantedBy=multi-user.target

systemctl daemon-reload
systemctl enable prometheus.service

systemctl start prometheus.service
systemctl status prometheus.service

sudo unzip ./loki-linux-amd64.zip -d /opt/loki

chmod 755 /opt/loki/loki-linux-amd64

sudo nano /etc/systemd/system/loki.service

[Unit]
Description=loki
After=network.target
StartLimitIntervalSec=0

[Service]
Type=simple
User=root
WorkingDirectory=/opt/loki
LimitAS=infinity
LimitRSS=infinity
LimitCORE=infinity
LimitNOFILE=65536
ExecStart=/opt/loki/loki-linux-amd64 -config.file=/opt/loki/loki-local-config.yaml
Restart=always

[Install]
WantedBy=multi-user.target

sudo nano /opt/loki/loki-local-config.yaml

auth_enabled: false

server:
http_listen_port: 3100

ingester:
    lifecycler:
        address: 127.0.0.1
        ring:
            kvstore:
                store: inmemory
            replication_factor: 1
        final_sleep: 0s
    chunk_idle_period: 5m
    chunk_retain_period: 30s
    max_transfer_retries: 0

schema_config:
    configs:
        - from: 2018-04-15
        store: boltdb
        object_store: filesystem
        schema: v11
        index:
            prefix: index_
            period: 168h

storage_config:
    boltdb:
        directory: /opt/loki/data/index

  filesystem:
    directory: /opt/loki/data/chunks

limits_config:
    enforce_metric_name: false
    reject_old_samples: true
    reject_old_samples_max_age: 168h

chunk_store_config:
    max_look_back_period: 0s

table_manager:
    retention_deletes_enabled: false
    retention_period: 0s

sudo systemctl daemon-reload
sudo systemctl enable loki.service
sudo systemctl start loki.service
sudo systemctl status loki.service

[Unit]
Description=fluentbit
After=network.target
StartLimitIntervalSec=0

[Service]
Type=simple
User=root
WorkingDirectory=/opt/fluent-bit
LimitAS=infinity
LimitRSS=infinity
LimitCORE=infinity
LimitNOFILE=65536
ExecStart=/opt/fluent-bit/fluent-bit -e /opt/fluent-bit/out_loki.so -c /opt/fluent-bit/fluentBit.conf
Restart=always

[Install]
WantedBy=multi-user.target

sudo nano /opt/fluent-bit/fluentBit.conf

[SERVICE]
    Flush 5
    Daemon Off
    Log_Level info
    Parsers_File /opt/fluent-bit/parsers.conf

[INPUT]
    Name tail
    Path /opt/artemis/brokers/entaxy-broker/log/artemis.log
    Refresh_Interval 5
    Buffer_Chunk_Size 100k
    Buffer_Max_Size 1M
    Tag entaxy
    Key message
    Multiline On
    Multiline_Flush 1
    Parser_Firstline entaxy

[OUTPUT]
    Name loki
    Match *
    host X.X.X.X
    port 3100
    line_format json
    labels job=amq-1
    label_keys $log_level

[SERVICE]
    Flush        5
    Daemon       Off
    Log_Level    info
    Parsers_File /opt/fluent-bit/parsers.conf

[INPUT]
    Name tail
    Path /opt/karaf/data/log/karaf.log
    Refresh_Interval  5
    Buffer_Chunk_Size 100k
    Buffer_Max_Size 1M
    Tag entaxy
    Key message
    Multiline On
    Multiline_Flush 1
    Parser_Firstline entaxy

[OUTPUT]
    Name loki
    Match *
    host X.X.X.X
    port 3100
    line_format json
    labels job=entaxy-1, service=$data['service'], operation=$data['operation'], objectId=$data['objectId'], parameter=$data['parameter'], sender=$data['sender'], receiver=$data['receiver']
    label_keys $log_level, $request

sudo nano /opt/fluent-bit/labelmap.json

{
    "instance": "instance",
    "log_level": "log_level",
    "service": "service"
}

[PARSER]
        Name        observability
        Format      json
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L

[PARSER]
        Name entaxy
        Format regex
        Regex ^(?<date>\d{2,4}\-\d{2,4}\-\d{2,4}T\d{2,4}\:\d{2,4}\:\d{2,4}\,\d{1,6}) \| (?<log_level>[^\s]+)[\s]+\| (?<thread>[^\|]+) \| (?<object>[^\s]+)[\s]+\| (?<bundle>\d{2,4} \- [^\s]+ \- [^\s]
+) \| (((?<request>\w{8}\-\w{4}\-\w{4}-\w{4}\-\w{12}))#(?<data>[^#]+)#[\s]*?(?<step>.*)| ?(?<message>.*))$
        Decode_Field_As json data
        Time_Key date
        Time_Format %Y-%m-%dT%H:%M:%S,%L
        #Time_Offset +0300
        Time_Keep    On

[PARSER]
        Name        observability
        Format      json
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L

[PARSER]
        Name entaxy
        Format regex
        Regex ^(?<date>\d{2,4}\-\d{2,4}\-\d{2,4}T\d{2,4}\:\d{2,4}\:\d{2,4}\,\d{1,6}) \| (?<log_level>[^\s]+)[\s]+\| (?<thread>[^\|]+) \| (?<object>[^\s]+)[\s]+\| (?<bundle>\d{2,4} \- [^\s]+ \- [^\s]
+) \| ?(?<message>.*)$
        Time_Key date
        Time_Format %Y-%m-%dT%H:%M:%S,%f
        Time_Offset +0300

sudo nano /etc/systemd/system/fluentbit.service

[Unit]
Description=fluentbit
After=network.target
StartLimitIntervalSec=0

[Service]
Type=simple
User=root
WorkingDirectory=/opt/fluent-bit
LimitAS=infinity
LimitRSS=infinity
LimitCORE=infinity
LimitNOFILE=65536
ExecStart=/opt/fluent-bit/fluent-bit -e /opt/fluent-bit/out_loki.so -c /opt/fluent-bit/fluentBit.conf
Restart=always

[Install]
WantedBy=multi-user.target

sudo systemctl daemon-reload
sudo systemctl enable fluentbit.service
sudo systemctl start fluentbit.service
sudo systemctl status fluentbit.service

sudo tar xvf node_exporter-1.5.0.linux-amd64.tar.gz --directory /opt/

sudo nano /etc/systemd/system/node_exporter.service

[Unit]
Description=Node Exporter
After=network.target

[Service]
User=root
Type=simple
ExecStart=/opt/node_exporter-1.5.0.linux-amd64/node_exporter

[Install]
WantedBy=multi-user.target

sudo systemctl daemon-reload
sudo systemctl enable node_exporter.service
sudo systemctl start node_exporter.service
sudo systemctl status node_exporter.service

wget -q https://repos.influxdata.com/influxdata-archive_compat.key

echo '393e8779c89ac8d958f81f942f9ad7fb82a25e133faddaf92e15b16e6ac9ce4c influxdata-archive_compat.key' | sha256sum -c && cat influxdata-arch
ive_compat.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg > /dev/null

echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg] https://repos.influxdata.com/debian stable main' | sudo tee /etc
/apt/sources.list.d/influxdata.list

sudo apt-get update && sudo apt-get install telegraf