Установка системы мониторинга

sudo tar xvf ./prometheus-2.33.0-rc.0.linux-amd64.tar.gz --directory /opt/

# my global config
global:
  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          # - alertmanager:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: "prometheus"

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
      - targets: ["localhost:9090"]

  - job_name: 'java-demo-entaxy-krf1'
    metrics_path: /metrics
    static_configs:
      - targets: ['X.X.X.X:XXXX']

  - job_name: 'java-demo-entaxy-krf2'
    metrics_path: /metrics
    static_configs:
      - targets: ['X.X.X.X:XXXX']

  - job_name: 'java-demo-entaxy-amq1'
    metrics_path: /metrics
    static_configs:
      - targets: ['X.X.X.X:XXXX']

  - job_name: 'demo-entaxy-krf1'
    metrics_path: /metrics
    static_configs:
      - targets: ['X.X.X.X:XXXX']

  - job_name: 'demo-entaxy-krf2'
    metrics_path: /metrics
    static_configs:
      - targets: ['X.X.X.X:XXXX']

  - job_name: 'demo-entaxy-amq1'
    metrics_path: /metrics
    static_configs:
      - targets: ['X.X.X.X:XXXX']

  - job_name: node
    metrics_path: /metrics
    static_configs:
      - targets: ['X.X.X.X:XXXX']
      - targets: ['X.X.X.X:XXXX']
      - targets: ['X.X.X.X:XXXX']

[Unit]
Description=prometheus
After=network.target
StartLimitIntervalSec=0

[Service]
Type=simple
User=root
WorkingDirectory=/opt/prometheus-2.33.0-rc.0.linux-amd64
LimitAS=infinity
LimitRSS=infinity
LimitCORE=infinity
LimitNOFILE=65536
ExecStart=/opt/prometheus-2.33.0-rc.0.linux-amd64/prometheus
Restart=always

[Install]
WantedBy=multi-user.target

systemctl daemon-reload
systemctl enable prometheus.service

systemctl start prometheus.service
systemctl status prometheus.service

[Unit]
Description=loki
After=network.target
StartLimitIntervalSec=0

[Service]
Type=simple
User=root
WorkingDirectory=/opt/loki
LimitAS=infinity
LimitRSS=infinity
LimitCORE=infinity
LimitNOFILE=65536
ExecStart=/opt/loki/loki-linux-amd64 -config.file=/opt/loki/loki-local-config.yaml
Restart=always

[Install]
WantedBy=multi-user.target

Создаем файл /opt/loki/loki-local-config.yaml
sudo nano /opt/loki/loki-local-config.yaml

auth_enabled: false

server:
  http_listen_port: 3100

ingester:
  lifecycler:
    address: 127.0.0.1
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
    final_sleep: 0s
  chunk_idle_period: 5m
  chunk_retain_period: 30s
  max_transfer_retries: 0

schema_config:
  configs:
    - from: 2018-04-15
      store: boltdb
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 168h

storage_config:
  boltdb:
    directory: /opt/loki/data/index

  filesystem:
    directory: /opt/loki/data/chunks

limits_config:
  enforce_metric_name: false
  reject_old_samples: true
  reject_old_samples_max_age: 168h

chunk_store_config:
  max_look_back_period: 0s

table_manager:
  retention_deletes_enabled: false
  retention_period: 0s

sudo systemctl daemon-reload
sudo systemctl enable loki.service
sudo systemctl start loki.service
sudo systemctl status loki.service

[Unit]
Description=fluentbit
fter=network.target
StartLimitIntervalSec=0

[Service]
Type=simple
User=root
WorkingDirectory=/opt/fluent-bit
LimitAS=infinity
LimitRSS=infinity
LimitCORE=infinity
LimitNOFILE=65536
ExecStart=/opt/fluent-bit/fluent-bit -e /opt/fluent-bit/out_loki.so -c /opt/fluent-bit/fluentBit.conf
Restart=always

[Install]
WantedBy=multi-user.target

[SERVICE]
    Flush        5
    Daemon       Off
    Log_Level    info
    Parsers_File /opt/fluent-bit/parsers.conf

[INPUT]
    Name tail
    Path /opt/artemis/brokers/entaxy-broker/log/artemis.log
    Refresh_Interval  5
    Buffer_Chunk_Size 100k
    Buffer_Max_Size 1M
    Tag entaxy
    Key message
    Multiline On
    Multiline_Flush 1
    Parser_Firstline entaxy

[OUTPUT]
    Name loki
    Match *
    host X.X.X.X
    port 3100
    line_format json
    labels job=amq-1
    label_keys $log_level

[SERVICE]
    Flush        5
    Daemon       Off
    Log_Level    info
    Parsers_File /opt/fluent-bit/parsers.conf

[INPUT]
    Name tail
    Path /opt/karaf/data/log/karaf.log
    Refresh_Interval  5
    Buffer_Chunk_Size 100k
    Buffer_Max_Size 1M
    Tag entaxy
    Key message
    Multiline On
    Multiline_Flush 1
    Parser_Firstline entaxy

[OUTPUT]
    Name loki
    Match *
    host X.X.X.X
    port 3100
    line_format json
    labels job=entaxy-1, service=$data['service'], operation=$data['operation'], objectId=$data['objectId'], parameter=$data['parameter'], sender=$data['sender'], receiver=$data['receiver']
    label_keys $log_level, $request

{
    "instance": "instance",
    "log_level": "log_level",
    "service": "service"
}

[PARSER]
        Name        observability
        Format      json
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L

[PARSER]
        Name entaxy
        Format regex
        Regex ^(?<date>\d{2,4}\-\d{2,4}\-\d{2,4}T\d{2,4}\:\d{2,4}\:\d{2,4}\,\d{1,6}) \| (?<log_level>[^\s]+)[\s]+\| (?<thread>[^\|]+) \| (?<object>[^\s]+)[\s]+\| (?<bundle>\d{2,4} \- [^\s]+ \- [^\s]
+) \| (((?<request>\w{8}\-\w{4}\-\w{4}-\w{4}\-\w{12}))#(?<data>[^#]+)#[\s]*?(?<step>.*)| ?(?<message>.*))$
        Decode_Field_As json data
        Time_Key date
        Time_Format %Y-%m-%dT%H:%M:%S,%L
        #Time_Offset +0300
        Time_Keep    On

[PARSER]
        Name        observability
        Format      json
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L

[PARSER]
        Name entaxy
        Format regex
        Regex ^(?<date>\d{2,4}\-\d{2,4}\-\d{2,4}T\d{2,4}\:\d{2,4}\:\d{2,4}\,\d{1,6}) \| (?<log_level>[^\s]+)[\s]+\| (?<thread>[^\|]+) \| (?<object>[^\s]+)[\s]+\| (?<bundle>\d{2,4} \- [^\s]+ \- [^\s]
+) \| ?(?<message>.*)$
        Time_Key date
        Time_Format %Y-%m-%dT%H:%M:%S,%f
        Time_Offset +0300

[Unit]
Description=fluentbit
After=network.target
StartLimitIntervalSec=0

[Service]
Type=simple
User=root
WorkingDirectory=/opt/fluent-bit
LimitAS=infinity
LimitRSS=infinity
LimitCORE=infinity
LimitNOFILE=65536
ExecStart=/opt/fluent-bit/fluent-bit -e /opt/fluent-bit/out_loki.so -c /opt/fluent-bit/fluentBit.conf
Restart=always

[Install]
WantedBy=multi-user.target

[Unit]
Description=Node Exporter
After=network.target

[Service]
User=root
Type=simple
ExecStart=/opt/node_exporter-1.5.0.linux-amd64/node_exporter

[Install]
WantedBy=multi-user.target

wget -q https://repos.influxdata.com/influxdata-archive_compat.key
echo '393e8779c89ac8d958f81f942f9ad7fb82a25e133faddaf92e15b16e6ac9ce4c influxdata-archive_compat.key' | sha256sum -c && cat influxdata-arch
ive_compat.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg > /dev/null
echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg] https://repos.influxdata.com/debian stable main' | sudo tee /etc
/apt/sources.list.d/influxdata.list
sudo apt-get update && sudo apt-get install telegraf