blackbox_exporter でicmp,http監視、grafanaでslackにアラート通知

icmp監視

docker-compose.yml

version: '3'
services:

  prometheus:
    image: prom/prometheus
    container_name: prometheus
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
      #- ./prometheus-data:/prometheus
    ports:
      - 9090:9090

  blackbox_exporter:
    image: prom/blackbox-exporter
    volumes:
      - ./blackbox.yml:/etc/blackbox_exporter/config.yml:ro
    ports:
      - 9115:9115

• blackbox.yml に記述

blackbox.yml

modules:
  icmp:
    prober: icmp

• prometheus.yml

prometheus.yml

global:
  scrape_interval:     15s
  evaluation_interval: 15s

scrape_configs:

  - job_name: 'blackbox'
    metrics_path: /probe
    params:
      module: [icmp]
    static_configs:
      - targets:
        - 192.168.100.55
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox_exporter:9115

prometheusでグラフ表示

• probe_successで検索。
• 1がUPで、0がdown

---

http監視

• prometheus - Probes returning zero - Stack Overflow

blackbox.yml

modules:

  icmp:
    prober: icmp

  http_2xx:
    prober: http
    timeout: 15s
    http:
      valid_status_codes: []
      method: GET
      preferred_ip_protocol: "ip4"

prometheus.yml

global:
  scrape_interval:     15s
  evaluation_interval: 15s

scrape_configs:

  - job_name: 'blackbox-icmp'
    metrics_path: /probe
    params:
      module: [icmp]
    static_configs:
      - targets:
        - 192.168.100.55
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox_exporter:9115

  - job_name: 'blackbox-http'
    scrape_timeout: 15s
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
        - https://qiita.com/
        labels:
          group: 'response'
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox_exporter:9115

• probe_success

---

Grafana

docker-compose

docker-compose.yml

version: '3'
services:

  prometheus:
    image: prom/prometheus
    container_name: prometheus
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
      #- ./prometheus-data:/prometheus
    ports:
      - 9090:9090

  blackbox_exporter:
    image: prom/blackbox-exporter
    volumes:
      - ./blackbox.yml:/etc/blackbox_exporter/config.yml:ro
    ports:
      - 9115:9115

  grafana:
    image: grafana/grafana
    container_name: grafana
    volumes:
      - ./grafana-data:/var/lib/grafana
    ports:
      - 3000:3000

mkdir grafana-data
chmod 777 grafana-data

• http://localhost:3000

  • admin / admin

• Data Sources

• Add data source

• Prometheus - Select

• URL に http://prometheus:9090/

• probe_success を 指定するとグラフが描画される。

• save

アラート設定

• 1回でも疎通に失敗した場合はアラート通知の例

docker stats

• そんなに負荷は無いので使いながら様子を見てみる。

plugins

• https://grafana.com/grafana/plugins に多く用意されている。