ubuntuでdockerを使用してprometheusを動かす

ubuntuにdocker-ceをインストール（公式参照）
https://matsuand.github.io/docs.docker.jp.onthefly/engine/install/ubuntu/
試してないけど多分これでもOK
https://github.com/docker/docker-install

docker-compose-pluginがdocker v2の事なので、
docker-composeはインストール不要。

構成情報

$ pwd
/usr/local/docker

/usr/local/docker$ tree
.
├ blackbox_exporter
│  └ config.yml
├ compose.yml
├ prometheus
│  ├ prometheus.yml
│  ├ target_blackbox.yml
│  └ target_server.yml
└ snmp_exporter
   ├ generator.yml
   ├ mibs
   └ snmp.yml

compose.yml

$ cat compose.yml
services:
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus/target_blackbox.yml:/etc/prometheus/target_blackbox.yml
      - ./prometheus/target_server.yml:/etc/prometheus/target_server.yml
      - prometheus-data:/prometheus/
    ports:
      - 9090:9090
    restart: always

  node-exporter:
    image: prom/node-exporter:latest
    container_name: node-exporter
    ports:
      - 9100:9100
    restart: always

  blackbox-exporter:
    image: prom/blackbox-exporter:latest
    container_name: blackbox-exporter
    volumes:
      - ./blackbox_exporter/config.yml:/etc/blackbox_exporter/config.yml
    ports:
      - 9115:9115
    restart: always

  snmp-exporter:
    image: prom/snmp-exporter:latest
    container_name: snmp-exporter
    volumes:
      - ./snmp_exporter/snmp.yml:/etc/snmp_exporter/snmp.yml
    ports:
      - 9116:9116
    restart: always

  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    volumes:
     - grafana-data:/var/lib/grafana
    ports:
      - 3000:3000
    environment:
      TZ: Asia/Tokyo
    restart: always

volumes:
  prometheus-data:
  grafana-data:
# ホストOS上に下記ディレクトリが生成される
# /var/lib/docker/volumes/docker_prometheus-data/
# /var/lib/docker/volumes/docker_grafana-data/

prometheus.yml

$ cat prometheus/prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: "prometheus"
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'node'
    static_configs:
  - targets: ['node-exporter:9100']

  - job_name: 'blackbox'
    metrics_path: /probe
    params:
      module: [icmp]
    file_sd_configs:
      - files:
        - './target_blackbox.yml'
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

  - job_name: 'snmp-exporter-server'
    scrape_interval: 5m
    file_sd_configs:
      - files:
        - './target_server.yml'
    metrics_path: /snmp
    params:
      module: [if_mib]
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: snmp-exporter:9116
    metric_relabel_configs:
      - source_labels: [ifName]
        regex: 'lo|br-.*|docker.*|veth.*|wlp20s0'
        action: drop

target_blackbox.yml

$ cat prometheus/target_blackbox.yml
- targets:
  - 192.168.1.X

grafanaのvariable
https://grafana.com/docs/grafana/latest/dashboards/variables/add-template-variables/