150 lines
3.6 KiB
HCL
150 lines
3.6 KiB
HCL
job "promtail" {
|
|
datacenters = ["dc1"]
|
|
type = "system" # Runs on every node
|
|
|
|
group "promtail" {
|
|
network {
|
|
port "http" {
|
|
static = 9080
|
|
}
|
|
}
|
|
|
|
task "promtail" {
|
|
driver = "docker"
|
|
|
|
config {
|
|
image = "grafana/promtail:2.9.3"
|
|
ports = ["http"]
|
|
|
|
args = [
|
|
"-config.file=/local/promtail-config.yml",
|
|
]
|
|
|
|
# Mount the Docker socket, journal, and machine-id for systemd logs
|
|
volumes = [
|
|
"/var/lib/docker/containers:/var/lib/docker/containers:ro",
|
|
"/var/log/journal:/var/log/journal:ro",
|
|
"/run/log/journal:/run/log/journal:ro",
|
|
"/etc/machine-id:/etc/machine-id:ro",
|
|
"/var/run/docker.sock:/var/run/docker.sock:ro",
|
|
]
|
|
|
|
# Run as root to access Docker socket
|
|
privileged = true
|
|
}
|
|
|
|
env {
|
|
HOSTNAME = "${node.unique.name}"
|
|
}
|
|
|
|
template {
|
|
data = <<EOH
|
|
server:
|
|
http_listen_port: 9080
|
|
grpc_listen_port: 0
|
|
|
|
positions:
|
|
filename: /alloc/data/positions.yaml
|
|
|
|
clients:
|
|
- url: http://{{ range service "loki" }}{{ .Address }}:{{ .Port }}{{ end }}/loki/api/v1/push
|
|
|
|
scrape_configs:
|
|
# Scrape Nomad server logs from journald
|
|
- job_name: nomad-server
|
|
journal:
|
|
json: false
|
|
max_age: 12h
|
|
path: /var/log/journal
|
|
labels:
|
|
job: nomad-server
|
|
component: server
|
|
host: {{ env "HOSTNAME" }}
|
|
relabel_configs:
|
|
- source_labels: ['__journal__systemd_unit']
|
|
regex: 'nomad.service'
|
|
action: keep
|
|
- source_labels: ['__journal__hostname']
|
|
target_label: 'journal_host'
|
|
- source_labels: ['__journal__systemd_unit']
|
|
target_label: 'unit'
|
|
|
|
# Scrape Nomad allocation logs
|
|
- job_name: nomad-alloc-logs
|
|
static_configs:
|
|
- targets:
|
|
- localhost
|
|
labels:
|
|
job: nomad-logs
|
|
host: {{ env "HOSTNAME" }}
|
|
__path__: /alloc/logs/*
|
|
|
|
pipeline_stages:
|
|
- match:
|
|
selector: '{job="nomad-logs"}'
|
|
stages:
|
|
- drop:
|
|
expression: '.*\.promtail\.(stdout|stderr)\.fifo.*'
|
|
|
|
# Scrape Docker container logs
|
|
- job_name: docker
|
|
docker_sd_configs:
|
|
- host: unix:///var/run/docker.sock
|
|
refresh_interval: 5s
|
|
|
|
relabel_configs:
|
|
# Add hostname label
|
|
- replacement: {{ env "HOSTNAME" }}
|
|
target_label: 'host'
|
|
|
|
# Extract container name
|
|
- source_labels: ['__meta_docker_container_name']
|
|
regex: '/(.*)'
|
|
target_label: 'container'
|
|
|
|
# Extract Nomad job name if available
|
|
- source_labels: ['__meta_docker_container_label_com_hashicorp_nomad_job_name']
|
|
target_label: 'nomad_job'
|
|
|
|
# Extract Nomad task name if available
|
|
- source_labels: ['__meta_docker_container_label_com_hashicorp_nomad_task_name']
|
|
target_label: 'nomad_task'
|
|
|
|
# Extract Nomad namespace if available
|
|
- source_labels: ['__meta_docker_container_label_com_hashicorp_nomad_namespace']
|
|
target_label: 'nomad_namespace'
|
|
|
|
# Set the log path
|
|
- source_labels: ['__meta_docker_container_id']
|
|
target_label: '__path__'
|
|
replacement: '/var/lib/docker/containers/$1/*.log'
|
|
EOH
|
|
|
|
destination = "local/promtail-config.yml"
|
|
change_mode = "restart"
|
|
}
|
|
|
|
resources {
|
|
cpu = 200
|
|
memory = 128
|
|
}
|
|
|
|
service {
|
|
name = "promtail"
|
|
port = "http"
|
|
|
|
tags = [
|
|
"logging",
|
|
]
|
|
|
|
check {
|
|
type = "http"
|
|
path = "/ready"
|
|
interval = "10s"
|
|
timeout = "2s"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|