job "promtail" {
  datacenters = ["dc1"]
  type        = "system"  # Runs on every node

  group "promtail" {
    network {
      port "http" {
        static = 9080
      }
    }

    task "promtail" {
      driver = "docker"

      config {
        image = "grafana/promtail:2.9.3"
        ports = ["http"]
        
        args = [
          "-config.file=/local/promtail-config.yml",
        ]

        # Mount the Docker socket, journal, and machine-id for systemd logs
        volumes = [
          "/var/lib/docker/containers:/var/lib/docker/containers:ro",
          "/var/log/journal:/var/log/journal:ro",
          "/run/log/journal:/run/log/journal:ro",
          "/etc/machine-id:/etc/machine-id:ro",
          "/var/run/docker.sock:/var/run/docker.sock:ro",
        ]
        
        # Run as root to access Docker socket
        privileged = true
      }

      env {
        HOSTNAME = "${node.unique.name}"
      }

      template {
        data = <<EOH
server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /alloc/data/positions.yaml

clients:
  - url: http://{{ range service "loki" }}{{ .Address }}:{{ .Port }}{{ end }}/loki/api/v1/push

scrape_configs:
  # Scrape Nomad server logs from journald
  - job_name: nomad-server
    journal:
      json: false
      max_age: 12h
      path: /var/log/journal
      labels:
        job: nomad-server
        component: server
        host: {{ env "HOSTNAME" }}
    relabel_configs:
      - source_labels: ['__journal__systemd_unit']
        regex: 'nomad.service'
        action: keep
      - source_labels: ['__journal__hostname']
        target_label: 'journal_host'
      - source_labels: ['__journal__systemd_unit']
        target_label: 'unit'

  # Scrape Nomad allocation logs
  - job_name: nomad-alloc-logs
    static_configs:
      - targets:
          - localhost
        labels:
          job: nomad-logs
          host: {{ env "HOSTNAME" }}
          __path__: /alloc/logs/*
    
    pipeline_stages:
      - match:
          selector: '{job="nomad-logs"}'
          stages:
            - drop:
                expression: '.*\.promtail\.(stdout|stderr)\.fifo.*'

  # Scrape Docker container logs
  - job_name: docker
    docker_sd_configs:
      - host: unix:///var/run/docker.sock
        refresh_interval: 5s
    
    relabel_configs:
      # Add hostname label
      - replacement: {{ env "HOSTNAME" }}
        target_label: 'host'
      
      # Extract container name
      - source_labels: ['__meta_docker_container_name']
        regex: '/(.*)'
        target_label: 'container'
      
      # Extract Nomad job name if available
      - source_labels: ['__meta_docker_container_label_com_hashicorp_nomad_job_name']
        target_label: 'nomad_job'
      
      # Extract Nomad task name if available
      - source_labels: ['__meta_docker_container_label_com_hashicorp_nomad_task_name']
        target_label: 'nomad_task'
      
      # Extract Nomad namespace if available
      - source_labels: ['__meta_docker_container_label_com_hashicorp_nomad_namespace']
        target_label: 'nomad_namespace'
      
      # Set the log path
      - source_labels: ['__meta_docker_container_id']
        target_label: '__path__'
        replacement: '/var/lib/docker/containers/$1/*.log'
EOH

        destination = "local/promtail-config.yml"
        change_mode = "restart"
      }

      resources {
        cpu    = 200
        memory = 128
      }

      service {
        name = "promtail"
        port = "http"
        
        tags = [
          "logging",
        ]

        check {
          type     = "http"
          path     = "/ready"
          interval = "10s"
          timeout  = "2s"
        }
      }
    }
  }
}