chore: backup infrastructure configurations [skip ci]

nomad_backup/promtail_backup.hcl

@@ -0,0 +1,151 @@
+job "promtail_backup" {
+  node_pool = "backup"
+
+  datacenters = ["dc1"]
+  type        = "system"  # Runs on every node
+
+  group "promtail" {
+    network {
+      port "http" {
+        static = 9080
+      }
+    }
+
+    task "promtail" {
+      driver = "docker"
+
+      config {
+        image = "grafana/promtail:2.9.3"
+        ports = ["http"]
+        
+        args = [
+          "-config.file=/local/promtail-config.yml",
+        ]
+
+        # Mount the Docker socket, journal, and machine-id for systemd logs
+        volumes = [
+          "/var/lib/docker/containers:/var/lib/docker/containers:ro",
+          "/var/log/journal:/var/log/journal:ro",
+          "/run/log/journal:/run/log/journal:ro",
+          "/etc/machine-id:/etc/machine-id:ro",
+          "/var/run/docker.sock:/var/run/docker.sock:ro",
+        ]
+        
+        # Run as root to access Docker socket
+        privileged = true
+      }
+
+      env {
+        HOSTNAME = "${node.unique.name}"
+      }
+
+      template {
+        data = <<EOH
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /alloc/data/positions.yaml
+
+clients:
+  - url: http://{{ range service "loki" }}{{ .Address }}:{{ .Port }}{{ end }}/loki/api/v1/push
+
+scrape_configs:
+  # Scrape Nomad server logs from journald
+  - job_name: nomad-server
+    journal:
+      json: false
+      max_age: 12h
+      path: /var/log/journal
+      labels:
+        job: nomad-server
+        component: server
+        host: {{ env "HOSTNAME" }}
+    relabel_configs:
+      - source_labels: ['__journal__systemd_unit']
+        regex: 'nomad.service'
+        action: keep
+      - source_labels: ['__journal__hostname']
+        target_label: 'journal_host'
+      - source_labels: ['__journal__systemd_unit']
+        target_label: 'unit'
+
+  # Scrape Nomad allocation logs
+  - job_name: nomad-alloc-logs
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: nomad-logs
+          host: {{ env "HOSTNAME" }}
+          __path__: /alloc/logs/*
+    
+    pipeline_stages:
+      - match:
+          selector: '{job="nomad-logs"}'
+          stages:
+            - drop:
+                expression: '.*\.promtail\.(stdout|stderr)\.fifo.*'
+
+  # Scrape Docker container logs
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+    
+    relabel_configs:
+      # Add hostname label
+      - replacement: {{ env "HOSTNAME" }}
+        target_label: 'host'
+      
+      # Extract container name
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: 'container'
+      
+      # Extract Nomad job name if available
+      - source_labels: ['__meta_docker_container_label_com_hashicorp_nomad_job_name']
+        target_label: 'nomad_job'
+      
+      # Extract Nomad task name if available
+      - source_labels: ['__meta_docker_container_label_com_hashicorp_nomad_task_name']
+        target_label: 'nomad_task'
+      
+      # Extract Nomad namespace if available
+      - source_labels: ['__meta_docker_container_label_com_hashicorp_nomad_namespace']
+        target_label: 'nomad_namespace'
+      
+      # Set the log path
+      - source_labels: ['__meta_docker_container_id']
+        target_label: '__path__'
+        replacement: '/var/lib/docker/containers/$1/*.log'
+EOH
+
+        destination = "local/promtail-config.yml"
+        change_mode = "restart"
+      }
+
+      resources {
+        cpu    = 200
+        memory = 128
+      }
+
+      service {
+        name = "promtail"
+        port = "http"
+        
+        tags = [
+          "logging",
+        ]
+
+        check {
+          type     = "http"
+          path     = "/ready"
+          interval = "10s"
+          timeout  = "2s"
+        }
+      }
+    }
+  }
+}