chore: backup infrastructure configurations [skip ci]

nomad_backup/promtail.hcl

@@ -3,6 +3,14 @@ job "promtail" {
   type        = "system"  # Runs on every node
 
   group "promtail" {
+    count = 1
+
+    # Allow spreading across multiple node pools
+    constraint {
+      operator = "regexp"
+      attribute = "${node.pool}"
+      value = "^(default|backup)$"
+    }
     network {
       port "http" {
         static = 9080
@@ -50,6 +58,44 @@ clients:
   - url: http://{{ range service "loki" }}{{ .Address }}:{{ .Port }}{{ end }}/loki/api/v1/push
 
 scrape_configs:
+  # Scrape Consul server logs from journald
+  - job_name: consul-server
+    journal:
+      json: false
+      max_age: 12h
+      path: /var/log/journal
+      labels:
+        job: consul-server
+        component: server
+        host: {{ env "HOSTNAME" }}
+    relabel_configs:
+      - source_labels: ['__journal__systemd_unit']
+        regex: 'consul.service'
+        action: keep
+      - source_labels: ['__journal__hostname']
+        target_label: 'journal_host'
+      - source_labels: ['__journal__systemd_unit']
+        target_label: 'unit'
+
+  # Scrape Consul Template logs from journald
+  - job_name: consul-template
+    journal:
+      json: false
+      max_age: 12h
+      path: /var/log/journal
+      labels:
+        job: consul-template
+        component: template
+        host: {{ env "HOSTNAME" }}
+    relabel_configs:
+      - source_labels: ['__journal__systemd_unit']
+        regex: 'consul-template.service'
+        action: keep
+      - source_labels: ['__journal__hostname']
+        target_label: 'journal_host'
+      - source_labels: ['__journal__systemd_unit']
+        target_label: 'unit'
+
   # Scrape Nomad server logs from journald
   - job_name: nomad-server
     journal:
@@ -69,6 +115,63 @@ scrape_configs:
       - source_labels: ['__journal__systemd_unit']
         target_label: 'unit'
 
+  # Scrape CoreDNS logs from journald
+  - job_name: coredns
+    journal:
+      json: false
+      max_age: 12h
+      path: /var/log/journal
+      labels:
+        job: coredns
+        component: dns
+        host: {{ env "HOSTNAME" }}
+    relabel_configs:
+      - source_labels: ['__journal__systemd_unit']
+        regex: 'coredns.service'
+        action: keep
+      - source_labels: ['__journal__hostname']
+        target_label: 'journal_host'
+      - source_labels: ['__journal__systemd_unit']
+        target_label: 'unit'
+
+  # Scrape Docker daemon logs from journald
+  - job_name: docker-daemon
+    journal:
+      json: false
+      max_age: 12h
+      path: /var/log/journal
+      labels:
+        job: docker-daemon
+        component: docker
+        host: {{ env "HOSTNAME" }}
+    relabel_configs:
+      - source_labels: ['__journal__systemd_unit']
+        regex: 'docker.service'
+        action: keep
+      - source_labels: ['__journal__hostname']
+        target_label: 'journal_host'
+      - source_labels: ['__journal__systemd_unit']
+        target_label: 'unit'
+
+  # Scrape Keepalived logs from journald
+  - job_name: keepalived
+    journal:
+      json: false
+      max_age: 12h
+      path: /var/log/journal
+      labels:
+        job: keepalived
+        component: ha
+        host: {{ env "HOSTNAME" }}
+    relabel_configs:
+      - source_labels: ['__journal__systemd_unit']
+        regex: 'keepalived.service'
+        action: keep
+      - source_labels: ['__journal__hostname']
+        target_label: 'journal_host'
+      - source_labels: ['__journal__systemd_unit']
+        target_label: 'unit'
+
   # Scrape Nomad allocation logs
   - job_name: nomad-alloc-logs
     static_configs:
@@ -79,12 +182,11 @@ scrape_configs:
           host: {{ env "HOSTNAME" }}
           __path__: /alloc/logs/*
     
+    # Exclude Promtail's own FIFO files to avoid seek errors
     pipeline_stages:
       - match:
-          selector: '{job="nomad-logs"}'
-          stages:
-            - drop:
-                expression: '.*\.promtail\.(stdout|stderr)\.fifo.*'
+          selector: '{filename=~".*/\\.promtail\\.(stdout|stderr)\\.fifo"}'
+          action: drop
 
   # Scrape Docker container logs
   - job_name: docker