sync - adding coredns!

hosts/opti1/default.nix

@@ -7,7 +7,7 @@
 }: {
   imports = [
     ./hardware-configuration.nix
-    # ../../modules/keepalived.nix
+    ../../modules/keepalived.nix
   ];
 
   nixpkgs.hostPlatform.system = "x86_64-linux";

modules/common.nix

@@ -10,6 +10,7 @@
     # ./mnt-clusterstore.nix
     ./nomad.nix
     ./consul.nix
+    ./coredns.nix
     inputs.sops-nix.nixosModules.sops
   ];
 
@@ -55,6 +56,7 @@
       pkgs.ncdu
       pkgs.killall
       pkgs.dig
+      pkgs.jq
     ];
 
     networking.search = ["node.dc1.consul" "service.dc1.consul"];

modules/consul.nix

@@ -7,69 +7,26 @@
 }: let
   secretstore = config._secretstore;
   NetworkInterface = config.custom._Networkinterface;
-
-  # oldpkgs = import (builtins.fetchGit {
-  #   # Descriptive name to make the store path easier to identify
-  #   name = "git_consul_1_9";
-  #   url = "https://github.com/NixOS/nixpkgs/";
-  #   ref = "refs/heads/nixpkgs-unstable";
-  #   rev = "3b05df1d13c1b315cecc610a2f3180f6669442f0";
-  # }) {};
-  # oldpkgs = import (builtins.fetchTarball {
-  #   url = "https://github.com/NixOS/nixpkgs/archive/3b05df1d13c1b315cecc610a2f3180f6669442f0.tar.gz";
-  #   sha256 = "1dr7kfdl4wvxhml4hd9k77xszl55vbjbb6ssirs2qv53mgw8c24w";
-  # }) {};
-  # myPkg = oldpkgs.consul;
 in {
-  # virtualisation.docker.enable = true;
   sops.secrets."consul_encrypt.json" = {
     sopsFile = "${secretstore}/consul_encrypt.json";
     format = "binary";
     owner = "consul";
     group = "consul";
-
-
-
   };
-
+  
   networking.firewall = {
     enable = true;
-    allowedTCPPorts = [8300 8301 8302 8500 8600 53];
+    # Remove port 53 - CoreDNS will handle it
-    allowedUDPPorts = [8301 3802 8600 53];
+    allowedTCPPorts = [8300 8301 8302 8500 8600];
+    allowedUDPPorts = [8301 3802 8600];
   };
 
-# Add the iptables rules directly via a systemd service
+  # REMOVE the consul-dns-redirect service - no longer needed
-  systemd.services.consul-dns-redirect = {
+  
-    description = "Redirect DNS port 53 to Consul port 8600";
-    after = [ "network.target" "firewall.service" ];
-    wantedBy = [ "multi-user.target" ];
-    
-    serviceConfig = {
-      Type = "oneshot";
-      RemainAfterExit = true;
-    };
-    
-    script = ''
-      ${pkgs.iptables}/bin/iptables -t nat -A PREROUTING -p udp --dport 53 -j REDIRECT --to-port 8600
-      ${pkgs.iptables}/bin/iptables -t nat -A PREROUTING -p tcp --dport 53 -j REDIRECT --to-port 8600
-      ${pkgs.iptables}/bin/iptables -t nat -A OUTPUT -p udp -d 127.0.0.1 --dport 53 -j REDIRECT --to-port 8600
-      ${pkgs.iptables}/bin/iptables -t nat -A OUTPUT -p tcp -d 127.0.0.1 --dport 53 -j REDIRECT --to-port 8600
-    '';
-    
-    preStop = ''
-      ${pkgs.iptables}/bin/iptables -t nat -D PREROUTING -p udp --dport 53 -j REDIRECT --to-port 8600 || true
-      ${pkgs.iptables}/bin/iptables -t nat -D PREROUTING -p tcp --dport 53 -j REDIRECT --to-port 8600 || true
-      ${pkgs.iptables}/bin/iptables -t nat -D OUTPUT -p udp -d 127.0.0.1 --dport 53 -j REDIRECT --to-port 8600 || true
-      ${pkgs.iptables}/bin/iptables -t nat -D OUTPUT -p tcp -d 127.0.0.1 --dport 53 -j REDIRECT --to-port 8600 || true
-    '';
-  };
-
-
   services.consul = {
-    # package = myPkg;
     enable = true;
     webUi = true;
-    # consulAddr = "0.0.0.0:8500";
     interface.bind = "${NetworkInterface}";
     extraConfigFiles = [config.sops.secrets."consul_encrypt.json".path];
     extraConfig = {
@@ -77,12 +34,12 @@ in {
       server = true;
       bootstrap_expect = 3;
       addresses = {
-        dns = "0.0.0.0";
+        # Bind DNS only to localhost since CoreDNS will forward to it
+        dns = "127.0.0.1";
         grpc = "0.0.0.0";
         http = "0.0.0.0";
         https = "0.0.0.0";
       };
-
       performance = {
         raft_multiplier = 7;
       };
@@ -90,7 +47,7 @@ in {
         "192.168.4.1"
         "8.8.8.8"
       ];
-
+      alt_domain = "fbleagh.duckdns.org";
       retry_join = [
         "192.168.4.221"
         "192.168.4.222"

modules/coredns.nix

@@ -0,0 +1,149 @@
+{
+  lib,
+  pkgs,
+  config,
+  ...
+}: let
+  # 1. Merged Hosts Template
+  consulAllHostsTemplate = pkgs.writeText "consul-all-hosts.ctmpl" ''
+# --- Static Hosts from Consul KV ---
+{{ printf "\n" }}
+{{- range ls "dns/hosts" -}}
+{{ .Value }} {{ .Key }}
+{{ printf "\n" }}
+{{- end -}}
+
+# --- Dynamic Hosts from Consul Services (Traefik Tags) ---
+{{ printf "\n" }}
+{{- range services -}}
+  {{- range service .Name -}}
+    {{- /* Determine IP: Use Service Address, fall back to Node Address */ -}}
+    {{- $ip := .Address -}}
+    {{- if eq $ip "" -}}
+      {{- $ip = .NodeAddress -}}
+    {{- end -}}
+
+    {{- /* Scan Tags */ -}}
+    {{- range .Tags -}}
+      {{- if . | regexMatch "traefik.http.routers.*.rule=Host" -}}
+        
+        {{- /* 1. Extract content inside Host(...) */ -}}
+        {{- $content := . | regexReplaceAll ".*Host\\(([^)]+)\\).*" "$1" -}}
+
+        {{- /* 2. Clean up quotes and spaces */ -}}
+        {{- $clean := $content | regexReplaceAll "[`'\"\\s]" "" -}}
+
+        {{- /* 3. Split by comma and print */ -}}
+        {{- range split "," $clean -}}
+          {{- if ne . "" -}}
+{{ $ip }} {{ . }}
+{{ printf "\n" }}
+          {{- end -}}
+        {{- end -}}
+      {{- end -}}
+    {{- end -}}
+  {{- end -}}
+{{- end -}}
+  '';
+
+  # 2. Wrapper script to ensure clean execution and environment setup
+  consulTemplateWrapper = pkgs.writeShellScript "consul-template-wrapper" ''
+    # Only render the single merged template file
+    ${pkgs.consul-template}/bin/consul-template \
+      -template "${consulAllHostsTemplate}:/etc/coredns/consul-all-hosts:${pkgs.systemd}/bin/systemctl reload coredns" \
+      -log-level info
+  '';
+
+in {
+  # --- CoreDNS Configuration ---
+  environment.etc."coredns/Corefile".text = ''
+    # Forward Consul DNS queries to the local Consul Agent
+    consul:53 {
+      forward . 127.0.0.1:8600
+      cache 30
+      errors
+      log
+    }
+
+    # Handle custom domain
+    fbleagh.duckdns.org:53 {
+      # CRITICAL FIX: Use only one hosts file/plugin definition
+      hosts /etc/coredns/consul-all-hosts {
+        fallthrough
+      }
+      # Fallback to upstream DNS
+      forward . 192.168.4.1 8.8.8.8
+      cache 30
+      errors
+      log
+    }
+
+    # Handle all other DNS queries
+    .:53 {
+      forward . 192.168.4.1 8.8.8.8
+      cache 30
+      errors
+      log
+    }
+  '';
+
+  systemd.tmpfiles.rules = [
+    "f /etc/coredns/consul-all-hosts 0644 root root - #"
+  ];
+
+  # --- Consul Template Service ---
+  systemd.services.consul-template = {
+    description = "Consul Template for CoreDNS Hosts";
+    wantedBy = ["multi-user.target"];
+    after = ["consul.service" "coredns.service" "network.target"];
+    requires = ["coredns.service"]; 
+
+    serviceConfig = {
+      # Use the robust wrapper script
+      ExecStart = "${consulTemplateWrapper}";
+      
+      Restart = "always";
+      RestartSec = "10s";
+    };
+  };
+
+  # --- CoreDNS Service ---
+  systemd.services.coredns = {
+    description = "CoreDNS DNS server";
+    wantedBy = ["multi-user.target"];
+    requires = ["consul.service"];
+    after = ["network.target" "consul.service"];
+    
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${pkgs.coredns}/bin/coredns -conf /etc/coredns/Corefile";
+      ExecReload = "${pkgs.coreutils}/bin/kill -SIGUSR1 $MAINPID";
+      Restart = "on-failure";
+      RestartSec = "5s";
+      DynamicUser = true;
+      AmbientCapabilities = "CAP_NET_BIND_SERVICE";
+      CapabilityBoundingSet = "CAP_NET_BIND_SERVICE";
+      NoNewPrivileges = true;
+      ProtectSystem = "strict";
+      ProtectHome = true;
+      PrivateTmp = true;
+      ReadWritePaths = "/etc/coredns";
+    };
+  };
+
+  # --- Helper Scripts and Firewall ---
+  environment.systemPackages = [
+    pkgs.consul-template
+    (pkgs.writeShellScriptBin "debug-consul-template" ''
+      echo "Rendering template to stdout..."
+      ${pkgs.consul-template}/bin/consul-template \
+        -template "${consulAllHostsTemplate}:-" \
+        -dry | grep -v "^$"
+    '') # <--- This is the crucial closing of the multi-line string
+  ];
+
+  networking.firewall = {
+    allowedTCPPorts = [53];
+    allowedUDPPorts = [53];
+  };
+}

modules/keepalived.nix

@@ -1,22 +1,49 @@
 { config, pkgs, lib, ... }:
 with lib; let
   NetworkInterface = config.custom._Networkinterface;
-  VIP_Priority = config.custom.VIP_Priority;
+  
+  # Extract the first IPv4 address from the configured network interface
+  # This gets the IP from networking.interfaces.<interface>.ipv4.addresses
+  getHostIp = interfaceName:
+    let
+      interfaceConfig = config.networking.interfaces.${interfaceName} or {};
+      addresses = interfaceConfig.ipv4.addresses or [];
+    in
+      if (length addresses) > 0
+      then (head addresses).address
+      else throw "No IPv4 address configured for interface ${interfaceName}";
+  
+  thisHostIp = getHostIp NetworkInterface;
+  
+  # Define priorities based on IP
+  priorityByIp = {
+    "192.168.4.226" = 100;  # Master
+    "192.168.4.227" = 90;   # Backup 1
+    "192.168.4.228" = 80;   # Backup 2
+    "192.168.4.36" = 70;    # Backup 3
+  };
+  
+  # All cluster nodes
+  allNodes = ["192.168.4.226" "192.168.4.227" "192.168.4.228" "192.168.4.36"];
+  
+  # Filter out this host from unicast peers
+  unicastPeers = filter (ip: ip != thisHostIp) allNodes;
+  
+  # Get priority for this host (default to 50 if not found)
+  VIP_Priority = priorityByIp.${thisHostIp} or 50;
+  
 in {
-
-
   services.keepalived = {
     enable = true;
     openFirewall = true;
     vrrpInstances.VIP_250 = {
-        interface = "${NetworkInterface}";
+      interface = "${NetworkInterface}";  
-        virtualRouterId  = 51;
+      virtualRouterId = 51;
-        priority = VIP_Priority;
+      priority = VIP_Priority;
-        unicastPeers = ["192.168.4.226" "192.168.4.227" "192.168.4.228" "192.168.4.36"];
+      unicastPeers = unicastPeers;
-        virtualIps = [{addr = "192.168.4.250/22";}];
+      virtualIps = [{addr = "192.168.4.250/22";}];
-          
+    };
-      };
   };
 
-  networking.firewall.allowedTCPPorts = [ 80 443 ]; # optional
+  networking.firewall.allowedTCPPorts = [ 80 443 ];
-}
+}

modules/nomad.nix

@@ -5,7 +5,20 @@
   inputs,
   ...
 }: {
-  virtualisation.docker.enable = true;
+  virtualisation.docker = {
+    enable = true;
+    # Configure insecure registries
+    daemon.settings = {
+      ipv6 = false;  # Add this line to disable IPv6
+      insecure-registries = [
+        "localhost:5000"
+        "gitea.service.dc1.consul:80"
+        "192.168.4.109:5000"
+      ];
+    };
+};
+
+
 
   networking.firewall = {
     allowedTCPPorts = [