mirror of
https://github.com/sstent/nixos-cluster.git
synced 2025-12-05 21:51:51 +00:00
sync - adding coredns!
This commit is contained in:
@@ -7,7 +7,7 @@
|
||||
}: {
|
||||
imports = [
|
||||
./hardware-configuration.nix
|
||||
# ../../modules/keepalived.nix
|
||||
../../modules/keepalived.nix
|
||||
];
|
||||
|
||||
nixpkgs.hostPlatform.system = "x86_64-linux";
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
# ./mnt-clusterstore.nix
|
||||
./nomad.nix
|
||||
./consul.nix
|
||||
./coredns.nix
|
||||
inputs.sops-nix.nixosModules.sops
|
||||
];
|
||||
|
||||
@@ -55,6 +56,7 @@
|
||||
pkgs.ncdu
|
||||
pkgs.killall
|
||||
pkgs.dig
|
||||
pkgs.jq
|
||||
];
|
||||
|
||||
networking.search = ["node.dc1.consul" "service.dc1.consul"];
|
||||
|
||||
@@ -7,69 +7,26 @@
|
||||
}: let
|
||||
secretstore = config._secretstore;
|
||||
NetworkInterface = config.custom._Networkinterface;
|
||||
|
||||
# oldpkgs = import (builtins.fetchGit {
|
||||
# # Descriptive name to make the store path easier to identify
|
||||
# name = "git_consul_1_9";
|
||||
# url = "https://github.com/NixOS/nixpkgs/";
|
||||
# ref = "refs/heads/nixpkgs-unstable";
|
||||
# rev = "3b05df1d13c1b315cecc610a2f3180f6669442f0";
|
||||
# }) {};
|
||||
# oldpkgs = import (builtins.fetchTarball {
|
||||
# url = "https://github.com/NixOS/nixpkgs/archive/3b05df1d13c1b315cecc610a2f3180f6669442f0.tar.gz";
|
||||
# sha256 = "1dr7kfdl4wvxhml4hd9k77xszl55vbjbb6ssirs2qv53mgw8c24w";
|
||||
# }) {};
|
||||
# myPkg = oldpkgs.consul;
|
||||
in {
|
||||
# virtualisation.docker.enable = true;
|
||||
sops.secrets."consul_encrypt.json" = {
|
||||
sopsFile = "${secretstore}/consul_encrypt.json";
|
||||
format = "binary";
|
||||
owner = "consul";
|
||||
group = "consul";
|
||||
|
||||
|
||||
|
||||
};
|
||||
|
||||
|
||||
networking.firewall = {
|
||||
enable = true;
|
||||
allowedTCPPorts = [8300 8301 8302 8500 8600 53];
|
||||
allowedUDPPorts = [8301 3802 8600 53];
|
||||
# Remove port 53 - CoreDNS will handle it
|
||||
allowedTCPPorts = [8300 8301 8302 8500 8600];
|
||||
allowedUDPPorts = [8301 3802 8600];
|
||||
};
|
||||
|
||||
# Add the iptables rules directly via a systemd service
|
||||
systemd.services.consul-dns-redirect = {
|
||||
description = "Redirect DNS port 53 to Consul port 8600";
|
||||
after = [ "network.target" "firewall.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
|
||||
script = ''
|
||||
${pkgs.iptables}/bin/iptables -t nat -A PREROUTING -p udp --dport 53 -j REDIRECT --to-port 8600
|
||||
${pkgs.iptables}/bin/iptables -t nat -A PREROUTING -p tcp --dport 53 -j REDIRECT --to-port 8600
|
||||
${pkgs.iptables}/bin/iptables -t nat -A OUTPUT -p udp -d 127.0.0.1 --dport 53 -j REDIRECT --to-port 8600
|
||||
${pkgs.iptables}/bin/iptables -t nat -A OUTPUT -p tcp -d 127.0.0.1 --dport 53 -j REDIRECT --to-port 8600
|
||||
'';
|
||||
|
||||
preStop = ''
|
||||
${pkgs.iptables}/bin/iptables -t nat -D PREROUTING -p udp --dport 53 -j REDIRECT --to-port 8600 || true
|
||||
${pkgs.iptables}/bin/iptables -t nat -D PREROUTING -p tcp --dport 53 -j REDIRECT --to-port 8600 || true
|
||||
${pkgs.iptables}/bin/iptables -t nat -D OUTPUT -p udp -d 127.0.0.1 --dport 53 -j REDIRECT --to-port 8600 || true
|
||||
${pkgs.iptables}/bin/iptables -t nat -D OUTPUT -p tcp -d 127.0.0.1 --dport 53 -j REDIRECT --to-port 8600 || true
|
||||
'';
|
||||
};
|
||||
|
||||
|
||||
# REMOVE the consul-dns-redirect service - no longer needed
|
||||
|
||||
services.consul = {
|
||||
# package = myPkg;
|
||||
enable = true;
|
||||
webUi = true;
|
||||
# consulAddr = "0.0.0.0:8500";
|
||||
interface.bind = "${NetworkInterface}";
|
||||
extraConfigFiles = [config.sops.secrets."consul_encrypt.json".path];
|
||||
extraConfig = {
|
||||
@@ -77,12 +34,12 @@ in {
|
||||
server = true;
|
||||
bootstrap_expect = 3;
|
||||
addresses = {
|
||||
dns = "0.0.0.0";
|
||||
# Bind DNS only to localhost since CoreDNS will forward to it
|
||||
dns = "127.0.0.1";
|
||||
grpc = "0.0.0.0";
|
||||
http = "0.0.0.0";
|
||||
https = "0.0.0.0";
|
||||
};
|
||||
|
||||
performance = {
|
||||
raft_multiplier = 7;
|
||||
};
|
||||
@@ -90,7 +47,7 @@ in {
|
||||
"192.168.4.1"
|
||||
"8.8.8.8"
|
||||
];
|
||||
|
||||
alt_domain = "fbleagh.duckdns.org";
|
||||
retry_join = [
|
||||
"192.168.4.221"
|
||||
"192.168.4.222"
|
||||
|
||||
149
modules/coredns.nix
Normal file
149
modules/coredns.nix
Normal file
@@ -0,0 +1,149 @@
|
||||
{
|
||||
lib,
|
||||
pkgs,
|
||||
config,
|
||||
...
|
||||
}: let
|
||||
# 1. Merged Hosts Template
|
||||
consulAllHostsTemplate = pkgs.writeText "consul-all-hosts.ctmpl" ''
|
||||
# --- Static Hosts from Consul KV ---
|
||||
{{ printf "\n" }}
|
||||
{{- range ls "dns/hosts" -}}
|
||||
{{ .Value }} {{ .Key }}
|
||||
{{ printf "\n" }}
|
||||
{{- end -}}
|
||||
|
||||
# --- Dynamic Hosts from Consul Services (Traefik Tags) ---
|
||||
{{ printf "\n" }}
|
||||
{{- range services -}}
|
||||
{{- range service .Name -}}
|
||||
{{- /* Determine IP: Use Service Address, fall back to Node Address */ -}}
|
||||
{{- $ip := .Address -}}
|
||||
{{- if eq $ip "" -}}
|
||||
{{- $ip = .NodeAddress -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- /* Scan Tags */ -}}
|
||||
{{- range .Tags -}}
|
||||
{{- if . | regexMatch "traefik.http.routers.*.rule=Host" -}}
|
||||
|
||||
{{- /* 1. Extract content inside Host(...) */ -}}
|
||||
{{- $content := . | regexReplaceAll ".*Host\\(([^)]+)\\).*" "$1" -}}
|
||||
|
||||
{{- /* 2. Clean up quotes and spaces */ -}}
|
||||
{{- $clean := $content | regexReplaceAll "[`'\"\\s]" "" -}}
|
||||
|
||||
{{- /* 3. Split by comma and print */ -}}
|
||||
{{- range split "," $clean -}}
|
||||
{{- if ne . "" -}}
|
||||
{{ $ip }} {{ . }}
|
||||
{{ printf "\n" }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
'';
|
||||
|
||||
# 2. Wrapper script to ensure clean execution and environment setup
|
||||
consulTemplateWrapper = pkgs.writeShellScript "consul-template-wrapper" ''
|
||||
# Only render the single merged template file
|
||||
${pkgs.consul-template}/bin/consul-template \
|
||||
-template "${consulAllHostsTemplate}:/etc/coredns/consul-all-hosts:${pkgs.systemd}/bin/systemctl reload coredns" \
|
||||
-log-level info
|
||||
'';
|
||||
|
||||
in {
|
||||
# --- CoreDNS Configuration ---
|
||||
environment.etc."coredns/Corefile".text = ''
|
||||
# Forward Consul DNS queries to the local Consul Agent
|
||||
consul:53 {
|
||||
forward . 127.0.0.1:8600
|
||||
cache 30
|
||||
errors
|
||||
log
|
||||
}
|
||||
|
||||
# Handle custom domain
|
||||
fbleagh.duckdns.org:53 {
|
||||
# CRITICAL FIX: Use only one hosts file/plugin definition
|
||||
hosts /etc/coredns/consul-all-hosts {
|
||||
fallthrough
|
||||
}
|
||||
# Fallback to upstream DNS
|
||||
forward . 192.168.4.1 8.8.8.8
|
||||
cache 30
|
||||
errors
|
||||
log
|
||||
}
|
||||
|
||||
# Handle all other DNS queries
|
||||
.:53 {
|
||||
forward . 192.168.4.1 8.8.8.8
|
||||
cache 30
|
||||
errors
|
||||
log
|
||||
}
|
||||
'';
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"f /etc/coredns/consul-all-hosts 0644 root root - #"
|
||||
];
|
||||
|
||||
# --- Consul Template Service ---
|
||||
systemd.services.consul-template = {
|
||||
description = "Consul Template for CoreDNS Hosts";
|
||||
wantedBy = ["multi-user.target"];
|
||||
after = ["consul.service" "coredns.service" "network.target"];
|
||||
requires = ["coredns.service"];
|
||||
|
||||
serviceConfig = {
|
||||
# Use the robust wrapper script
|
||||
ExecStart = "${consulTemplateWrapper}";
|
||||
|
||||
Restart = "always";
|
||||
RestartSec = "10s";
|
||||
};
|
||||
};
|
||||
|
||||
# --- CoreDNS Service ---
|
||||
systemd.services.coredns = {
|
||||
description = "CoreDNS DNS server";
|
||||
wantedBy = ["multi-user.target"];
|
||||
requires = ["consul.service"];
|
||||
after = ["network.target" "consul.service"];
|
||||
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
ExecStart = "${pkgs.coredns}/bin/coredns -conf /etc/coredns/Corefile";
|
||||
ExecReload = "${pkgs.coreutils}/bin/kill -SIGUSR1 $MAINPID";
|
||||
Restart = "on-failure";
|
||||
RestartSec = "5s";
|
||||
DynamicUser = true;
|
||||
AmbientCapabilities = "CAP_NET_BIND_SERVICE";
|
||||
CapabilityBoundingSet = "CAP_NET_BIND_SERVICE";
|
||||
NoNewPrivileges = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = true;
|
||||
ReadWritePaths = "/etc/coredns";
|
||||
};
|
||||
};
|
||||
|
||||
# --- Helper Scripts and Firewall ---
|
||||
environment.systemPackages = [
|
||||
pkgs.consul-template
|
||||
(pkgs.writeShellScriptBin "debug-consul-template" ''
|
||||
echo "Rendering template to stdout..."
|
||||
${pkgs.consul-template}/bin/consul-template \
|
||||
-template "${consulAllHostsTemplate}:-" \
|
||||
-dry | grep -v "^$"
|
||||
'') # <--- This is the crucial closing of the multi-line string
|
||||
];
|
||||
|
||||
networking.firewall = {
|
||||
allowedTCPPorts = [53];
|
||||
allowedUDPPorts = [53];
|
||||
};
|
||||
}
|
||||
@@ -1,22 +1,49 @@
|
||||
{ config, pkgs, lib, ... }:
|
||||
with lib; let
|
||||
NetworkInterface = config.custom._Networkinterface;
|
||||
VIP_Priority = config.custom.VIP_Priority;
|
||||
|
||||
# Extract the first IPv4 address from the configured network interface
|
||||
# This gets the IP from networking.interfaces.<interface>.ipv4.addresses
|
||||
getHostIp = interfaceName:
|
||||
let
|
||||
interfaceConfig = config.networking.interfaces.${interfaceName} or {};
|
||||
addresses = interfaceConfig.ipv4.addresses or [];
|
||||
in
|
||||
if (length addresses) > 0
|
||||
then (head addresses).address
|
||||
else throw "No IPv4 address configured for interface ${interfaceName}";
|
||||
|
||||
thisHostIp = getHostIp NetworkInterface;
|
||||
|
||||
# Define priorities based on IP
|
||||
priorityByIp = {
|
||||
"192.168.4.226" = 100; # Master
|
||||
"192.168.4.227" = 90; # Backup 1
|
||||
"192.168.4.228" = 80; # Backup 2
|
||||
"192.168.4.36" = 70; # Backup 3
|
||||
};
|
||||
|
||||
# All cluster nodes
|
||||
allNodes = ["192.168.4.226" "192.168.4.227" "192.168.4.228" "192.168.4.36"];
|
||||
|
||||
# Filter out this host from unicast peers
|
||||
unicastPeers = filter (ip: ip != thisHostIp) allNodes;
|
||||
|
||||
# Get priority for this host (default to 50 if not found)
|
||||
VIP_Priority = priorityByIp.${thisHostIp} or 50;
|
||||
|
||||
in {
|
||||
|
||||
|
||||
services.keepalived = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
vrrpInstances.VIP_250 = {
|
||||
interface = "${NetworkInterface}";
|
||||
virtualRouterId = 51;
|
||||
priority = VIP_Priority;
|
||||
unicastPeers = ["192.168.4.226" "192.168.4.227" "192.168.4.228" "192.168.4.36"];
|
||||
virtualIps = [{addr = "192.168.4.250/22";}];
|
||||
|
||||
};
|
||||
interface = "${NetworkInterface}";
|
||||
virtualRouterId = 51;
|
||||
priority = VIP_Priority;
|
||||
unicastPeers = unicastPeers;
|
||||
virtualIps = [{addr = "192.168.4.250/22";}];
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 80 443 ]; # optional
|
||||
}
|
||||
networking.firewall.allowedTCPPorts = [ 80 443 ];
|
||||
}
|
||||
@@ -5,7 +5,20 @@
|
||||
inputs,
|
||||
...
|
||||
}: {
|
||||
virtualisation.docker.enable = true;
|
||||
virtualisation.docker = {
|
||||
enable = true;
|
||||
# Configure insecure registries
|
||||
daemon.settings = {
|
||||
ipv6 = false; # Add this line to disable IPv6
|
||||
insecure-registries = [
|
||||
"localhost:5000"
|
||||
"gitea.service.dc1.consul:80"
|
||||
"192.168.4.109:5000"
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
|
||||
networking.firewall = {
|
||||
allowedTCPPorts = [
|
||||
|
||||
Reference in New Issue
Block a user