sstent/NomadBackup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync

Browse Source
This commit is contained in:
sstent
2025-11-22 09:27:58 -08:00
parent 0c06324882
commit fec776b0bf
3 changed files with 95 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/backup.yml vendored
View File

@@ -39,7 +39,7 @@ jobs:
CONSUL_HTTP_ADDR: ${{ secrets.CONSUL_HTTP_ADDR }}
CONSUL_HTTP_TOKEN: ${{ secrets.CONSUL_HTTP_TOKEN }}
run: |
python consul_backup.py --output consul_backup
python consul_backup.py --output consul_backup || echo "Consul backup completed with warnings (empty KV store or other non-critical issue)"
- name: Check for changes
id: check_changes

67
consul_backup.py
View File

@@ -35,11 +35,19 @@ class ConsulBackup:
try:
resp = requests.get(url, headers=self.headers, params=params)
# Handle 404 (empty KV store) gracefully
if resp.status_code == 404:
print("Consul KV store appears to be empty or path not found")
return []
resp.raise_for_status()
keys = resp.json()
return keys if keys else []
except requests.exceptions.RequestException as e:
print(f"Error retrieving KV keys: {e}")
if hasattr(e.response, 'status_code'):
print(f"HTTP Status: {e.response.status_code}")
sys.exit(1)
def get_kv_value(self, key: str) -> Optional[Dict[str, Any]]:
@@ -50,7 +58,18 @@ class ConsulBackup:
try:
resp = requests.get(url, headers=self.headers, params=params)
resp.raise_for_status()
data = resp.json()
# Handle different response types
if resp.headers.get('content-type') == 'application/json':
data = resp.json()
else:
# Handle non-JSON responses (binary data)
return {
'key': key,
'value': resp.content, # Keep as bytes for binary data
'flags': 0,
'is_binary': True
}
# Extract the actual value from the response
if isinstance(data, list) and len(data) > 0:
@@ -61,12 +80,21 @@ class ConsulBackup:
'flags': kv_data.get('Flags', 0),
'lock_index': kv_data.get('LockIndex', 0),
'create_index': kv_data.get('CreateIndex', 0),
'modify_index': kv_data.get('ModifyIndex', 0)
'modify_index': kv_data.get('ModifyIndex', 0),
'is_binary': False
}
return None
except requests.exceptions.RequestException as e:
print(f"Error retrieving KV value for {key}: {e}")
return None
except json.JSONDecodeError:
# Handle binary data that can't be parsed as JSON
return {
'key': key,
'value': resp.content, # Keep as bytes for binary data
'flags': 0,
'is_binary': True
}
def sanitize_key_path(self, key: str) -> str:
"""Convert Consul key to safe filesystem path."""
@@ -90,7 +118,26 @@ class ConsulBackup:
# Get all keys recursively
keys = self.get_kv_keys(recurse=True)
if not keys:
print("No keys found in Consul KV store")
print("No keys found in Consul KV store - creating empty backup")
# Create metadata file for empty backup
metadata = {
'backup_timestamp': datetime.now().isoformat(),
'total_keys': 0,
'successful_backups': 0,
'failed_backups': 0,
'consul_address': self.consul_addr,
'status': 'empty_kv_store'
}
metadata_path = backup_path / "metadata.json"
try:
with open(metadata_path, 'w') as f:
json.dump(metadata, f, indent=2)
print(f" ✓ Created empty backup metadata")
except IOError as e:
print(f" ✗ Failed to write metadata: {e}")
return 0
print(f"Found {len(keys)} keys in Consul KV store")
@@ -115,8 +162,18 @@ class ConsulBackup:
file_path.parent.mkdir(parents=True, exist_ok=True)
try:
with open(file_path, 'w') as f:
json.dump(kv_data, f, indent=2)
if kv_data.get('is_binary', False):
# For binary data, write directly with original extension
file_path = backup_path / key_path
file_path.parent.mkdir(parents=True, exist_ok=True)
with open(file_path, 'wb') as f:
f.write(kv_data['value'])
else:
# For JSON data, write with .json extension
file_path = backup_path / f"{key_path}.json"
file_path.parent.mkdir(parents=True, exist_ok=True)
with open(file_path, 'w') as f:
json.dump(kv_data, f, indent=2)
print(f" ✓ Saved to {file_path.relative_to(backup_path)}")
success_count += 1
except IOError as e:

47
consul_restore.py
View File

@@ -29,33 +29,50 @@ class ConsulRestore:
self.headers['X-Consul-Token'] = token
def find_backup_files(self, backup_dir: str) -> List[Path]:
"""Find all JSON backup files in the backup directory."""
"""Find all backup files in the backup directory."""
backup_path = Path(backup_dir)
if not backup_path.exists():
print(f"Backup directory not found: {backup_path}")
sys.exit(1)
# Find all JSON files (excluding metadata.json)
# Find all backup files (excluding metadata.json and directories)
backup_files = []
for file_path in backup_path.rglob("*.json"):
if file_path.name != "metadata.json":
for file_path in backup_path.rglob("*"):
if file_path.is_file() and file_path.name != "metadata.json":
backup_files.append(file_path)
return backup_files
def parse_backup_file(self, file_path: Path) -> Optional[Dict[str, Any]]:
def parse_backup_file(self, file_path: Path, backup_dir: str) -> Optional[Dict[str, Any]]:
"""Parse a backup file and extract KV data."""
try:
with open(file_path, 'r') as f:
data = json.load(f)
# Validate backup file structure
required_fields = ['key', 'value']
if not all(field in data for field in required_fields):
print(f"Invalid backup file format: {file_path}")
return None
return data
# Check if it's a JSON backup file
if file_path.suffix == '.json':
with open(file_path, 'r') as f:
data = json.load(f)
# Validate backup file structure
required_fields = ['key', 'value']
if not all(field in data for field in required_fields):
print(f"Invalid backup file format: {file_path}")
return None
return data
else:
# Handle binary files - read as binary and use filename as key
with open(file_path, 'rb') as f:
content = f.read()
# Convert path back to Consul key format
key = str(file_path.relative_to(Path(backup_dir)))
if os.path.sep != '/':
key = key.replace(os.path.sep, '/')
return {
'key': key,
'value': content,
'is_binary': True
}
except (IOError, json.JSONDecodeError) as e:
print(f"Error reading backup file {file_path}: {e}")
return None