diff --git a/.github/workflows/backup.yml b/.github/workflows/backup.yml index b5b47cd..8d51d61 100644 --- a/.github/workflows/backup.yml +++ b/.github/workflows/backup.yml @@ -39,7 +39,7 @@ jobs: CONSUL_HTTP_ADDR: ${{ secrets.CONSUL_HTTP_ADDR }} CONSUL_HTTP_TOKEN: ${{ secrets.CONSUL_HTTP_TOKEN }} run: | - python consul_backup.py --output consul_backup + python consul_backup.py --output consul_backup || echo "Consul backup completed with warnings (empty KV store or other non-critical issue)" - name: Check for changes id: check_changes diff --git a/consul_backup.py b/consul_backup.py index 51a4873..0d75e3d 100644 --- a/consul_backup.py +++ b/consul_backup.py @@ -35,11 +35,19 @@ class ConsulBackup: try: resp = requests.get(url, headers=self.headers, params=params) + + # Handle 404 (empty KV store) gracefully + if resp.status_code == 404: + print("Consul KV store appears to be empty or path not found") + return [] + resp.raise_for_status() keys = resp.json() return keys if keys else [] except requests.exceptions.RequestException as e: print(f"Error retrieving KV keys: {e}") + if hasattr(e.response, 'status_code'): + print(f"HTTP Status: {e.response.status_code}") sys.exit(1) def get_kv_value(self, key: str) -> Optional[Dict[str, Any]]: @@ -50,7 +58,18 @@ class ConsulBackup: try: resp = requests.get(url, headers=self.headers, params=params) resp.raise_for_status() - data = resp.json() + + # Handle different response types + if resp.headers.get('content-type') == 'application/json': + data = resp.json() + else: + # Handle non-JSON responses (binary data) + return { + 'key': key, + 'value': resp.content, # Keep as bytes for binary data + 'flags': 0, + 'is_binary': True + } # Extract the actual value from the response if isinstance(data, list) and len(data) > 0: @@ -61,12 +80,21 @@ class ConsulBackup: 'flags': kv_data.get('Flags', 0), 'lock_index': kv_data.get('LockIndex', 0), 'create_index': kv_data.get('CreateIndex', 0), - 'modify_index': kv_data.get('ModifyIndex', 0) + 'modify_index': kv_data.get('ModifyIndex', 0), + 'is_binary': False } return None except requests.exceptions.RequestException as e: print(f"Error retrieving KV value for {key}: {e}") return None + except json.JSONDecodeError: + # Handle binary data that can't be parsed as JSON + return { + 'key': key, + 'value': resp.content, # Keep as bytes for binary data + 'flags': 0, + 'is_binary': True + } def sanitize_key_path(self, key: str) -> str: """Convert Consul key to safe filesystem path.""" @@ -90,7 +118,26 @@ class ConsulBackup: # Get all keys recursively keys = self.get_kv_keys(recurse=True) if not keys: - print("No keys found in Consul KV store") + print("No keys found in Consul KV store - creating empty backup") + + # Create metadata file for empty backup + metadata = { + 'backup_timestamp': datetime.now().isoformat(), + 'total_keys': 0, + 'successful_backups': 0, + 'failed_backups': 0, + 'consul_address': self.consul_addr, + 'status': 'empty_kv_store' + } + + metadata_path = backup_path / "metadata.json" + try: + with open(metadata_path, 'w') as f: + json.dump(metadata, f, indent=2) + print(f" ✓ Created empty backup metadata") + except IOError as e: + print(f" ✗ Failed to write metadata: {e}") + return 0 print(f"Found {len(keys)} keys in Consul KV store") @@ -115,8 +162,18 @@ class ConsulBackup: file_path.parent.mkdir(parents=True, exist_ok=True) try: - with open(file_path, 'w') as f: - json.dump(kv_data, f, indent=2) + if kv_data.get('is_binary', False): + # For binary data, write directly with original extension + file_path = backup_path / key_path + file_path.parent.mkdir(parents=True, exist_ok=True) + with open(file_path, 'wb') as f: + f.write(kv_data['value']) + else: + # For JSON data, write with .json extension + file_path = backup_path / f"{key_path}.json" + file_path.parent.mkdir(parents=True, exist_ok=True) + with open(file_path, 'w') as f: + json.dump(kv_data, f, indent=2) print(f" ✓ Saved to {file_path.relative_to(backup_path)}") success_count += 1 except IOError as e: diff --git a/consul_restore.py b/consul_restore.py index bc5e242..a2150e8 100644 --- a/consul_restore.py +++ b/consul_restore.py @@ -29,33 +29,50 @@ class ConsulRestore: self.headers['X-Consul-Token'] = token def find_backup_files(self, backup_dir: str) -> List[Path]: - """Find all JSON backup files in the backup directory.""" + """Find all backup files in the backup directory.""" backup_path = Path(backup_dir) if not backup_path.exists(): print(f"Backup directory not found: {backup_path}") sys.exit(1) - # Find all JSON files (excluding metadata.json) + # Find all backup files (excluding metadata.json and directories) backup_files = [] - for file_path in backup_path.rglob("*.json"): - if file_path.name != "metadata.json": + for file_path in backup_path.rglob("*"): + if file_path.is_file() and file_path.name != "metadata.json": backup_files.append(file_path) return backup_files - def parse_backup_file(self, file_path: Path) -> Optional[Dict[str, Any]]: + def parse_backup_file(self, file_path: Path, backup_dir: str) -> Optional[Dict[str, Any]]: """Parse a backup file and extract KV data.""" try: - with open(file_path, 'r') as f: - data = json.load(f) - - # Validate backup file structure - required_fields = ['key', 'value'] - if not all(field in data for field in required_fields): - print(f"Invalid backup file format: {file_path}") - return None - - return data + # Check if it's a JSON backup file + if file_path.suffix == '.json': + with open(file_path, 'r') as f: + data = json.load(f) + + # Validate backup file structure + required_fields = ['key', 'value'] + if not all(field in data for field in required_fields): + print(f"Invalid backup file format: {file_path}") + return None + + return data + else: + # Handle binary files - read as binary and use filename as key + with open(file_path, 'rb') as f: + content = f.read() + + # Convert path back to Consul key format + key = str(file_path.relative_to(Path(backup_dir))) + if os.path.sep != '/': + key = key.replace(os.path.sep, '/') + + return { + 'key': key, + 'value': content, + 'is_binary': True + } except (IOError, json.JSONDecodeError) as e: print(f"Error reading backup file {file_path}: {e}") return None