From 6972c9b8f9e7268a427cf1c115a30588a95085b8 Mon Sep 17 00:00:00 2001 From: sstent Date: Mon, 12 Jan 2026 16:37:03 -0800 Subject: [PATCH] Migrate to PostgreSQL and add Config Status page --- MIGRATION.md | 63 +++++++++++++++ app/api/routes/admin.py | 41 +++++++++- docker-compose.yml | 6 +- migrate_to_postgres.py | 150 ++++++++++++++++++++++++++++++++++++ planner.nomad | 98 +++++++++++++++++++++++ requirements.txt | 2 +- templates/admin/config.html | 73 ++++++++++++++++++ templates/admin/index.html | 3 + 8 files changed, 432 insertions(+), 4 deletions(-) create mode 100644 MIGRATION.md create mode 100644 migrate_to_postgres.py create mode 100644 planner.nomad create mode 100644 templates/admin/config.html diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 0000000..9eb0d5b --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,63 @@ +# Database Migration Guide + +This guide outlines the offline workflow to migrate your `meal_planner` data from SQLite to PostgreSQL. + +## Prerequisites +- Docker Compose installed. +- The application running (or capable of running) via `docker-compose`. + +## Migration Steps + +### 1. Backup your SQLite Database +First, create a safety copy of your current database. +```bash +cp data/meal_planner.db meal_planner_backup.db +``` + +### 2. Stop the Application +Stop the running application container to ensure no new data is written. +```bash +docker-compose stop foodtracker +``` + +### 3. Start PostgreSQL +Ensure the new PostgreSQL service is running. +```bash +docker-compose up -d postgres +``` + +### 4. Run the Migration +Use a temporary container to run the migration script. We mount your backup file and connect to the postgres service. +```bash +# Syntax: python migrate_to_postgres.py --sqlite-path --pg-url + +docker-compose run --rm \ + -v $(pwd)/meal_planner_backup.db:/backup.db \ + -v $(pwd)/migrate_to_postgres.py:/app/migrate_to_postgres.py \ + foodtracker \ + python migrate_to_postgres.py \ + --sqlite-path /backup.db \ + --pg-url postgresql://user:password@postgres/meal_planner +``` + +### 5. Update Configuration +Edit `docker-compose.yml` to switch the active database. +1. Comment out the SQLite `DATABASE_URL`. +2. Uncomment the PostgreSQL `DATABASE_URL`. + +```yaml + environment: + # - DATABASE_URL=sqlite:////app/data/meal_planner.db + - DATABASE_URL=postgresql://user:password@postgres/meal_planner +``` + +### 6. Restart the Application +Rebuild and start the application to use the new database. +```bash +docker-compose up -d --build foodtracker +``` + +## Verification +1. Log in to the application. +2. Verify your Foods, Meals, and Plans are present. +3. Check `docker logs foodplanner-foodtracker-1` to ensure no database connection errors. diff --git a/app/api/routes/admin.py b/app/api/routes/admin.py index 1b59776..e26a079 100644 --- a/app/api/routes/admin.py +++ b/app/api/routes/admin.py @@ -187,6 +187,45 @@ async def restore_backup(request: Request, backup_file: str = Form(...)): # You might want to add some user-facing error feedback here pass + # Redirect back to the backups page # Redirect back to the backups page from fastapi.responses import RedirectResponse - return RedirectResponse(url="/admin/backups", status_code=303) \ No newline at end of file + return RedirectResponse(url="/admin/backups", status_code=303) + +@router.get("/admin/config-status", response_class=HTMLResponse) +async def admin_config_status_page(request: Request): + """Display current system configuration and database status.""" + from urllib.parse import urlparse + + # Analyze DATABASE_URL securely + db_url = DATABASE_URL + masked_url = db_url + db_host = "Unknown" + db_type = "Unknown" + + try: + # Simple parsing logic to avoid exposing credentials if urlparse fails or acts unexpectedly + if "sqlite" in db_url: + db_type = "SQLite" + db_host = db_url.replace("sqlite:///", "") + masked_url = "sqlite:///" + db_host + elif "postgresql" in db_url: + db_type = "PostgreSQL" + parsed = urlparse(db_url) + db_host = parsed.hostname + # Mask password + if parsed.password: + masked_url = db_url.replace(parsed.password, "******") + except Exception as e: + logging.error(f"Error parsing database URL: {e}") + masked_url = "Error parsing URL" + + config_data = { + "database_url": db_url, + "database_url_masked": masked_url, + "database_type": db_type, + "database_host": db_host, + "debug": True + } + + return templates.TemplateResponse(request, "admin/config.html", {"request": request, "config": config_data}) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 6ec436a..0bd2216 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,11 +4,13 @@ services: ports: - "8999:8999" environment: - - DATABASE_URL=sqlite:////app/data/meal_planner.db + #- DATABASE_URL=sqlite:////app/data/meal_planner.db + - DATABASE_URL=postgresql://postgres:postgres@master.postgres.service.dc1.consul/meal_planner - PYTHONUNBUFFERED=1 volumes: - ./alembic:/app/alembic - ./data:/app/data + - ./backups:/app/backups - ./app:/app/app - ./templates:/app/templates - - ./main.py:/app/main.py \ No newline at end of file + - ./main.py:/app/main.py diff --git a/migrate_to_postgres.py b/migrate_to_postgres.py new file mode 100644 index 0000000..4b1eb39 --- /dev/null +++ b/migrate_to_postgres.py @@ -0,0 +1,150 @@ +import os +import sys +import logging +from sqlalchemy import create_engine, text, inspect +from sqlalchemy.orm import sessionmaker + +# Import models to ensure simple table discovery if needed, +# though we will mostly work with raw tables or inspection. +from app.database import Base, Food, Meal, MealFood, Plan, Template, TemplateMeal, WeeklyMenu, WeeklyMenuDay, TrackedDay, TrackedMeal, TrackedMealFood, FitbitConfig, WeightLog + +# Configure logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +def migrate(): + import argparse + + parser = argparse.ArgumentParser(description='Migrate data from SQLite to PostgreSQL') + parser.add_argument('--sqlite-path', help='Path to source SQLite database file', default=os.getenv('SQLITE_PATH', '/app/data/meal_planner.db')) + parser.add_argument('--pg-url', help='PostgreSQL connection URL', default=os.getenv('PG_DATABASE_URL')) + + args = parser.parse_args() + + # Configuration + # Source: SQLite + sqlite_path = args.sqlite_path + sqlite_url = f"sqlite:///{sqlite_path}" + + # Destination: Postgres + if args.pg_url: + pg_url = args.pg_url + else: + # update this if running externally + pg_user = os.getenv('POSTGRES_USER', 'user') + pg_password = os.getenv('POSTGRES_PASSWORD', 'password') + pg_host = os.getenv('POSTGRES_HOST', 'postgres') + pg_db = os.getenv('POSTGRES_DB', 'meal_planner') + pg_url = f"postgresql://{pg_user}:{pg_password}@{pg_host}/{pg_db}" + + logger.info(f"Source SQLite: {sqlite_url}") + logger.info(f"Destination Postgres: {pg_url}") + + # Create Engines + try: + sqlite_engine = create_engine(sqlite_url) + pg_engine = create_engine(pg_url) + + # Test connections + with sqlite_engine.connect() as conn: + pass + logger.info("Connected to SQLite.") + + with pg_engine.connect() as conn: + pass + logger.info("Connected to Postgres.") + + except Exception as e: + logger.error(f"Failed to connect to databases: {e}") + return + + # Create tables in Postgres if they don't exist + # Using the Base metadata from the app + logger.info("Creating tables in Postgres...") + Base.metadata.drop_all(pg_engine) # Clean start to avoid conflicts + Base.metadata.create_all(pg_engine) + logger.info("Tables created.") + + # Define table order to respect Foreign Keys + tables_ordered = [ + 'foods', + 'meals', + 'meal_foods', + 'templates', + 'template_meals', + 'weekly_menus', + 'weekly_menu_days', + 'plans', + 'tracked_days', + 'tracked_meals', + 'tracked_meal_foods', + 'fitbit_config', + 'weight_logs' + ] + + # Migration Loop + with sqlite_engine.connect() as sqlite_conn, pg_engine.connect() as pg_conn: + for table_name in tables_ordered: + logger.info(f"Migrating table: {table_name}") + + # Read from SQLite + try: + # Use raw SQL to get all data, handling potential missing tables gracefully if app changed + result = sqlite_conn.execute(text(f"SELECT * FROM {table_name}")) + rows = result.fetchall() + keys = result.keys() + + if not rows: + logger.info(f" No data in {table_name}, skipping.") + continue + + # Insert into Postgres + # We simply create a list of dicts + data = [dict(zip(keys, row)) for row in rows] + + # Setup insert statement + # We use SQLAlchemy core to make it db-agnostic enough + table_obj = Base.metadata.tables[table_name] + + pg_conn.execute(table_obj.insert(), data) + pg_conn.commit() + + logger.info(f" Migrated {len(rows)} rows.") + + # Reset Sequence for Serial ID columns + # Postgres sequences usually named table_id_seq + if 'id' in keys: + # Find max id + max_id = max(row[0] for row in rows) # Assuming 'id' is first or we can look it up. + # Safer: + max_id_val = 0 + for d in data: + if d['id'] > max_id_val: + max_id_val = d['id'] + + if max_id_val > 0: + seq_name = f"{table_name}_id_seq" + # Check if sequence exists (it should for Serial) + try: + pg_conn.execute(text(f"SELECT setval('{seq_name}', {max_id_val})")) + pg_conn.commit() + logger.info(f" Sequence {seq_name} reset to {max_id_val}") + except Exception as seq_err: + logger.warn(f" Could not reset sequence {seq_name} (might not exist): {seq_err}") + pg_conn.rollback() + + except Exception as e: + # Check for "no such table" specific error which is common if a feature isn't used + if "no such table" in str(e): + logger.warning(f" Table {table_name} not found in source SQLite. Skipping.") + continue + + logger.error(f"Error migrating {table_name}: {e}") + pg_conn.rollback() + # Decide whether to stop or continue. Stopping is safer. + return + + logger.info("Migration completed successfully.") + +if __name__ == "__main__": + migrate() diff --git a/planner.nomad b/planner.nomad new file mode 100644 index 0000000..f3a8916 --- /dev/null +++ b/planner.nomad @@ -0,0 +1,98 @@ +job "foodplanner" { + datacenters = ["dc1"] + + type = "service" + + group "app" { + count = 1 + + network { + port "http" { + to = 8999 + } + } + + service { + name = "foodplanner" + port = "http" + + check { + type = "http" + path = "/" + interval = "10s" + timeout = "2s" + } + } + + # Prestart restore task + task "restore" { + driver = "docker" + lifecycle { + hook = "prestart" + sidecar = false + } + config { + # image = "litestream/litestream:latest" + image = "litestream/litestream:0.3" + args = [ + "restore", + # "-if-replica-exists", + #"-if-db-not-exists", + "-o", "/alloc/tmp/meal_planner.db", + "sftp://root:odroid@192.168.4.63/mnt/Shares/litestream/foodplanner.db" + ] + volumes = [ + "/opt/nomad/data:/data" + ] + } + } + + task "app" { + driver = "docker" + + config { + image = "ghcr.io/sstent/foodplanner:main" + ports = ["http"] + + # Mount the SQLite database file to persist data + # Adjust the source path as needed for your environment + volumes = [ + "/mnt/Public/configs/FoodPlanner_backups:/app/backups/", + ] + } + env { + DATABASE_PATH = "/alloc/tmp" + DATABASE_URL = "sqlite:////alloc/tmp/meal_planner.db" + } + resources { + cpu = 500 + memory = 1024 + } + + # Restart policy + restart { + attempts = 3 + interval = "10m" + delay = "15s" + mode = "fail" + } + } + + # Litestream sidecar for continuous replication + task "litestream" { + driver = "docker" + lifecycle { + hook = "poststart" # runs after main task starts + sidecar = true + } + config { + # image = "litestream/litestream:0.5.0-test.10" + image = "litestream/litestream:0.3" + args = [ + "replicate", + "/alloc/tmp/meal_planner.db", + "sftp://root:odroid@192.168.4.63/mnt/Shares/litestream/foodplanner.db" + ] + } + } + } diff --git a/requirements.txt b/requirements.txt index 3b9b40b..c9be189 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ starlette==0.37.2 anyio==4.4.0 uvicorn[standard]==0.24.0 sqlalchemy>=2.0.24 -#psycopg2-binary==2.9.9 +psycopg2-binary==2.9.9 python-multipart>=0.0.7 jinja2==3.1.2 openfoodfacts>=0.2.0 diff --git a/templates/admin/config.html b/templates/admin/config.html new file mode 100644 index 0000000..6794cd2 --- /dev/null +++ b/templates/admin/config.html @@ -0,0 +1,73 @@ +{% extends "admin/index.html" %} + +{% block admin_content %} +
+
+
System Configuration Status
+
+
+

Current system environment and database connection details.

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
SettingValue
Database Type + {% if 'sqlite' in config.database_url %} + SQLite + {% elif 'postgresql' in config.database_url %} + PostgreSQL + {% else %} + {{ config.database_type }} + {% endif %} +
Connection URL{{ config.database_url_masked }}
Database Host/Path{{ config.database_host }}
Environment + {% if config.debug %} + Debug Mode + {% else %} + Production + {% endif %} +
+ +
+ + {% if 'sqlite' in config.database_url %} + Running in portable SQLite mode. To switch to PostgreSQL, please refer to the migration guide. + {% else %} + Running in PostgreSQL mode. Database is hosted at {{ config.database_host }}. + {% endif %} +
+
+
+ + +{% endblock %} \ No newline at end of file diff --git a/templates/admin/index.html b/templates/admin/index.html index cf6971e..082d0d3 100644 --- a/templates/admin/index.html +++ b/templates/admin/index.html @@ -16,6 +16,9 @@ +