From 6c1fe70fa2d7788c248b153762467a69fbfa5afb Mon Sep 17 00:00:00 2001 From: sstent Date: Fri, 22 Aug 2025 20:29:04 -0700 Subject: [PATCH] checkpoint 2 --- Dockerfile | 1 + cyclingpower.md | 221 ++++++++++++++++++ entrypoint.sh | 33 ++- garminsync/activity_parser.py | 130 +++++++++++ garminsync/cli.py | 9 +- garminsync/daemon.py | 43 +++- garminsync/database.py | 72 ++++-- garminsync/migrate_activities.py | 59 +++-- justfile | 9 +- requirements.txt | 1 + .../test_sync.cpython-310-pytest-8.1.1.pyc | Bin 3491 -> 4769 bytes tests/test_sync.py | 122 +++++----- workflows.md | 100 ++++++++ 13 files changed, 678 insertions(+), 122 deletions(-) create mode 100644 cyclingpower.md create mode 100644 garminsync/activity_parser.py create mode 100644 workflows.md diff --git a/Dockerfile b/Dockerfile index 06b8dba..1fce15b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,7 @@ RUN pip install --upgrade pip && \ # Copy application code COPY garminsync/ ./garminsync/ COPY migrations/ ./migrations/ +COPY migrations/alembic.ini ./alembic.ini COPY tests/ ./tests/ COPY entrypoint.sh . COPY patches/ ./patches/ diff --git a/cyclingpower.md b/cyclingpower.md new file mode 100644 index 0000000..dc5806e --- /dev/null +++ b/cyclingpower.md @@ -0,0 +1,221 @@ +# Cycling FIT Analysis Implementation Plan + +## Overview +Extend the existing GarminSync FIT parser to calculate cycling-specific metrics including power estimation and singlespeed gear ratio analysis for activities without native power data. + +## Phase 1: Core Infrastructure Setup + +### 1.1 Database Schema Extensions +**File: `garminsync/database.py`** +- Extend existing `PowerAnalysis` table with cycling-specific fields: + ```python + # Add to PowerAnalysis class: + peak_power_1s = Column(Float, nullable=True) + peak_power_5s = Column(Float, nullable=True) + peak_power_20s = Column(Float, nullable=True) + peak_power_300s = Column(Float, nullable=True) + normalized_power = Column(Float, nullable=True) + intensity_factor = Column(Float, nullable=True) + training_stress_score = Column(Float, nullable=True) + ``` + +- Extend existing `GearingAnalysis` table: + ```python + # Add to GearingAnalysis class: + estimated_chainring_teeth = Column(Integer, nullable=True) + estimated_cassette_teeth = Column(Integer, nullable=True) + gear_ratio = Column(Float, nullable=True) + gear_inches = Column(Float, nullable=True) + development_meters = Column(Float, nullable=True) + confidence_score = Column(Float, nullable=True) + analysis_method = Column(String, default="singlespeed_estimation") + ``` + +### 1.2 Enhanced FIT Parser +**File: `garminsync/fit_processor/parser.py`** +- Extend `FITParser` to extract cycling-specific data points: + ```python + def _extract_cycling_data(self, message): + """Extract cycling-specific metrics from FIT records""" + # GPS coordinates for elevation/gradient + # Speed and cadence for gear analysis + # Power data (if available) for validation + # Temperature for air density calculations + ``` + +## Phase 2: Power Estimation Engine + +### 2.1 Physics-Based Power Calculator +**New file: `garminsync/fit_processor/power_estimator.py`** + +**Key Components:** +- **Environmental factors**: Air density, wind resistance, temperature +- **Bike specifications**: Weight (22 lbs = 10 kg), aerodynamic drag coefficient +- **Rider assumptions**: Weight (75 kg default), position (road bike) +- **Terrain analysis**: Gradient calculation from GPS elevation data + +**Core Algorithm:** +```python +class PowerEstimator: + def __init__(self): + self.bike_weight_kg = 10.0 # 22 lbs + self.rider_weight_kg = 75.0 # Default assumption + self.drag_coefficient = 0.88 # Road bike + self.frontal_area_m2 = 0.4 # Typical road cycling position + self.rolling_resistance = 0.004 # Road tires + self.drivetrain_efficiency = 0.97 + self.air_density = 1.225 # kg/m³ at sea level, 20°C + + def calculate_power(self, speed_ms, gradient_percent, + air_temp_c=20, altitude_m=0): + """Calculate estimated power using physics model""" + # Power = (Rolling + Gravity + Aerodynamic + Kinetic) / Efficiency +``` + +**Power Components:** +1. **Rolling resistance**: `P_roll = Crr × (m_bike + m_rider) × g × cos(θ) × v` +2. **Gravitational**: `P_grav = (m_bike + m_rider) × g × sin(θ) × v` +3. **Aerodynamic**: `P_aero = 0.5 × ρ × Cd × A × v³` +4. **Acceleration**: `P_accel = (m_bike + m_rider) × a × v` + +### 2.2 Peak Power Analysis +**Methods:** +- 1-second, 5-second, 20-second, 5-minute peak power windows +- Normalized Power (NP) calculation using 30-second rolling average +- Training Stress Score (TSS) estimation based on NP and ride duration + +## Phase 3: Singlespeed Gear Ratio Analysis + +### 3.1 Gear Ratio Calculator +**New file: `garminsync/fit_processor/gear_analyzer.py`** + +**Strategy:** +- Analyze flat terrain segments (gradient < 3%) +- Use speed/cadence relationship to determine gear ratio +- Test against common singlespeed ratios for 38t and 46t chainrings +- Calculate confidence scores based on data consistency + +**Core Algorithm:** +```python +class SinglespeedAnalyzer: + def __init__(self): + self.chainring_options = [38, 46] # teeth + self.common_cogs = list(range(11, 28)) # 11t to 27t rear cogs + self.wheel_circumference_m = 2.096 # 700x25c tire + + def analyze_gear_ratio(self, speed_data, cadence_data, gradient_data): + """Determine most likely singlespeed gear ratio""" + # Filter for flat terrain segments + # Calculate gear ratio from speed/cadence + # Match against common ratios + # Return best fit with confidence score +``` + +**Gear Metrics:** +- **Gear ratio**: Chainring teeth ÷ Cog teeth +- **Gear inches**: Gear ratio × wheel diameter (inches) +- **Development**: Distance traveled per pedal revolution (meters) + +### 3.2 Analysis Methodology +1. **Segment filtering**: Identify flat terrain (gradient < 3%, speed > 15 km/h) +2. **Ratio calculation**: `gear_ratio = (speed_ms × 60) ÷ (cadence_rpm × wheel_circumference_m)` +3. **Ratio matching**: Compare calculated ratios against theoretical singlespeed options +4. **Confidence scoring**: Based on data consistency and segment duration + +## Phase 4: Integration with Existing System + +### 4.1 FIT Processing Workflow Enhancement +**File: `garminsync/fit_processor/analyzer.py`** +- Integrate power estimation and gear analysis into existing analysis workflow +- Add cycling-specific analysis triggers (detect cycling activities) +- Store results in database using existing schema + +### 4.2 Database Population +**Migration strategy:** +- Extend existing migration system to handle new fields +- Process existing FIT files retroactively +- Add processing status tracking for cycling analysis + +### 4.3 CLI Integration +**File: `garminsync/cli.py`** +- Add new command: `garminsync analyze --cycling --activity-id ` +- Add batch processing: `garminsync analyze --cycling --missing` +- Add reporting: `garminsync report --power-analysis --gear-analysis` + +## Phase 5: Validation and Testing + +### 5.1 Test Data Requirements +- FIT files with known power data for validation +- Various singlespeed configurations for gear ratio testing +- Different terrain types (flat, climbing, mixed) + +### 5.2 Validation Methodology +- Compare estimated vs. actual power (where available) +- Validate gear ratio estimates against known bike configurations +- Test edge cases (very low/high cadence, extreme gradients) + +### 5.3 Performance Optimization +- Efficient gradient calculation from GPS data +- Optimize power calculation loops for large datasets +- Cache intermediate calculations + +## Phase 6: Advanced Features (Future) + +### 6.1 Environmental Corrections +- Wind speed/direction integration +- Barometric pressure for accurate altitude +- Temperature-based air density adjustments + +### 6.2 Machine Learning Enhancement +- Train models on validated power data +- Improve gear ratio detection accuracy +- Personalized power estimation based on rider history + +### 6.3 Comparative Analysis +- Compare estimated metrics across rides +- Trend analysis for fitness progression +- Gear ratio optimization recommendations + +## Implementation Priority + +**High Priority:** +1. Database schema extensions +2. Basic power estimation using physics model +3. Singlespeed gear ratio analysis for flat terrain +4. Integration with existing FIT processing pipeline + +**Medium Priority:** +1. Peak power analysis (1s, 5s, 20s, 5min) +2. Normalized Power and TSS calculations +3. Advanced gear analysis with confidence scoring +4. CLI commands for analysis and reporting + +**Low Priority:** +1. Environmental corrections (wind, pressure) +2. Machine learning enhancements +3. Advanced comparative analysis features +4. Web UI integration for visualizing results + +## Success Criteria + +1. **Power Estimation**: Within ±10% of actual power data (where available for validation) +2. **Gear Ratio Detection**: Correctly identify gear ratios within ±1 tooth accuracy +3. **Processing Speed**: Analyze typical FIT file (1-hour ride) in <5 seconds +4. **Data Coverage**: Successfully analyze 90%+ of cycling FIT files +5. **Integration**: Seamlessly integrate with existing GarminSync workflow + +## File Structure Summary + +``` +garminsync/ +├── fit_processor/ +│ ├── parser.py (enhanced) +│ ├── analyzer.py (enhanced) +│ ├── power_estimator.py (new) +│ └── gear_analyzer.py (new) +├── database.py (enhanced) +├── cli.py (enhanced) +└── migrate_cycling_analysis.py (new) +``` + +This plan provides a comprehensive roadmap for implementing cycling-specific FIT analysis while building on the existing GarminSync infrastructure and maintaining compatibility with current functionality. \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index a61423c..01c26d4 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,16 +1,31 @@ #!/bin/bash -# Run database migrations -echo "Running database migrations..." -export ALEMBIC_CONFIG=${ALEMBIC_CONFIG:-./migrations/alembic.ini} -export ALEMBIC_SCRIPT_LOCATION=${ALEMBIC_SCRIPT_LOCATION:-./migrations/versions} -alembic upgrade head -if [ $? -ne 0 ]; then - echo "Migration failed!" >&2 - exit 1 +# Conditionally run database migrations +if [ "${RUN_MIGRATIONS:-1}" = "1" ]; then + echo "$(date) - Starting database migrations..." + echo "ALEMBIC_CONFIG: ${ALEMBIC_CONFIG:-/app/migrations/alembic.ini}" + echo "ALEMBIC_SCRIPT_LOCATION: ${ALEMBIC_SCRIPT_LOCATION:-/app/migrations/versions}" + + # Run migrations with timing + start_time=$(date +%s) + export ALEMBIC_CONFIG=${ALEMBIC_CONFIG:-/app/migrations/alembic.ini} + export ALEMBIC_SCRIPT_LOCATION=${ALEMBIC_SCRIPT_LOCATION:-/app/migrations/versions} + alembic upgrade head + migration_status=$? + end_time=$(date +%s) + duration=$((end_time - start_time)) + + if [ $migration_status -ne 0 ]; then + echo "$(date) - Migration failed after ${duration} seconds!" >&2 + exit 1 + else + echo "$(date) - Migrations completed successfully in ${duration} seconds" + fi +else + echo "$(date) - Skipping database migrations (RUN_MIGRATIONS=${RUN_MIGRATIONS})" fi # Start the application -echo "Starting application..." +echo "$(date) - Starting application..." exec python -m garminsync.cli daemon --start --port 8888 sleep infinity diff --git a/garminsync/activity_parser.py b/garminsync/activity_parser.py new file mode 100644 index 0000000..5469e89 --- /dev/null +++ b/garminsync/activity_parser.py @@ -0,0 +1,130 @@ +import os +import gzip +import fitdecode +import xml.etree.ElementTree as ET +from datetime import datetime + +def detect_file_type(file_path): + """Detect file format (FIT, XML, or unknown)""" + try: + with open(file_path, 'rb') as f: + header = f.read(128) + if b'= 8 and header[4:8] == b'.FIT': + return 'fit' + if (len(header) >= 8 and + (header[0:4] == b'.FIT' or + header[4:8] == b'FIT.' or + header[8:12] == b'.FIT')): + return 'fit' + return 'unknown' + except Exception as e: + return 'error' + +def parse_xml_file(file_path): + """Parse XML (TCX) file to extract activity metrics""" + try: + tree = ET.parse(file_path) + root = tree.getroot() + namespaces = {'ns': 'http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2'} + + sport = root.find('.//ns:Activity', namespaces).get('Sport', 'other') + distance = root.find('.//ns:DistanceMeters', namespaces) + distance = float(distance.text) if distance is not None else None + duration = root.find('.//ns:TotalTimeSeconds', namespaces) + duration = float(duration.text) if duration is not None else None + calories = root.find('.//ns:Calories', namespaces) + calories = int(calories.text) if calories is not None else None + + hr_values = [] + for hr in root.findall('.//ns:HeartRateBpm/ns:Value', namespaces): + try: + hr_values.append(int(hr.text)) + except: + continue + max_hr = max(hr_values) if hr_values else None + + return { + "activityType": {"typeKey": sport}, + "summaryDTO": { + "duration": duration, + "distance": distance, + "maxHR": max_hr, + "avgPower": None, + "calories": calories + } + } + except Exception: + return None + +def parse_fit_file(file_path): + """Parse FIT file to extract activity metrics""" + metrics = {} + try: + with open(file_path, 'rb') as f: + magic = f.read(2) + f.seek(0) + is_gzipped = magic == b'\x1f\x8b' + + if is_gzipped: + with gzip.open(file_path, 'rb') as gz_file: + from io import BytesIO + with BytesIO(gz_file.read()) as fit_data: + fit = fitdecode.FitReader(fit_data) + for frame in fit: + if frame.frame_type == fitdecode.FrameType.DATA and frame.name == 'session': + metrics = { + "sport": frame.get_value("sport"), + "total_timer_time": frame.get_value("total_timer_time"), + "total_distance": frame.get_value("total_distance"), + "max_heart_rate": frame.get_value("max_heart_rate"), + "avg_power": frame.get_value("avg_power"), + "total_calories": frame.get_value("total_calories") + } + break + else: + with fitdecode.FitReader(file_path) as fit: + for frame in fit: + if frame.frame_type == fitdecode.FrameType.DATA and frame.name == 'session': + metrics = { + "sport": frame.get_value("sport"), + "total_timer_time": frame.get_value("total_timer_time"), + "total_distance": frame.get_value("total_distance"), + "max_heart_rate": frame.get_value("max_heart_rate"), + "avg_power": frame.get_value("avg_power"), + "total_calories": frame.get_value("total_calories") + } + break + + return { + "activityType": {"typeKey": metrics.get("sport", "other")}, + "summaryDTO": { + "duration": metrics.get("total_timer_time"), + "distance": metrics.get("total_distance"), + "maxHR": metrics.get("max_heart_rate"), + "avgPower": metrics.get("avg_power"), + "calories": metrics.get("total_calories") + } + } + except Exception: + return None + +def get_activity_metrics(activity, client=None): + """ + Get activity metrics from local file or Garmin API + Returns parsed metrics or None + """ + metrics = None + if activity.filename and os.path.exists(activity.filename): + file_type = detect_file_type(activity.filename) + if file_type == 'fit': + metrics = parse_fit_file(activity.filename) + elif file_type == 'xml': + metrics = parse_xml_file(activity.filename) + if not metrics and client: + try: + metrics = client.get_activity_details(activity.activity_id) + except Exception: + pass + return metrics diff --git a/garminsync/cli.py b/garminsync/cli.py index ae64c56..1a16ae5 100644 --- a/garminsync/cli.py +++ b/garminsync/cli.py @@ -173,13 +173,20 @@ def daemon_mode( bool, typer.Option("--status", help="Show daemon status") ] = False, port: Annotated[int, typer.Option("--port", help="Web UI port")] = 8080, + run_migrations: Annotated[ + bool, + typer.Option( + "--run-migrations/--skip-migrations", + help="Run database migrations on startup (default: run)" + ) + ] = True, ): """Daemon mode operations""" from .daemon import GarminSyncDaemon if start: daemon = GarminSyncDaemon() - daemon.start(web_port=port) + daemon.start(web_port=port, run_migrations=run_migrations) elif stop: # Implementation for stopping daemon (PID file or signal) typer.echo("Stopping daemon...") diff --git a/garminsync/daemon.py b/garminsync/daemon.py index 7d743bf..6c7e58b 100644 --- a/garminsync/daemon.py +++ b/garminsync/daemon.py @@ -10,6 +10,7 @@ from apscheduler.triggers.cron import CronTrigger from .database import Activity, DaemonConfig, SyncLog, get_session from .garmin import GarminClient from .utils import logger +from .activity_parser import get_activity_metrics class GarminSyncDaemon: @@ -18,8 +19,17 @@ class GarminSyncDaemon: self.running = False self.web_server = None - def start(self, web_port=8888): - """Start daemon with scheduler and web UI""" + def start(self, web_port=8888, run_migrations=True): + """Start daemon with scheduler and web UI + :param web_port: Port for the web UI + :param run_migrations: Whether to run database migrations on startup + """ + # Set migration flag for entrypoint + if run_migrations: + os.environ['RUN_MIGRATIONS'] = "1" + else: + os.environ['RUN_MIGRATIONS'] = "0" + try: # Load configuration from database config_data = self.load_config() @@ -105,26 +115,38 @@ class GarminSyncDaemon: for activity in missing_activities: try: - # Use the correct method name + # Download FIT file fit_data = client.download_activity_fit(activity.activity_id) - - # Save the file + + # Save to file import os from pathlib import Path - data_dir = Path(os.getenv("DATA_DIR", "data")) data_dir.mkdir(parents=True, exist_ok=True) - timestamp = activity.start_time.replace(":", "-").replace(" ", "_") filename = f"activity_{activity.activity_id}_{timestamp}.fit" filepath = data_dir / filename - + with open(filepath, "wb") as f: f.write(fit_data) - + + # Update activity record activity.filename = str(filepath) activity.downloaded = True activity.last_sync = datetime.now().isoformat() + + # Get metrics immediately after download + metrics = get_activity_metrics(activity, client) + if metrics: + # Update metrics if available + activity.activity_type = metrics.get("activityType", {}).get("typeKey") + activity.duration = int(float(metrics.get("summaryDTO", {}).get("duration", 0))) + activity.distance = float(metrics.get("summaryDTO", {}).get("distance", 0)) + activity.max_heart_rate = int(float(metrics.get("summaryDTO", {}).get("maxHR", 0))) + activity.avg_power = float(metrics.get("summaryDTO", {}).get("avgPower", 0)) + activity.calories = int(float(metrics.get("summaryDTO", {}).get("calories", 0))) + + session.commit() downloaded_count += 1 session.commit() @@ -135,7 +157,8 @@ class GarminSyncDaemon: session.rollback() self.log_operation( - "sync", "success", f"Downloaded {downloaded_count} new activities" + "sync", "success", + f"Downloaded {downloaded_count} new activities and updated metrics" ) # Update last run time diff --git a/garminsync/database.py b/garminsync/database.py index 2479684..35c2d42 100644 --- a/garminsync/database.py +++ b/garminsync/database.py @@ -120,6 +120,8 @@ def get_session(): return Session() +from garminsync.activity_parser import get_activity_metrics + def sync_database(garmin_client): """Sync local database with Garmin Connect activities. @@ -134,36 +136,70 @@ def sync_database(garmin_client): print("No activities returned from Garmin API") return - for activity in activities: - # Check if activity is a dictionary and has required fields - if not isinstance(activity, dict): - print(f"Invalid activity data: {activity}") + for activity_data in activities: + if not isinstance(activity_data, dict): + print(f"Invalid activity data: {activity_data}") continue - # Safely access dictionary keys - activity_id = activity.get("activityId") - start_time = activity.get("startTimeLocal") - avg_heart_rate = activity.get("averageHR", None) - calories = activity.get("calories", None) - + activity_id = activity_data.get("activityId") + start_time = activity_data.get("startTimeLocal") + if not activity_id or not start_time: - print(f"Missing required fields in activity: {activity}") + print(f"Missing required fields in activity: {activity_data}") continue - existing = ( - session.query(Activity).filter_by(activity_id=activity_id).first() - ) + existing = session.query(Activity).filter_by(activity_id=activity_id).first() + + # Create or update basic activity info if not existing: - new_activity = Activity( + activity = Activity( activity_id=activity_id, start_time=start_time, - avg_heart_rate=avg_heart_rate, - calories=calories, downloaded=False, created_at=datetime.now().isoformat(), last_sync=datetime.now().isoformat(), ) - session.add(new_activity) + session.add(activity) + session.flush() # Assign ID + else: + activity = existing + + # Update metrics using shared parser + metrics = get_activity_metrics(activity, garmin_client) + if metrics: + activity.activity_type = metrics.get("activityType", {}).get("typeKey") + + # Extract duration in seconds + duration = metrics.get("summaryDTO", {}).get("duration") + if duration is not None: + activity.duration = int(float(duration)) + + # Extract distance in meters + distance = metrics.get("summaryDTO", {}).get("distance") + if distance is not None: + activity.distance = float(distance) + + # Extract heart rates + max_hr = metrics.get("summaryDTO", {}).get("maxHR") + if max_hr is not None: + activity.max_heart_rate = int(float(max_hr)) + + avg_hr = metrics.get("summaryDTO", {}).get("avgHR", None) or \ + metrics.get("summaryDTO", {}).get("averageHR", None) + if avg_hr is not None: + activity.avg_heart_rate = int(float(avg_hr)) + + # Extract power and calories + avg_power = metrics.get("summaryDTO", {}).get("avgPower") + if avg_power is not None: + activity.avg_power = float(avg_power) + + calories = metrics.get("summaryDTO", {}).get("calories") + if calories is not None: + activity.calories = int(float(calories)) + + # Update sync timestamp + activity.last_sync = datetime.now().isoformat() session.commit() except SQLAlchemyError as e: diff --git a/garminsync/migrate_activities.py b/garminsync/migrate_activities.py index 5df1a88..c98605c 100644 --- a/garminsync/migrate_activities.py +++ b/garminsync/migrate_activities.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -Migration script to populate new activity fields from Garmin API +Migration script to populate new activity fields from FIT files or Garmin API """ import os @@ -16,11 +16,22 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from garminsync.database import Activity, get_session, init_db from garminsync.garmin import GarminClient +from garminsync.activity_parser import get_activity_metrics def add_columns_to_database(): """Add new columns to the activities table if they don't exist""" - print("Adding new columns to database...") + +# Add the parent directory to the path to import garminsync modules +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from garminsync.database import Activity, get_session, init_db +from garminsync.garmin import GarminClient + + +def add_columns_to_database(): + """Add new columns to the activities table if they don't exist""" + print("Adding new columns to database...", flush=True) # Get database engine db_path = os.path.join(os.getenv("DATA_DIR", "data"), "garmin.db") @@ -49,7 +60,7 @@ def add_columns_to_database(): with engine.connect() as conn: for column_name in new_columns: if column_name not in existing_columns: - print(f"Adding column {column_name}...") + print(f"Adding column {column_name}...", flush=True) if column_name in ["distance", "avg_power"]: conn.execute( text( @@ -69,21 +80,22 @@ def add_columns_to_database(): ) ) conn.commit() - print(f"Column {column_name} added successfully") + print(f"Column {column_name} added successfully", flush=True) else: - print(f"Column {column_name} already exists") + print(f"Column {column_name} already exists", flush=True) - print("Database schema updated successfully") + print("Database schema updated successfully", flush=True) return True except Exception as e: - print(f"Failed to update database schema: {e}") + print(f"Failed to update database schema: {e}", flush=True) return False + def migrate_activities(): - """Migrate activities to populate new fields from Garmin API""" - print("Starting activity migration...") + """Migrate activities to populate new fields from FIT files or Garmin API""" + print("Starting activity migration...", flush=True) # First, add columns to database if not add_columns_to_database(): @@ -92,9 +104,9 @@ def migrate_activities(): # Initialize Garmin client try: client = GarminClient() - print("Garmin client initialized successfully") + print("Garmin client initialized successfully", flush=True) except Exception as e: - print(f"Failed to initialize Garmin client: {e}") + print(f"Failed to initialize Garmin client: {e}", flush=True) # Continue with migration but without Garmin data client = None @@ -106,12 +118,12 @@ def migrate_activities(): activities = ( session.query(Activity).filter(Activity.activity_type.is_(None)).all() ) - print(f"Found {len(activities)} activities to migrate") + print(f"Found {len(activities)} activities to migrate", flush=True) # If no activities found, try to get all activities (in case activity_type column was just added) if len(activities) == 0: activities = session.query(Activity).all() - print(f"Found {len(activities)} total activities") + print(f"Found {len(activities)} total activities", flush=True) updated_count = 0 error_count = 0 @@ -119,13 +131,16 @@ def migrate_activities(): for i, activity in enumerate(activities): try: print( - f"Processing activity {i+1}/{len(activities)} (ID: {activity.activity_id})" + f"Processing activity {i+1}/{len(activities)} (ID: {activity.activity_id})", + flush=True ) - # Fetch detailed activity data from Garmin (if client is available) - activity_details = None - if client: - activity_details = client.get_activity_details(activity.activity_id) + # Use shared parser to get activity metrics + activity_details = get_activity_metrics(activity, client) + if activity_details is not None: + print(f" Successfully parsed metrics for activity {activity.activity_id}", flush=True) + else: + print(f" Could not retrieve metrics for activity {activity.activity_id}", flush=True) # Update activity fields if we have details if activity_details: @@ -170,19 +185,19 @@ def migrate_activities(): # Print progress every 10 activities if (i + 1) % 10 == 0: - print(f" Progress: {i+1}/{len(activities)} activities processed") + print(f" Progress: {i+1}/{len(activities)} activities processed", flush=True) except Exception as e: - print(f" Error processing activity {activity.activity_id}: {e}") + print(f" Error processing activity {activity.activity_id}: {e}", flush=True) session.rollback() error_count += 1 continue - print(f"Migration completed. Updated: {updated_count}, Errors: {error_count}") + print(f"Migration completed. Updated: {updated_count}, Errors: {error_count}", flush=True) return True # Allow partial success except Exception as e: - print(f"Migration failed: {e}") + print(f"Migration failed: {e}", flush=True) return False finally: session.close() diff --git a/justfile b/justfile index aa2d91c..ed7bd1f 100644 --- a/justfile +++ b/justfile @@ -9,11 +9,10 @@ dev: just build docker run -it --rm --env-file .env -v $(pwd)/garminsync:/app/garminsync -v $(pwd)/data:/app/data -p 8888:8888 --name garminsync-dev garminsync uvicorn garminsync.web.app:app --reload --host 0.0.0.0 --port 8080 -# Run database migrations (container-based) +# Run database migrations with enhanced logging (container-based) migrate: just build - docker run --rm --env-file .env -v $(pwd)/data:/app/data --entrypoint "alembic" garminsync upgrade head - + docker run --rm --env-file .env -v $(pwd)/data:/app/data --entrypoint "python" garminsync -m garminsync.cli migrate # Run validation tests (container-based) test: just build @@ -41,7 +40,7 @@ format: # Start production server run_server: just build - docker run -d --rm --env-file .env -v $(pwd)/data:/app/data -p 8888:8888 --name garminsync garminsync daemon --start + docker run -d --rm --env-file .env -e RUN_MIGRATIONS=1 -v $(pwd)/data:/app/data -p 8888:8888 --name garminsync garminsync daemon --start # Stop production server stop_server: @@ -50,7 +49,7 @@ stop_server: # Run server in live mode for debugging run_server_live: just build - docker run -it --rm --env-file .env -v $(pwd)/data:/app/data -p 8888:8888 --name garminsync garminsync daemon --start + docker run -it --rm --env-file .env -e RUN_MIGRATIONS=1 -v $(pwd)/data:/app/data -p 8888:8888 --name garminsync garminsync daemon --start # Clean up any existing container cleanup: diff --git a/requirements.txt b/requirements.txt index 07e3f38..c7bd972 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,4 @@ tqdm==4.66.1 sqlalchemy==2.0.30 pylint==3.1.0 pygments==2.18.0 +fitdecode diff --git a/tests/__pycache__/test_sync.cpython-310-pytest-8.1.1.pyc b/tests/__pycache__/test_sync.cpython-310-pytest-8.1.1.pyc index bcebf2059ccf4a6610a3d0072e3a1d87f2e33d36..61ed967c6768a838a55d05b9a9c2aa85c37872cd 100644 GIT binary patch literal 4769 zcmb7IU2NP~7523~9?wrEGsz@r+QJTHyY1}KG?SnHEX|U#yRg-6Lt(>D{eb1pbtcY^ zJ+s%IwlmDUP~KRyJn|4wBCWKH1P@3no{)Ip0f|>0ko*b~iae(UwGk2)3-H zNHC*l3ZF@9X}F6`)y`-cJF8{woR%Ykncm6oO~8z_FY+hkX2G7+ChekDv`bnE#!^)I zo}f+9G*#g~O*1qL_cG1VJltpK1TDaQmXddc>f`}fBy3bAU%uO_zvrjghF9PAGk1-K zRmYK@eAQ^TVSd8xI(3~Ip7FqNO<%oT_pFDO*Y(Q{)6}b9KG~6X#GMpe3XG;{x=*P31M$O8 zhyXqKESj?j&Ep5rNm|^MX$jtj8s7&EwJ$s&M1Z$s24Y{_$qpnsHIVyKUw%&k$$F{gn_+1*^<$XeD zcjZ}OpwKybrmt88+G(!@T2uY>K6ygE3*RIQ0}(C>t`yipcBZ*)<$=_f++Wc7*y=xm zCxg|0%vTz-Iz789!Ak0AB{ja%K>DVDYs$hCi3m=yCb++T1U|vv`*6i)sq*fLKIs1; zu8%s^bG?^0P1ma=;-T`$^0q4vjiyB_x9{DlFw%U2d_@V8l8 ze0353hN7Phqvj6v3$AA{Z_~2PcUpC$d9air@#mH{Yqj9Hjl&lj2D2>(*Ix+ZaRGzS zHC@+gIYZ*-ndxVgTFN#azx6dug)e9^%XF)vmJb)%gjf4nw`1D|>%Ou1 zl>-r8HYA7Q+S-ux-oALb-Ce1=+3$=Rm9+}Cmck)ujD{Ag?t(#ScCY};H1&F`<9OLY zWi5E%de!ywHe_z(4%wI5-9;2jC{|EhKP(uoYch{7auc3`!JEA=#RmU(W9lv2w!j2# zj|qlVCH86fIQ9aH3W(|q!>{!VF{}Xhuo5aDI{K;iJ0|P;nJug7nM{As^;26Gb3G=Z zN(xmZ_-H?S^v!;Dz1{U76m^67C7n+&v(3Pqw!ut457YXC4!jV8#aEgbSFV3Xceh)e zCe=|@Z!}vEjHd1ww&`bd*Vr;~*e~4X&VkK+ld%@_r}eECvkgx-AGe!^!?#^ig75Yf zT-l`6jGyCvuQx5z@z{B^Ifpg`POZf|%kj>&cxN@}1nb=hJ|gJU7UNT^LxFuBmN=Y4 zfWC@;aJkfZVzj&dW$v3qn(COQm=zWcAse;<$;-VALJ?I_CF)ZpRU`#bCJGTnk>IcV zFZHSXOwH{|l-&C}TLa|;#N+O&aB+zL7BoVLk3+gAQxPCs0)UFeATNnpF8!g9-kf*Z6S~g;jmelZhf0K<3GXr)(XD60Xd_ zK*W}RO|T`fokV4jK=CvUj48$4OVtyMnE`ia!QH>(j0pz-C70GR8O#EW@nnt&I@=A)V{h?9*8tG z$n?d22Dlu#^S2`_CE)To9OqnqCbIIU+)7|a_H%9-w1CUcB&^H;htG31&C;{ zTp2N|GGf-qx(Mc>gUC-Y`}@-+Ogjc#nt#61HMnz;sRn zg8luD#Y|e+vdkuR8O9F7JY;zk7;bC=MDK`!mp5xm0h2P!`&f3cDO8_BF$3bya0x=? zXX^(Lg94jJaSjE~f=^-V0*cR~KowxI<>ygWcto^-AwsrV1?DLLuA31ab%!zXKvr5! zP!-S{`wT7?Fv{^%zKF(PsD!09^lDNe6HZxbh4@THY0l8BD!x05Xq5O!SyZ5pqSkET1oDbASvVfi*?os z&HzF_R%JomMbi;y!0rZh7H|fRj^m7cf-^jh#uJ<~3K3^aMriz1j>cG7eS;%#P}@u< z&^TUML2bt?s}h|W!x>mrB`YiBjA`HuxFXJw5}fgyk(J09<%lzm>c5Y;mB<-GZdt$? z$@&jDV|K(DSofXI8Ig6IGoTmajNiwj8?qsqr6SLUL?5!@FL8a;sm}M7Pss*eky%bq zlZ60Q!GfuRRgT-shm$zA@qgt7`yvMRf2RcYI_e<~vNutD83n>O!|-NzP#nvVE8}ux zWNnxtUH-p_`|ySl8sbs)xoEfny45*f4NC%!|Ngul6cG!-sSl3)3(S1PEDs8TXb9@T zV2rbeuLO_ylG}B)iH>7=X!HV}{sXw+yu<4n8UCTN?JH1$K(VDwpHhhi{G^~O(kSq3 y%}<3KzIl!_G(F}h?lOM*RBzcW+G(1vL9PpK8N>xaC1p~O%6OJPujbUbeCa>8uc#>i literal 3491 zcmb7HTaO#X67HUx$K%W1vI%fFFa!ZHkd3|Jl0*a%1%wc=LShBNFQbWjc6XM!nC@|c zjjbb{g$JaA;0N%q`^LVUAJnhBuzvu-N?fXX#@=yQBjIdUS6B7SR9AoXbti2!90I@e zs~`6MvqZ>`I4J)N7`y>Z{0{^pj7FqORjPMQIZA6W@*=;J_jF#f%?lR~Wt9(hiE6iq9=vSG;YS6E-I%`0Gfi>9@ z^y`d%NLtG$5E0pGY0`KM#eS2-)~Xb5W}SBF+cO~`62*A{cWGeVRAPL zgCy(Wv(ujjiVf{9H1QpXjAWG2Z|DS8!E|PPuQ3y1gML>VlYPz8GlNwQ=(nv2W%iiv z>-#1&3t9zy_Lv+{N^s_3l~tL8YgV(I!`cz#OzHJp|Clh_AO<I!X^4K%JGt2=jFxeED&$I0?4&F<9esr=~rRC=~w>zEuxzi_iRSq$X zjcIB(c7x0lfe>MuY;g+dJNZc`q}AY8OAS&IP~ zUQcbgtgAs0=`rE|h8Le+LbzPZAh|+GPpz##lvmKEMl%qpFJjCgD=nrDuMrnPIGU~5 z)c)NuSE;Qv^}61qmw)C@f@?9{67ZJcHiMTA@Ny?KGfQ9s2r_d9rplo`15-8E0ZiHq zOqv2ydk#!sYgwa>MPS-WlVFA;cZa7jB8yuPix(UaBqQ zm=b;ogj^}MCrPIp$R9=LV<>PJa`6PteQr_O69}!8^`~I6z&If56kCz+UpRW^@?8Z3 zw!6^86#!^C-%JlFW)&om53`}f_1PX&VGjh9xU;atq4X!(Slh2o^s&D0Obk|=m}6sX zeyUBZu_bthv{xTnJCrrhzM~x(teG_qn@8k65$}S3X-tnO{{XaQ)DjAQ)U_k3dcEk4 zqBoByTl(I-NG27w%vQ#gkb-V!D=>R?Y#-4B`d^6IAQQmCM2BWTvmh=csr|J{yEQVD3&BL`*2u@ERnYBe`;{6W&D8 z05Lk_rR&?B&75C2o<$00TPH{)$MjUYefyY>;%Bb(hp)7R^F^V!w_Vg;nwmfO3CwIj~vZ=Ygz) z+q*Xaehd>Ktq8eV$VWV}mkuJPrh2;(pMS*jFb1dZC|0Co6%5Q*wjsfNl_%V3r89*KEbXN5y zRQ=PyyyPH1sLwmvv68BX&WwMd%8RP9W>saWs(N*L-kgnVO8+}Ht`}_FV9f<=j4b^3Yz#9BHr5qxzMjJk*jh`Y>*wG` zT>`@-zXnb(buM1c+-%qnavC13pF3kKXXbe*@ym#Xr%^nE0$G4Ri{g0{FQ9l4#mguj zI=-(if=luJDO!f=QuPtMp(tcc)Bak>x4_@JDD5ff@oQj^k9s*xZse>4)CKJRfQK1S z-pnK`g^gduXRQ1Ysf(fLHU>$UVW1m$Bg{!#$5{sa7h+FZK(BDu?5>@2*;UNlT`75T zRb6w{RZrbN6a^{Fb6i6yODbRLrenVrr)&@fuj5IlvT&K$bQv!)TAkXuy>2htYxUJX E0ioS(&;S4c diff --git a/tests/test_sync.py b/tests/test_sync.py index 2243039..f45ed44 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -1,102 +1,110 @@ import pytest import sys -from unittest.mock import Mock, patch +from unittest.mock import Mock, patch, MagicMock # Add the project root to the Python path sys.path.insert(0, '/app') -from garminsync.database import sync_database -from garminsync.garmin import GarminClient - +from garminsync.database import sync_database, Activity, get_activity_metrics def test_sync_database_with_valid_activities(): """Test sync_database with valid API response""" - mock_client = Mock(spec=GarminClient) + mock_client = Mock() mock_client.get_activities.return_value = [ {"activityId": 12345, "startTimeLocal": "2023-01-01T10:00:00"}, {"activityId": 67890, "startTimeLocal": "2023-01-02T11:00:00"} ] - with patch('garminsync.database.get_session') as mock_session: - mock_session.return_value.query.return_value.filter_by.return_value.first.return_value = None + mock_session = MagicMock() + mock_session.query.return_value.filter_by.return_value.first.return_value = None + + with patch('garminsync.database.get_session', return_value=mock_session), \ + patch('garminsync.database.get_activity_metrics', return_value={ + "activityType": {"typeKey": "running"}, + "summaryDTO": { + "duration": 3600, + "distance": 10.0, + "maxHR": 180, + "calories": 400 + } + }): sync_database(mock_client) - # Verify get_activities was called - mock_client.get_activities.assert_called_once_with(0, 1000) - - # Verify database operations - mock_session.return_value.add.assert_called() - mock_session.return_value.commit.assert_called() - + # Verify activities processed + assert mock_session.add.call_count == 2 + assert mock_session.commit.called def test_sync_database_with_none_activities(): """Test sync_database with None response from API""" - mock_client = Mock(spec=GarminClient) + mock_client = Mock() mock_client.get_activities.return_value = None - with patch('garminsync.database.get_session') as mock_session: + mock_session = MagicMock() + + with patch('garminsync.database.get_session', return_value=mock_session): sync_database(mock_client) - - # Verify get_activities was called - mock_client.get_activities.assert_called_once_with(0, 1000) - - # Verify no database operations - mock_session.return_value.add.assert_not_called() - mock_session.return_value.commit.assert_not_called() - + mock_session.add.assert_not_called() def test_sync_database_with_missing_fields(): """Test sync_database with activities missing required fields""" - mock_client = Mock(spec=GarminClient) + mock_client = Mock() mock_client.get_activities.return_value = [ - {"activityId": 12345}, # Missing startTimeLocal - {"startTimeLocal": "2023-01-02T11:00:00"}, # Missing activityId - {"activityId": 67890, "startTimeLocal": "2023-01-03T12:00:00"} # Valid + {"activityId": 12345}, + {"startTimeLocal": "2023-01-02T11:00:00"}, + {"activityId": 67890, "startTimeLocal": "2023-01-03T12:00:00"} ] - with patch('garminsync.database.get_session') as mock_session: - mock_session.return_value.query.return_value.filter_by.return_value.first.return_value = None - + # Create a mock that returns None for existing activity + mock_session = MagicMock() + mock_session.query.return_value.filter_by.return_value.first.return_value = None + + with patch('garminsync.database.get_session', return_value=mock_session), \ + patch('garminsync.database.get_activity_metrics', return_value={ + "summaryDTO": {"duration": 3600.0} + }): sync_database(mock_client) - - # Verify only one activity was added (the valid one) - assert mock_session.return_value.add.call_count == 1 - mock_session.return_value.commit.assert_called() - + # Only valid activity should be added + assert mock_session.add.call_count == 1 + added_activity = mock_session.add.call_args[0][0] + assert added_activity.activity_id == 67890 def test_sync_database_with_existing_activities(): """Test sync_database doesn't duplicate existing activities""" - mock_client = Mock(spec=GarminClient) + mock_client = Mock() mock_client.get_activities.return_value = [ {"activityId": 12345, "startTimeLocal": "2023-01-01T10:00:00"} ] - with patch('garminsync.database.get_session') as mock_session: - # Mock existing activity - mock_session.return_value.query.return_value.filter_by.return_value.first.return_value = Mock() - + mock_session = MagicMock() + mock_session.query.return_value.filter_by.return_value.first.return_value = Mock() + + with patch('garminsync.database.get_session', return_value=mock_session), \ + patch('garminsync.database.get_activity_metrics', return_value={ + "summaryDTO": {"duration": 3600.0} + }): sync_database(mock_client) - - # Verify no new activities were added - mock_session.return_value.add.assert_not_called() - mock_session.return_value.commit.assert_called() - + mock_session.add.assert_not_called() def test_sync_database_with_invalid_activity_data(): """Test sync_database with invalid activity data types""" - mock_client = Mock(spec=GarminClient) + mock_client = Mock() mock_client.get_activities.return_value = [ - "invalid activity data", # Not a dict - None, # None value - {"activityId": 12345, "startTimeLocal": "2023-01-01T10:00:00"} # Valid + "invalid data", + None, + {"activityId": 12345, "startTimeLocal": "2023-01-01T10:00:00"} ] - with patch('garminsync.database.get_session') as mock_session: - mock_session.return_value.query.return_value.filter_by.return_value.first.return_value = None - + # Create a mock that returns None for existing activity + mock_session = MagicMock() + mock_session.query.return_value.filter_by.return_value.first.return_value = None + + with patch('garminsync.database.get_session', return_value=mock_session), \ + patch('garminsync.database.get_activity_metrics', return_value={ + "summaryDTO": {"duration": 3600.0} + }): sync_database(mock_client) - - # Verify only one activity was added (the valid one) - assert mock_session.return_value.add.call_count == 1 - mock_session.return_value.commit.assert_called() + # Only valid activity should be added + assert mock_session.add.call_count == 1 + added_activity = mock_session.add.call_args[0][0] + assert added_activity.activity_id == 12345 diff --git a/workflows.md b/workflows.md new file mode 100644 index 0000000..389d4d2 --- /dev/null +++ b/workflows.md @@ -0,0 +1,100 @@ +# GarminSync Workflows + +## Migration Workflow + +### Purpose +Add new columns to database and populate with activity metrics + +### Trigger +`python cli.py migrate` + +### Steps +1. Add required columns to activities table: + - activity_type (TEXT) + - duration (INTEGER) + - distance (REAL) + - max_heart_rate (INTEGER) + - avg_power (REAL) + - calories (INTEGER) +2. For each activity: + - Parse metrics from local FIT/XML files + - Fetch from Garmin API if local files missing + - Update database fields +3. Commit changes +4. Report migration status + +### Error Handling +- Logs errors per activity +- Marks unprocessable activities as "Unknown" +- Continues processing other activities on error + +## Sync Workflow + +### Purpose +Keep local database synchronized with Garmin Connect + +### Triggers +- CLI commands (`list`, `download`) +- Scheduled daemon (every 6 hours by default) +- Web UI requests + +### Core Components +- `sync_database()`: Syncs activity metadata +- `download()`: Fetches missing FIT files +- Daemon: Background scheduler and web UI + +### Process Flow +1. Authenticate with Garmin API +2. Fetch latest activities +3. For each activity: + - Parse metrics from FIT/XML files + - Fetch from Garmin API if local files missing + - Update database fields +4. Download missing activity files +5. Update sync timestamps +6. Log operations + +### Database Schema +```mermaid +erDiagram + activities { + integer activity_id PK + string start_time + string activity_type + integer duration + float distance + integer max_heart_rate + integer avg_heart_rate + float avg_power + integer calories + string filename + boolean downloaded + string created_at + string last_sync + } + + daemon_config { + integer id PK + boolean enabled + string schedule_cron + string last_run + string next_run + string status + } + + sync_logs { + integer id PK + string timestamp + string operation + string status + string message + integer activities_processed + integer activities_downloaded + } +``` + +### Key Notes +- Data directory: `data/` (configurable via DATA_DIR) +- Web UI port: 8080 (default) +- Downloaded files: `activity_{id}_{timestamp}.fit` +- Metrics include: heart rate, power, calories, distance