Use only parquet files for events

This commit is contained in:
Dan Milne
2025-12-03 17:16:38 +11:00
parent 032243ba6a
commit 693851f664
12 changed files with 673 additions and 165 deletions

View File

@@ -30,8 +30,20 @@ cleanup_old_events:
queue: background
schedule: every hour
# Sync events from PostgreSQL to DuckDB for fast analytics
sync_events_to_duckdb:
class: SyncEventsToDuckdbJob
# Export events from PostgreSQL to Parquet files for fast analytics
export_events_to_parquet:
class: ExportEventsToParquetJob
queue: default
schedule: every 1 minutes
# Consolidate completed hours into day files
consolidate_parquet_hourly:
class: ConsolidateParquetHourlyJob
queue: default
schedule: "5 * * * *" # At 5 minutes past every hour
# Consolidate completed week into archive (Monday 00:05)
consolidate_parquet_weekly:
class: ConsolidateParquetWeeklyJob
queue: default
schedule: "5 0 * * 1" # Monday at 00:05