Add 'tags' to event model. Add a dataimport system - currently for MaxMind zip files

This commit is contained in:
Dan Milne
2025-11-11 10:31:36 +11:00
parent 772fae7e8b
commit 26216da9ca
34 changed files with 3580 additions and 14 deletions

96
app/models/data_import.rb Normal file
View File

@@ -0,0 +1,96 @@
class DataImport < ApplicationRecord
has_one_attached :file
validates :import_type, presence: true, inclusion: { in: %w[asn country] }
validates :status, presence: true, inclusion: { in: %w[pending processing completed failed] }
validates :filename, presence: true
attribute :import_stats, default: -> { {} }
# Scopes
scope :recent, -> { order(created_at: :desc) }
scope :by_type, ->(type) { where(import_type: type) }
scope :by_status, ->(status) { where(status: status) }
scope :completed, -> { where(status: 'completed') }
scope :failed, -> { where(status: 'failed') }
scope :processing, -> { where(status: 'processing') }
scope :pending, -> { where(status: 'pending') }
# State management
def pending?
status == 'pending'
end
def processing?
status == 'processing'
end
def completed?
status == 'completed'
end
def failed?
status == 'failed'
end
def start_processing!
update!(
status: 'processing',
started_at: Time.current
)
end
def complete!
updates = {
status: 'completed',
completed_at: Time.current
}
updates[:total_records] = processed_records if total_records.zero?
update!(updates)
end
def fail!(error_message = nil)
update!(
status: 'failed',
completed_at: Time.current,
error_message: error_message
)
end
def progress_percentage
if total_records.zero?
processing? ? 0.1 : 0 # Show minimal progress for processing jobs
else
(processed_records.to_f / total_records * 100).round(2)
end
end
def duration
return 0 unless started_at
end_time = completed_at || Time.current
duration_seconds = (end_time - started_at).round(2)
duration_seconds.negative? ? 0 : duration_seconds
end
def records_per_second
# Handle very fast imports that complete in less than 1 second
if duration.zero?
# Use time since started if no duration available yet
time_elapsed = started_at ? (Time.current - started_at) : 0
return 0 if time_elapsed < 1
(processed_records.to_f / time_elapsed).round(2)
else
(processed_records.to_f / duration).round(2)
end
end
def update_progress(processed: nil, failed: nil, total_records: nil, stats: nil)
updates = {}
updates[:processed_records] = processed if processed
updates[:failed_records] = failed if failed
updates[:total_records] = total_records if total_records
updates[:import_stats] = stats if stats
update!(updates) if updates.any?
end
end