Add 'tags' to event model. Add a dataimport system - currently for MaxMind zip files

app/controllers/data_imports_controller.rb

@@ -0,0 +1,131 @@
+class DataImportsController < ApplicationController
+  before_action :require_admin!
+  before_action :set_data_import, only: [:show, :destroy, :progress]
+
+  def index
+    @data_imports = DataImport.all
+
+    # Apply filters
+    @data_imports = @data_imports.where(import_type: params[:import_type]) if params[:import_type].present?
+    @data_imports = @data_imports.where(status: params[:status]) if params[:status].present?
+    @data_imports = @data_imports.where("filename ILIKE ?", "%#{params[:filename]}%") if params[:filename].present?
+
+    @pagy, @data_imports = pagy(@data_imports.order(created_at: :desc))
+  end
+
+  def new
+    @data_import = DataImport.new
+  end
+
+  def create
+    # Save uploaded file and queue import job
+    uploaded_file = params[:data_import][:file]
+    if uploaded_file.nil?
+      @data_import = DataImport.new
+      flash.now[:alert] = "Please select a file to import"
+      render :new, status: :unprocessable_entity
+      return
+    end
+
+    # Validate file type
+    unless valid_file?(uploaded_file)
+      @data_import = DataImport.new
+      flash.now[:alert] = "Invalid file type. Please upload a .csv or .zip file."
+      render :new, status: :unprocessable_entity
+      return
+    end
+
+    # Determine import type based on filename
+    import_type = detect_import_type_from_filename(uploaded_file.original_filename)
+
+    # Create the DataImport record with the attached file
+    @data_import = DataImport.create!(
+      import_type: import_type,
+      filename: uploaded_file.original_filename,
+      status: 'pending'
+    )
+
+    # Attach the file using Active Storage
+    @data_import.file.attach(uploaded_file)
+
+    # Queue appropriate import job - pass the entire DataImport object
+    if import_type == 'asn'
+      GeoliteAsnImportJob.perform_later(@data_import)
+    else
+      GeoliteCountryImportJob.perform_later(@data_import)
+    end
+
+    redirect_to @data_import, notice: "Import has been queued and will begin processing shortly."
+  rescue => e
+    Rails.logger.error "Error creating import: #{e.message}"
+    Rails.logger.error e.backtrace.join("\n")
+
+    @data_import = DataImport.new if @data_import.nil?
+    flash.now[:alert] = "Error processing file: #{e.message}"
+    render :new, status: :unprocessable_entity
+  end
+
+  def show
+    # Show will display import details and progress
+  end
+
+  def progress
+    # JSON endpoint for real-time progress updates
+    render json: {
+      id: @data_import.id,
+      status: @data_import.status,
+      progress_percentage: @data_import.progress_percentage,
+      processed_records: @data_import.processed_records,
+      total_records: @data_import.total_records,
+      failed_records: @data_import.failed_records,
+      duration: @data_import.duration,
+      records_per_second: @data_import.records_per_second,
+      import_stats: @data_import.import_stats,
+      error_message: @data_import.error_message,
+      started_at: @data_import.started_at,
+      completed_at: @data_import.completed_at
+    }
+  end
+
+  def destroy
+    if @data_import.processing?
+      redirect_to @data_import, alert: "Cannot delete an import that is currently processing."
+    else
+      @data_import.destroy
+      redirect_to data_imports_path, notice: "Import was successfully deleted."
+    end
+  end
+
+  private
+
+  def set_data_import
+    @data_import = DataImport.find(params[:id])
+  end
+
+  def data_import_params
+    # No parameters needed since we detect everything automatically
+    {}
+  end
+
+  def valid_file?(uploaded_file)
+    return false unless uploaded_file.respond_to?(:original_filename)
+
+    filename = uploaded_file.original_filename.downcase
+    filename.end_with?('.csv', '.zip')
+  end
+
+  def detect_import_type_from_filename(filename)
+    # Try to detect based on filename first
+    if filename.downcase.include?('asn')
+      'asn'
+    elsif filename.downcase.include?('country')
+      'country'
+    else
+      'country' # Default fallback
+    end
+  end
+
+  def require_admin!
+    redirect_to root_path, alert: "Access denied. Admin privileges required." unless current_user&.admin?
+  end
+end