Smarter backfil

app/models/event_ddb.rb

@@ -6,6 +6,25 @@ require 'ostruct'
 # Provides an ActiveRecord-like interface for querying DuckDB events table
 # Falls back to PostgreSQL Event model if DuckDB is unavailable
 class EventDdb
+  # Enum mappings from integer to string (matching Event model)
+  ACTION_MAP = {
+    0 => "deny",
+    1 => "allow",
+    2 => "redirect",
+    3 => "challenge",
+    4 => "log"
+  }.freeze
+
+  METHOD_MAP = {
+    0 => "get",
+    1 => "post",
+    2 => "put",
+    3 => "patch",
+    4 => "delete",
+    5 => "head",
+    6 => "options"
+  }.freeze
+
   class << self
     # Get DuckDB service
     def service
@@ -624,5 +643,151 @@ class EventDdb
       Rails.logger.error "[EventDdb] Error in bot_traffic_timeline: #{e.message}"
       nil
     end
+
+    # Search events with filters and pagination
+    # Returns { total_count:, events:[], page:, per_page: }
+    # Supports filters: ip, waf_action, country, rule_id, company, asn, network_type, network_range_id, exclude_bots
+    def search(filters = {}, page: 1, per_page: 50)
+      service.with_connection do |conn|
+        # Build WHERE clause
+        where_clause, params = build_where_clause(filters)
+
+        # Get total count
+        count_sql = "SELECT COUNT(*) FROM events#{where_clause}"
+        count_result = conn.query(count_sql, *params)
+        total_count = count_result.first&.first || 0
+
+        # Get paginated results
+        offset = (page - 1) * per_page
+
+        data_sql = <<~SQL
+          SELECT
+            id, timestamp, ip_address, network_range_id, country, company,
+            asn, asn_org, is_datacenter, is_vpn, is_proxy, is_bot,
+            waf_action, request_method, response_status, rule_id,
+            request_path, user_agent, tags
+          FROM events
+          #{where_clause}
+          ORDER BY timestamp DESC
+          LIMIT ? OFFSET ?
+        SQL
+
+        result = conn.query(data_sql, *params, per_page, offset)
+
+        # Convert rows to event-like objects
+        events = result.to_a.map { |row| row_to_event(row) }
+
+        {
+          total_count: total_count,
+          events: events,
+          page: page,
+          per_page: per_page
+        }
+      end
+    rescue StandardError => e
+      Rails.logger.error "[EventDdb] Error in search: #{e.message}"
+      Rails.logger.error e.backtrace.join("\n")
+      nil
+    end
+
+    private
+
+    # Build WHERE clause and params from filters hash
+    # Returns [where_clause_string, params_array]
+    def build_where_clause(filters)
+      conditions = []
+      params = []
+
+      if filters[:ip].present?
+        conditions << "ip_address = ?"
+        params << filters[:ip]
+      end
+
+      if filters[:waf_action].present?
+        # Convert string action to integer
+        action_int = ACTION_MAP.key(filters[:waf_action].to_s)
+        if action_int
+          conditions << "waf_action = ?"
+          params << action_int
+        end
+      end
+
+      if filters[:country].present?
+        conditions << "country = ?"
+        params << filters[:country]
+      end
+
+      if filters[:rule_id].present?
+        conditions << "rule_id = ?"
+        params << filters[:rule_id].to_i
+      end
+
+      if filters[:company].present?
+        conditions << "company ILIKE ?"
+        params << "%#{filters[:company]}%"
+      end
+
+      if filters[:asn].present?
+        conditions << "asn = ?"
+        params << filters[:asn].to_i
+      end
+
+      if filters[:network_range_id].present?
+        conditions << "network_range_id = ?"
+        params << filters[:network_range_id].to_i
+      end
+
+      # Network type filter
+      if filters[:network_type].present?
+        case filters[:network_type].to_s.downcase
+        when "datacenter"
+          conditions << "is_datacenter = true"
+        when "vpn"
+          conditions << "is_vpn = true"
+        when "proxy"
+          conditions << "is_proxy = true"
+        when "standard"
+          conditions << "(is_datacenter = false AND is_vpn = false AND is_proxy = false)"
+        end
+      end
+
+      # Bot filtering
+      if filters[:exclude_bots] == true || filters[:exclude_bots] == "true"
+        conditions << "is_bot = false"
+      end
+
+      where_clause = conditions.any? ? " WHERE #{conditions.join(' AND ')}" : ""
+      [where_clause, params]
+    end
+
+    # Convert DuckDB row array to event-like OpenStruct
+    def row_to_event(row)
+      OpenStruct.new(
+        id: row[0],
+        timestamp: row[1],
+        ip_address: row[2],
+        network_range_id: row[3],
+        country: row[4],
+        company: row[5],
+        asn: row[6],
+        asn_org: row[7],
+        is_datacenter: row[8],
+        is_vpn: row[9],
+        is_proxy: row[10],
+        is_bot: row[11],
+        waf_action: ACTION_MAP[row[12]] || "unknown",
+        request_method: METHOD_MAP[row[13]],
+        response_status: row[14],
+        rule_id: row[15],
+        request_path: row[16],
+        user_agent: row[17],
+        tags: row[18] || [],
+        # Add helper method for country lookup
+        lookup_country: row[4],
+        # Network range will be loaded separately in controller
+        network_range: nil,
+        rule: nil
+      )
+    end
   end
 end

app/models/waf_policy.rb

@@ -159,16 +159,30 @@ validate :targets_must_be_array
       return nil
     end
 
-    rule = Rule.create!(
-      waf_rule_type: 'network',
-      waf_action: policy_action.to_sym,
-      network_range: network_range,
-      waf_policy: self,
-      user: user,
-      source: "policy",
-      metadata: build_rule_metadata(network_range),
-      priority: network_range.prefix_length
-    )
+    # Try to create the rule, handling duplicates gracefully
+    begin
+      rule = Rule.create!(
+        waf_rule_type: 'network',
+        waf_action: policy_action.to_sym,
+        network_range: network_range,
+        waf_policy: self,
+        user: user,
+        source: "policy",
+        metadata: build_rule_metadata(network_range),
+        priority: network_range.prefix_length
+      )
+    rescue ActiveRecord::RecordNotUnique
+      # Rule already exists (created by another job or earlier in this job)
+      # Find and return the existing rule
+      Rails.logger.debug "Rule already exists for #{network_range.cidr} with policy #{name}"
+      return Rule.find_by(
+        waf_rule_type: 'network',
+        waf_action: policy_action,
+        network_range: network_range,
+        waf_policy: self,
+        source: "policy"
+      )
+    end
 
     # Handle redirect/challenge specific data
     if redirect_action? && additional_data['redirect_url']