Fix some blocked/allow laggards after migrating. Add DuckDB for outstanding analyitcs performance. Start adding an import for all bot networks
This commit is contained in:
@@ -23,9 +23,10 @@ class AnalyticsController < ApplicationController
|
||||
# Cache key includes period and start_time (hour-aligned for consistency)
|
||||
cache_key_base = "analytics/#{@time_period}/#{@start_time.to_i}"
|
||||
|
||||
# Core statistics - cached
|
||||
# Core statistics - cached (uses DuckDB if available)
|
||||
@total_events = Rails.cache.fetch("#{cache_key_base}/total_events", expires_in: cache_ttl) do
|
||||
Event.where("timestamp >= ?", @start_time).count
|
||||
with_duckdb_fallback { EventDdb.count_since(@start_time) } ||
|
||||
Event.where("timestamp >= ?", @start_time).count
|
||||
end
|
||||
|
||||
@total_rules = Rails.cache.fetch("analytics/total_rules", expires_in: 5.minutes) do
|
||||
@@ -40,31 +41,33 @@ class AnalyticsController < ApplicationController
|
||||
NetworkRange.count
|
||||
end
|
||||
|
||||
# Event breakdown by action - cached
|
||||
# Event breakdown by action - cached (uses DuckDB if available)
|
||||
@event_breakdown = Rails.cache.fetch("#{cache_key_base}/event_breakdown", expires_in: cache_ttl) do
|
||||
Event.where("timestamp >= ?", @start_time)
|
||||
.group(:waf_action)
|
||||
.count
|
||||
# Keys are already strings ("allow", "deny", etc.) from the enum
|
||||
with_duckdb_fallback { EventDdb.breakdown_by_action(@start_time) } ||
|
||||
Event.where("timestamp >= ?", @start_time)
|
||||
.group(:waf_action)
|
||||
.count
|
||||
end
|
||||
|
||||
# Top countries by event count - cached (now uses denormalized country column)
|
||||
# Top countries by event count - cached (uses DuckDB if available)
|
||||
@top_countries = Rails.cache.fetch("#{cache_key_base}/top_countries", expires_in: cache_ttl) do
|
||||
Event.where("timestamp >= ? AND country IS NOT NULL", @start_time)
|
||||
.group(:country)
|
||||
.count
|
||||
.sort_by { |_, count| -count }
|
||||
.first(10)
|
||||
with_duckdb_fallback { EventDdb.top_countries(@start_time, 10) } ||
|
||||
Event.where("timestamp >= ? AND country IS NOT NULL", @start_time)
|
||||
.group(:country)
|
||||
.count
|
||||
.sort_by { |_, count| -count }
|
||||
.first(10)
|
||||
end
|
||||
|
||||
# Top blocked IPs - cached
|
||||
# Top blocked IPs - cached (uses DuckDB if available)
|
||||
@top_blocked_ips = Rails.cache.fetch("#{cache_key_base}/top_blocked_ips", expires_in: cache_ttl) do
|
||||
Event.where("timestamp >= ?", @start_time)
|
||||
.where(waf_action: 1) # deny action in enum
|
||||
.group(:ip_address)
|
||||
.count
|
||||
.sort_by { |_, count| -count }
|
||||
.first(10)
|
||||
with_duckdb_fallback { EventDdb.top_blocked_ips(@start_time, 10) } ||
|
||||
Event.where("timestamp >= ?", @start_time)
|
||||
.where(waf_action: 0) # deny action in enum
|
||||
.group(:ip_address)
|
||||
.count
|
||||
.sort_by { |_, count| -count }
|
||||
.first(10)
|
||||
end
|
||||
|
||||
# Network range intelligence breakdown - cached
|
||||
@@ -92,7 +95,7 @@ class AnalyticsController < ApplicationController
|
||||
total_users: User.count,
|
||||
active_rules: Rule.enabled.count,
|
||||
disabled_rules: Rule.where(enabled: false).count,
|
||||
recent_errors: Event.where("timestamp >= ? AND waf_action = ?", @start_time, 1).count # 1 = deny
|
||||
recent_errors: Event.where("timestamp >= ? AND waf_action = ?", @start_time, 0).count # 0 = deny
|
||||
}
|
||||
end
|
||||
|
||||
@@ -117,38 +120,90 @@ class AnalyticsController < ApplicationController
|
||||
@time_period = params[:period]&.to_sym || :day
|
||||
@start_time = calculate_start_time(@time_period)
|
||||
|
||||
# Top networks by request volume (using denormalized network_range_id)
|
||||
# Use a subquery approach to avoid PostgreSQL GROUP BY issues with network_ranges.*
|
||||
event_stats = Event.where("timestamp >= ?", @start_time)
|
||||
.where.not(network_range_id: nil)
|
||||
.group(:network_range_id)
|
||||
.select("network_range_id, COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips")
|
||||
# Top networks by request volume - use DuckDB if available
|
||||
network_stats = with_duckdb_fallback { EventDdb.top_networks(@start_time, 50) }
|
||||
|
||||
# Join the stats back to NetworkRange to get full network details
|
||||
@top_networks = NetworkRange.joins("INNER JOIN (#{event_stats.to_sql}) stats ON stats.network_range_id = network_ranges.id")
|
||||
.select("network_ranges.*, stats.event_count, stats.unique_ips")
|
||||
.order("stats.event_count DESC")
|
||||
.limit(50)
|
||||
if network_stats
|
||||
# DuckDB path: array format [network_range_id, event_count, unique_ips]
|
||||
network_ids = network_stats.map { |row| row[0] }
|
||||
stats_by_id = network_stats.to_h { |row| [row[0], { event_count: row[1], unique_ips: row[2] }] }
|
||||
|
||||
@top_networks = NetworkRange.where(id: network_ids)
|
||||
.to_a
|
||||
.map do |network|
|
||||
stats = stats_by_id[network.id]
|
||||
network.define_singleton_method(:event_count) { stats[:event_count] }
|
||||
network.define_singleton_method(:unique_ips) { stats[:unique_ips] }
|
||||
|
||||
# Add inherited intelligence support
|
||||
intelligence = network.inherited_intelligence
|
||||
if intelligence[:inherited]
|
||||
network.define_singleton_method(:display_company) { intelligence[:company] }
|
||||
network.define_singleton_method(:display_country) { intelligence[:country] }
|
||||
network.define_singleton_method(:inherited_from) { intelligence[:parent_cidr] }
|
||||
network.define_singleton_method(:has_inherited_data?) { true }
|
||||
else
|
||||
network.define_singleton_method(:display_company) { network.company }
|
||||
network.define_singleton_method(:display_country) { network.country }
|
||||
network.define_singleton_method(:inherited_from) { nil }
|
||||
network.define_singleton_method(:has_inherited_data?) { false }
|
||||
end
|
||||
|
||||
network
|
||||
end
|
||||
.sort_by { |n| -n.event_count }
|
||||
else
|
||||
# PostgreSQL fallback
|
||||
event_stats = Event.where("timestamp >= ?", @start_time)
|
||||
.where.not(network_range_id: nil)
|
||||
.group(:network_range_id)
|
||||
.select("network_range_id, COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips")
|
||||
|
||||
@top_networks = NetworkRange.joins("INNER JOIN (#{event_stats.to_sql}) stats ON stats.network_range_id = network_ranges.id")
|
||||
.select("network_ranges.*, stats.event_count, stats.unique_ips")
|
||||
.order("stats.event_count DESC")
|
||||
.limit(50)
|
||||
|
||||
# Add inherited intelligence support for PostgreSQL fallback
|
||||
@top_networks = @top_networks.to_a.map do |network|
|
||||
intelligence = network.inherited_intelligence
|
||||
if intelligence[:inherited]
|
||||
network.define_singleton_method(:display_company) { intelligence[:company] }
|
||||
network.define_singleton_method(:display_country) { intelligence[:country] }
|
||||
network.define_singleton_method(:inherited_from) { intelligence[:parent_cidr] }
|
||||
network.define_singleton_method(:has_inherited_data?) { true }
|
||||
else
|
||||
network.define_singleton_method(:display_company) { network.company }
|
||||
network.define_singleton_method(:display_country) { network.country }
|
||||
network.define_singleton_method(:inherited_from) { nil }
|
||||
network.define_singleton_method(:has_inherited_data?) { false }
|
||||
end
|
||||
network
|
||||
end
|
||||
end
|
||||
|
||||
# Network type breakdown with traffic stats
|
||||
@network_breakdown = calculate_network_type_stats(@start_time)
|
||||
|
||||
# Company breakdown for top traffic sources (using denormalized company column)
|
||||
@top_companies = Event.where("timestamp >= ? AND company IS NOT NULL", @start_time)
|
||||
# Company breakdown for top traffic sources - use DuckDB if available
|
||||
@top_companies = with_duckdb_fallback { EventDdb.top_companies(@start_time, 20) } ||
|
||||
Event.where("timestamp >= ? AND company IS NOT NULL", @start_time)
|
||||
.group(:company)
|
||||
.select("company, COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips, COUNT(DISTINCT network_range_id) as network_count")
|
||||
.order("event_count DESC")
|
||||
.limit(20)
|
||||
|
||||
# ASN breakdown (using denormalized asn columns)
|
||||
@top_asns = Event.where("timestamp >= ? AND asn IS NOT NULL", @start_time)
|
||||
# ASN breakdown - use DuckDB if available
|
||||
@top_asns = with_duckdb_fallback { EventDdb.top_asns(@start_time, 15) } ||
|
||||
Event.where("timestamp >= ? AND asn IS NOT NULL", @start_time)
|
||||
.group(:asn, :asn_org)
|
||||
.select("asn, asn_org, COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips, COUNT(DISTINCT network_range_id) as network_count")
|
||||
.order("event_count DESC")
|
||||
.limit(15)
|
||||
|
||||
# Geographic breakdown (using denormalized country column)
|
||||
@top_countries = Event.where("timestamp >= ? AND country IS NOT NULL", @start_time)
|
||||
# Geographic breakdown - use DuckDB if available
|
||||
@top_countries = with_duckdb_fallback { EventDdb.top_countries_with_stats(@start_time, 15) } ||
|
||||
Event.where("timestamp >= ? AND country IS NOT NULL", @start_time)
|
||||
.group(:country)
|
||||
.select("country, COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips")
|
||||
.order("event_count DESC")
|
||||
@@ -191,12 +246,15 @@ class AnalyticsController < ApplicationController
|
||||
# Historical hours are cached for full TTL, current hour cached briefly for freshness
|
||||
|
||||
# Cache historical hours (1-23 hours ago) - these are complete and won't change
|
||||
# No expiration - will stick around until evicted by cache store
|
||||
# No expiration - will stick around until evicted by cache store (uses DuckDB if available)
|
||||
historical_timeline = Rails.cache.fetch("#{cache_key_base}/chart_historical") do
|
||||
historical_start = 23.hours.ago.beginning_of_hour
|
||||
events_by_hour = Event.where("timestamp >= ? AND timestamp < ?", historical_start, Time.current.beginning_of_hour)
|
||||
.group("DATE_TRUNC('hour', timestamp)")
|
||||
.count
|
||||
current_hour_start = Time.current.beginning_of_hour
|
||||
|
||||
events_by_hour = with_duckdb_fallback { EventDdb.hourly_timeline(historical_start, current_hour_start) } ||
|
||||
Event.where("timestamp >= ? AND timestamp < ?", historical_start, current_hour_start)
|
||||
.group("DATE_TRUNC('hour', timestamp)")
|
||||
.count
|
||||
|
||||
(1..23).map do |hour_ago|
|
||||
hour_time = hour_ago.hours.ago.beginning_of_hour
|
||||
@@ -209,6 +267,7 @@ class AnalyticsController < ApplicationController
|
||||
end
|
||||
|
||||
# Current hour (0 hours ago) - cache very briefly since it's actively accumulating
|
||||
# ALWAYS use PostgreSQL for current hour to get real-time data (DuckDB syncs every minute)
|
||||
current_hour_data = Rails.cache.fetch("#{cache_key_base}/chart_current_hour", expires_in: 1.minute) do
|
||||
hour_time = Time.current.beginning_of_hour
|
||||
count = Event.where("timestamp >= ?", hour_time).count
|
||||
@@ -290,6 +349,12 @@ class AnalyticsController < ApplicationController
|
||||
end
|
||||
|
||||
def calculate_network_type_stats(start_time)
|
||||
# Try DuckDB first, fallback to PostgreSQL
|
||||
duckdb_stats = with_duckdb_fallback { EventDdb.network_type_stats(start_time) }
|
||||
|
||||
return duckdb_stats if duckdb_stats
|
||||
|
||||
# PostgreSQL fallback
|
||||
# Get all network types with their traffic statistics using denormalized columns
|
||||
network_types = [
|
||||
{ type: 'datacenter', label: 'Datacenter', column: :is_datacenter },
|
||||
@@ -333,6 +398,12 @@ class AnalyticsController < ApplicationController
|
||||
end
|
||||
|
||||
def calculate_suspicious_patterns(start_time)
|
||||
# Try DuckDB first, fallback to PostgreSQL
|
||||
duckdb_patterns = with_duckdb_fallback { EventDdb.suspicious_patterns(start_time) }
|
||||
|
||||
return duckdb_patterns if duckdb_patterns
|
||||
|
||||
# PostgreSQL fallback
|
||||
patterns = {}
|
||||
|
||||
# High volume networks (top 1% by request count) - using denormalized network_range_id
|
||||
@@ -358,9 +429,9 @@ class AnalyticsController < ApplicationController
|
||||
high_deny_networks = Event.where("timestamp >= ? AND network_range_id IS NOT NULL", start_time)
|
||||
.group(:network_range_id)
|
||||
.select("network_range_id,
|
||||
COUNT(CASE WHEN waf_action = 1 THEN 1 END) as denied_count,
|
||||
COUNT(CASE WHEN waf_action = 0 THEN 1 END) as denied_count,
|
||||
COUNT(*) as total_count")
|
||||
.having("COUNT(CASE WHEN waf_action = 1 THEN 1 END)::float / COUNT(*) > 0.5")
|
||||
.having("COUNT(CASE WHEN waf_action = 0 THEN 1 END)::float / COUNT(*) > 0.5")
|
||||
.having("COUNT(*) >= 10") # minimum threshold
|
||||
|
||||
patterns[:high_deny_rate] = {
|
||||
@@ -392,12 +463,14 @@ class AnalyticsController < ApplicationController
|
||||
{
|
||||
id: network.id,
|
||||
cidr: network.cidr,
|
||||
company: network.company,
|
||||
company: network.display_company,
|
||||
asn: network.asn,
|
||||
country: network.country,
|
||||
country: network.display_country,
|
||||
network_type: network.network_type,
|
||||
event_count: network.event_count,
|
||||
unique_ips: network.unique_ips
|
||||
unique_ips: network.unique_ips,
|
||||
has_inherited_data: network.has_inherited_data?,
|
||||
inherited_from: network.inherited_from
|
||||
}
|
||||
},
|
||||
network_breakdown: @network_breakdown,
|
||||
@@ -449,4 +522,27 @@ class AnalyticsController < ApplicationController
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
# Helper method to try DuckDB first, fall back to PostgreSQL
|
||||
def with_duckdb_fallback(&block)
|
||||
result = yield
|
||||
result.nil? ? nil : result # Return result or nil to trigger fallback
|
||||
rescue StandardError => e
|
||||
Rails.logger.warn "[Analytics] DuckDB query failed, falling back to PostgreSQL: #{e.message}"
|
||||
nil # Return nil to trigger fallback
|
||||
end
|
||||
|
||||
# Check if DuckDB has recent data (within last 2 minutes)
|
||||
# Returns true if DuckDB is up-to-date, false if potentially stale
|
||||
def duckdb_is_fresh?
|
||||
newest = AnalyticsDuckdbService.instance.newest_event_timestamp
|
||||
return false if newest.nil?
|
||||
|
||||
# Consider fresh if newest event is within 2 minutes
|
||||
# (sync job runs every 1 minute, so 2 minutes allows for some lag)
|
||||
newest >= 2.minutes.ago
|
||||
rescue StandardError => e
|
||||
Rails.logger.warn "[Analytics] Error checking DuckDB freshness: #{e.message}"
|
||||
false
|
||||
end
|
||||
end
|
||||
126
app/controllers/bot_network_ranges_controller.rb
Normal file
126
app/controllers/bot_network_ranges_controller.rb
Normal file
@@ -0,0 +1,126 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class BotNetworkRangesController < ApplicationController
|
||||
before_action :authenticate_user!
|
||||
before_action :require_admin
|
||||
|
||||
def index
|
||||
@bot_sources = BotNetworkRangeImporter::BOT_SOURCES
|
||||
@recent_imports = DataImport.where(import_type: 'bot_network_ranges').order(created_at: :desc).limit(10)
|
||||
@bot_network_ranges = NetworkRange.where("source LIKE 'bot_import_%'").order(created_at: :desc).limit(50)
|
||||
end
|
||||
|
||||
def import
|
||||
source_key = params[:source]
|
||||
options = import_options
|
||||
|
||||
if source_key.present?
|
||||
# Perform import synchronously for immediate feedback
|
||||
begin
|
||||
result = BotNetworkRangeImporter.import_from_source(source_key, options)
|
||||
|
||||
# Create a data import record
|
||||
DataImport.create!(
|
||||
import_type: 'bot_network_ranges',
|
||||
source: source_key.to_s,
|
||||
status: 'completed',
|
||||
records_processed: result[:imported],
|
||||
notes: "Imported from #{result[:source]}: #{result[:note] || 'Success'}"
|
||||
)
|
||||
|
||||
flash[:notice] = "Successfully imported #{result[:imported]} ranges from #{result[:source]}"
|
||||
rescue => e
|
||||
flash[:alert] = "Failed to import from #{source_key}: #{e.message}"
|
||||
end
|
||||
else
|
||||
flash[:alert] = "Please select a source to import from"
|
||||
end
|
||||
|
||||
redirect_to bot_network_ranges_path
|
||||
end
|
||||
|
||||
def import_async
|
||||
source_key = params[:source]
|
||||
options = import_options
|
||||
|
||||
if source_key.present?
|
||||
# Create a data import record for tracking
|
||||
data_import = DataImport.create!(
|
||||
import_type: 'bot_network_ranges',
|
||||
source: source_key.to_s,
|
||||
status: 'pending',
|
||||
records_processed: 0,
|
||||
notes: "Import job queued for #{source_key}"
|
||||
)
|
||||
|
||||
# Queue the background job
|
||||
ImportBotNetworkRangesJob.perform_later(source_key, options.merge(data_import_id: data_import.id))
|
||||
|
||||
flash[:notice] = "Import job queued for #{source_key}. You'll be notified when it's complete."
|
||||
else
|
||||
flash[:alert] = "Please select a source to import from"
|
||||
end
|
||||
|
||||
redirect_to bot_network_ranges_path
|
||||
end
|
||||
|
||||
def import_all
|
||||
options = import_options
|
||||
|
||||
# Create a data import record for batch import
|
||||
data_import = DataImport.create!(
|
||||
import_type: 'bot_network_ranges',
|
||||
source: 'all_sources',
|
||||
status: 'pending',
|
||||
records_processed: 0,
|
||||
notes: "Batch import job queued for all available sources"
|
||||
)
|
||||
|
||||
# Queue the batch import job
|
||||
ImportAllBotNetworkRangesJob.perform_later(options.merge(data_import_id: data_import.id))
|
||||
|
||||
flash[:notice] = "Batch import job queued for all sources. This may take several minutes."
|
||||
redirect_to bot_network_ranges_path
|
||||
end
|
||||
|
||||
def show
|
||||
@network_ranges = NetworkRange.where("source LIKE 'bot_import_#{params[:source]}%'")
|
||||
.order(created_at: :desc)
|
||||
.page(params[:page])
|
||||
.per(50)
|
||||
|
||||
@source_name = BotNetworkRangeImporter::BOT_SOURCES[params[:source].to_sym]&.dig(:name) || params[:source]
|
||||
@import_stats = NetworkRange.where("source LIKE 'bot_import_#{params[:source]}%'")
|
||||
.group(:source)
|
||||
.count
|
||||
end
|
||||
|
||||
def destroy
|
||||
source = params[:source]
|
||||
deleted_count = NetworkRange.where("source LIKE 'bot_import_#{source}%'").delete_all
|
||||
|
||||
flash[:notice] = "Deleted #{deleted_count} network ranges from #{source}"
|
||||
redirect_to bot_network_ranges_path
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def require_admin
|
||||
redirect_to root_path, alert: 'Admin access required' unless current_user&.admin?
|
||||
end
|
||||
|
||||
def import_options
|
||||
options = {}
|
||||
|
||||
# AWS-specific options
|
||||
if params[:aws_services].present?
|
||||
options[:aws_services] = params[:aws_services].split(',').map(&:strip)
|
||||
end
|
||||
|
||||
# Batch size control
|
||||
options[:batch_size] = params[:batch_size].to_i if params[:batch_size].present?
|
||||
options[:batch_size] = 1000 if options[:batch_size].zero?
|
||||
|
||||
options
|
||||
end
|
||||
end
|
||||
@@ -46,8 +46,10 @@ class NetworkRangesController < ApplicationController
|
||||
authorize @network_range
|
||||
|
||||
if @network_range.persisted?
|
||||
# Real network - use direct IP containment for consistency with stats
|
||||
events_scope = Event.where("ip_address <<= ?", @network_range.cidr).recent
|
||||
# Real network - use indexed network_range_id for much better performance
|
||||
# Include child network ranges to capture all traffic within this network block
|
||||
network_ids = [@network_range.id] + @network_range.child_ranges.pluck(:id)
|
||||
events_scope = Event.where(network_range_id: network_ids).recent
|
||||
else
|
||||
# Virtual network - find events by IP range containment
|
||||
events_scope = Event.where("ip_address <<= ?::inet", @network_range.to_s).recent
|
||||
@@ -58,22 +60,24 @@ class NetworkRangesController < ApplicationController
|
||||
|
||||
@child_ranges = @network_range.child_ranges.limit(20)
|
||||
@parent_ranges = @network_range.parent_ranges.limit(10)
|
||||
@associated_rules = @network_range.persisted? ? @network_range.rules.includes(:user).order(created_at: :desc) : []
|
||||
@associated_rules = @network_range.persisted? ? @network_range.rules.includes(:user, :network_range, :waf_policy).order(created_at: :desc) : []
|
||||
|
||||
# Load rules from supernets and subnets
|
||||
@supernet_rules = @network_range.persisted? ? @network_range.supernet_rules.includes(:network_range, :user).limit(10) : []
|
||||
@subnet_rules = @network_range.persisted? ? @network_range.child_rules.includes(:network_range, :user).limit(20) : []
|
||||
@supernet_rules = @network_range.persisted? ? @network_range.supernet_rules.includes(:network_range, :user, :waf_policy).limit(10) : []
|
||||
@subnet_rules = @network_range.persisted? ? @network_range.child_rules.includes(:network_range, :user, :waf_policy).limit(20) : []
|
||||
|
||||
# Traffic analytics (if we have events)
|
||||
@traffic_stats = calculate_traffic_stats(@network_range)
|
||||
|
||||
# Check if we have IPAPI data (or if parent has it)
|
||||
# Check if we have IPAPI data (or if parent has it) - cache expensive parent lookup
|
||||
@has_ipapi_data = @network_range.has_network_data_from?(:ipapi)
|
||||
@parent_with_ipapi = nil
|
||||
|
||||
unless @has_ipapi_data
|
||||
# Check if parent has IPAPI data
|
||||
parent = @network_range.parent_with_intelligence
|
||||
# Cache expensive parent intelligence lookup
|
||||
parent = Rails.cache.fetch("network_parent_intel:#{@network_range.cache_key}", expires_in: 1.hour) do
|
||||
@network_range.parent_with_intelligence
|
||||
end
|
||||
if parent&.has_network_data_from?(:ipapi)
|
||||
@parent_with_ipapi = parent
|
||||
@has_ipapi_data = true
|
||||
@@ -194,6 +198,15 @@ class NetworkRangesController < ApplicationController
|
||||
|
||||
private
|
||||
|
||||
# Helper method to try DuckDB first, fall back to PostgreSQL
|
||||
def with_duckdb_fallback(&block)
|
||||
result = yield
|
||||
result.nil? ? nil : result # Return result or nil to trigger fallback
|
||||
rescue StandardError => e
|
||||
Rails.logger.warn "[NetworkRanges] DuckDB query failed, falling back to PostgreSQL: #{e.message}"
|
||||
nil # Return nil to trigger fallback
|
||||
end
|
||||
|
||||
def set_network_range
|
||||
# Handle CIDR slugs (e.g., "40.77.167.100_32" -> "40.77.167.100/32")
|
||||
cidr = params[:id].gsub('_', '/')
|
||||
@@ -248,27 +261,37 @@ class NetworkRangesController < ApplicationController
|
||||
# Use indexed network_range_id for much better performance instead of expensive CIDR operator
|
||||
# Include child network ranges to capture all traffic within this network block
|
||||
network_ids = [network_range.id] + network_range.child_ranges.pluck(:id)
|
||||
base_query = Event.where(network_range_id: network_ids)
|
||||
|
||||
# Use separate queries: one for grouping (without ordering), one for recent activity (with ordering)
|
||||
events_for_grouping = base_query.limit(1000)
|
||||
events_for_activity = base_query.recent.limit(20)
|
||||
# Try DuckDB first for stats (much faster)
|
||||
duckdb_stats = with_duckdb_fallback { EventDdb.network_traffic_stats(network_ids) }
|
||||
duckdb_top_paths = with_duckdb_fallback { EventDdb.network_top_paths(network_ids, 10) }
|
||||
duckdb_top_agents = with_duckdb_fallback { EventDdb.network_top_user_agents(network_ids, 5) }
|
||||
|
||||
# Calculate counts properly - use consistent base_query for all counts
|
||||
total_requests = base_query.count
|
||||
unique_ips = base_query.except(:order).distinct.count(:ip_address)
|
||||
blocked_requests = base_query.blocked.count
|
||||
allowed_requests = base_query.allowed.count
|
||||
if duckdb_stats
|
||||
# DuckDB success - use fast aggregated stats
|
||||
stats = duckdb_stats.merge(
|
||||
top_paths: duckdb_top_paths&.to_h || {},
|
||||
top_user_agents: duckdb_top_agents&.to_h || {},
|
||||
recent_activity: Event.where(network_range_id: network_ids).recent.limit(20)
|
||||
)
|
||||
else
|
||||
# PostgreSQL fallback
|
||||
base_query = Event.where(network_range_id: network_ids)
|
||||
events_for_grouping = base_query.limit(1000)
|
||||
events_for_activity = base_query.recent.limit(20)
|
||||
|
||||
{
|
||||
total_requests: total_requests,
|
||||
unique_ips: unique_ips,
|
||||
blocked_requests: blocked_requests,
|
||||
allowed_requests: allowed_requests,
|
||||
top_paths: events_for_grouping.group(:request_path).count.sort_by { |_, count| -count }.first(10),
|
||||
top_user_agents: events_for_grouping.group(:user_agent).count.sort_by { |_, count| -count }.first(5),
|
||||
recent_activity: events_for_activity
|
||||
}
|
||||
stats = {
|
||||
total_requests: base_query.count,
|
||||
unique_ips: base_query.except(:order).distinct.count(:ip_address),
|
||||
blocked_requests: base_query.blocked.count,
|
||||
allowed_requests: base_query.allowed.count,
|
||||
top_paths: events_for_grouping.group(:request_path).count.sort_by { |_, count| -count }.first(10).to_h,
|
||||
top_user_agents: events_for_grouping.group(:user_agent).count.sort_by { |_, count| -count }.first(5).to_h,
|
||||
recent_activity: events_for_activity
|
||||
}
|
||||
end
|
||||
|
||||
stats
|
||||
else
|
||||
# No events - return empty stats
|
||||
{
|
||||
@@ -296,8 +319,8 @@ class NetworkRangesController < ApplicationController
|
||||
unique_ips: base_query.except(:order).distinct.count(:ip_address),
|
||||
blocked_requests: base_query.blocked.count,
|
||||
allowed_requests: base_query.allowed.count,
|
||||
top_paths: events_for_grouping.group(:request_path).count.sort_by { |_, count| -count }.first(10),
|
||||
top_user_agents: events_for_grouping.group(:user_agent).count.sort_by { |_, count| -count }.first(5),
|
||||
top_paths: events_for_grouping.group(:request_path).count.sort_by { |_, count| -count }.first(10).to_h,
|
||||
top_user_agents: events_for_grouping.group(:user_agent).count.sort_by { |_, count| -count }.first(5).to_h,
|
||||
recent_activity: events_for_activity
|
||||
}
|
||||
else
|
||||
|
||||
@@ -46,12 +46,9 @@ class RulesController < ApplicationController
|
||||
process_quick_create_parameters
|
||||
|
||||
# Handle network range creation if CIDR is provided
|
||||
if params[:cidr].present? && @rule.network_rule?
|
||||
network_range = NetworkRange.find_or_create_by(cidr: params[:cidr]) do |range|
|
||||
range.user = Current.user
|
||||
range.source = 'manual'
|
||||
range.creation_reason = "Created for rule ##{@rule.id}"
|
||||
end
|
||||
cidr_param = params[:new_cidr].presence || params[:cidr].presence
|
||||
if cidr_param.present? && @rule.network_rule?
|
||||
network_range = NetworkRange.find_or_create_by_cidr(cidr_param, user: Current.user, source: 'manual')
|
||||
@rule.network_range = network_range
|
||||
end
|
||||
|
||||
@@ -132,7 +129,9 @@ class RulesController < ApplicationController
|
||||
:expires_at,
|
||||
:enabled,
|
||||
:source,
|
||||
:network_range_id
|
||||
:network_range_id,
|
||||
:header_name,
|
||||
:header_value
|
||||
]
|
||||
|
||||
# Only include conditions for non-network rules
|
||||
@@ -250,15 +249,24 @@ def process_quick_create_parameters
|
||||
})
|
||||
end
|
||||
|
||||
# Parse metadata if it's a string that looks like JSON
|
||||
if @rule.metadata.is_a?(String) && @rule.metadata.starts_with?('{')
|
||||
# Parse metadata textarea first if it's JSON
|
||||
if @rule.metadata.is_a?(String) && @rule.metadata.present? && @rule.metadata.starts_with?('{')
|
||||
begin
|
||||
@rule.metadata = JSON.parse(@rule.metadata)
|
||||
rescue JSON::ParserError
|
||||
# Keep as string if not valid JSON
|
||||
# Keep as string if not valid JSON - will be caught by validation
|
||||
end
|
||||
end
|
||||
|
||||
# Ensure metadata is a hash
|
||||
@rule.metadata = {} unless @rule.metadata.is_a?(Hash)
|
||||
|
||||
# Handle add_header fields - use provided params or existing metadata values
|
||||
if @rule.add_header_action? && (params[:header_name].present? || params[:header_value].present?)
|
||||
@rule.metadata['header_name'] = params[:header_name].presence || @rule.metadata['header_name'] || 'X-Bot-Agent'
|
||||
@rule.metadata['header_value'] = params[:header_value].presence || @rule.metadata['header_value'] || 'Unknown'
|
||||
end
|
||||
|
||||
# Handle expires_at parsing for text input
|
||||
if params.dig(:rule, :expires_at).present?
|
||||
expires_at_str = params[:rule][:expires_at].strip
|
||||
|
||||
Reference in New Issue
Block a user