Fix some blocked/allow laggards after migrating. Add DuckDB for outstanding analyitcs performance. Start adding an import for all bot networks

This commit is contained in:
Dan Milne
2025-11-18 16:40:05 +11:00
parent ef56779584
commit 3f274c842c
37 changed files with 3522 additions and 151 deletions

View File

@@ -23,9 +23,10 @@ class AnalyticsController < ApplicationController
# Cache key includes period and start_time (hour-aligned for consistency)
cache_key_base = "analytics/#{@time_period}/#{@start_time.to_i}"
# Core statistics - cached
# Core statistics - cached (uses DuckDB if available)
@total_events = Rails.cache.fetch("#{cache_key_base}/total_events", expires_in: cache_ttl) do
Event.where("timestamp >= ?", @start_time).count
with_duckdb_fallback { EventDdb.count_since(@start_time) } ||
Event.where("timestamp >= ?", @start_time).count
end
@total_rules = Rails.cache.fetch("analytics/total_rules", expires_in: 5.minutes) do
@@ -40,31 +41,33 @@ class AnalyticsController < ApplicationController
NetworkRange.count
end
# Event breakdown by action - cached
# Event breakdown by action - cached (uses DuckDB if available)
@event_breakdown = Rails.cache.fetch("#{cache_key_base}/event_breakdown", expires_in: cache_ttl) do
Event.where("timestamp >= ?", @start_time)
.group(:waf_action)
.count
# Keys are already strings ("allow", "deny", etc.) from the enum
with_duckdb_fallback { EventDdb.breakdown_by_action(@start_time) } ||
Event.where("timestamp >= ?", @start_time)
.group(:waf_action)
.count
end
# Top countries by event count - cached (now uses denormalized country column)
# Top countries by event count - cached (uses DuckDB if available)
@top_countries = Rails.cache.fetch("#{cache_key_base}/top_countries", expires_in: cache_ttl) do
Event.where("timestamp >= ? AND country IS NOT NULL", @start_time)
.group(:country)
.count
.sort_by { |_, count| -count }
.first(10)
with_duckdb_fallback { EventDdb.top_countries(@start_time, 10) } ||
Event.where("timestamp >= ? AND country IS NOT NULL", @start_time)
.group(:country)
.count
.sort_by { |_, count| -count }
.first(10)
end
# Top blocked IPs - cached
# Top blocked IPs - cached (uses DuckDB if available)
@top_blocked_ips = Rails.cache.fetch("#{cache_key_base}/top_blocked_ips", expires_in: cache_ttl) do
Event.where("timestamp >= ?", @start_time)
.where(waf_action: 1) # deny action in enum
.group(:ip_address)
.count
.sort_by { |_, count| -count }
.first(10)
with_duckdb_fallback { EventDdb.top_blocked_ips(@start_time, 10) } ||
Event.where("timestamp >= ?", @start_time)
.where(waf_action: 0) # deny action in enum
.group(:ip_address)
.count
.sort_by { |_, count| -count }
.first(10)
end
# Network range intelligence breakdown - cached
@@ -92,7 +95,7 @@ class AnalyticsController < ApplicationController
total_users: User.count,
active_rules: Rule.enabled.count,
disabled_rules: Rule.where(enabled: false).count,
recent_errors: Event.where("timestamp >= ? AND waf_action = ?", @start_time, 1).count # 1 = deny
recent_errors: Event.where("timestamp >= ? AND waf_action = ?", @start_time, 0).count # 0 = deny
}
end
@@ -117,38 +120,90 @@ class AnalyticsController < ApplicationController
@time_period = params[:period]&.to_sym || :day
@start_time = calculate_start_time(@time_period)
# Top networks by request volume (using denormalized network_range_id)
# Use a subquery approach to avoid PostgreSQL GROUP BY issues with network_ranges.*
event_stats = Event.where("timestamp >= ?", @start_time)
.where.not(network_range_id: nil)
.group(:network_range_id)
.select("network_range_id, COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips")
# Top networks by request volume - use DuckDB if available
network_stats = with_duckdb_fallback { EventDdb.top_networks(@start_time, 50) }
# Join the stats back to NetworkRange to get full network details
@top_networks = NetworkRange.joins("INNER JOIN (#{event_stats.to_sql}) stats ON stats.network_range_id = network_ranges.id")
.select("network_ranges.*, stats.event_count, stats.unique_ips")
.order("stats.event_count DESC")
.limit(50)
if network_stats
# DuckDB path: array format [network_range_id, event_count, unique_ips]
network_ids = network_stats.map { |row| row[0] }
stats_by_id = network_stats.to_h { |row| [row[0], { event_count: row[1], unique_ips: row[2] }] }
@top_networks = NetworkRange.where(id: network_ids)
.to_a
.map do |network|
stats = stats_by_id[network.id]
network.define_singleton_method(:event_count) { stats[:event_count] }
network.define_singleton_method(:unique_ips) { stats[:unique_ips] }
# Add inherited intelligence support
intelligence = network.inherited_intelligence
if intelligence[:inherited]
network.define_singleton_method(:display_company) { intelligence[:company] }
network.define_singleton_method(:display_country) { intelligence[:country] }
network.define_singleton_method(:inherited_from) { intelligence[:parent_cidr] }
network.define_singleton_method(:has_inherited_data?) { true }
else
network.define_singleton_method(:display_company) { network.company }
network.define_singleton_method(:display_country) { network.country }
network.define_singleton_method(:inherited_from) { nil }
network.define_singleton_method(:has_inherited_data?) { false }
end
network
end
.sort_by { |n| -n.event_count }
else
# PostgreSQL fallback
event_stats = Event.where("timestamp >= ?", @start_time)
.where.not(network_range_id: nil)
.group(:network_range_id)
.select("network_range_id, COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips")
@top_networks = NetworkRange.joins("INNER JOIN (#{event_stats.to_sql}) stats ON stats.network_range_id = network_ranges.id")
.select("network_ranges.*, stats.event_count, stats.unique_ips")
.order("stats.event_count DESC")
.limit(50)
# Add inherited intelligence support for PostgreSQL fallback
@top_networks = @top_networks.to_a.map do |network|
intelligence = network.inherited_intelligence
if intelligence[:inherited]
network.define_singleton_method(:display_company) { intelligence[:company] }
network.define_singleton_method(:display_country) { intelligence[:country] }
network.define_singleton_method(:inherited_from) { intelligence[:parent_cidr] }
network.define_singleton_method(:has_inherited_data?) { true }
else
network.define_singleton_method(:display_company) { network.company }
network.define_singleton_method(:display_country) { network.country }
network.define_singleton_method(:inherited_from) { nil }
network.define_singleton_method(:has_inherited_data?) { false }
end
network
end
end
# Network type breakdown with traffic stats
@network_breakdown = calculate_network_type_stats(@start_time)
# Company breakdown for top traffic sources (using denormalized company column)
@top_companies = Event.where("timestamp >= ? AND company IS NOT NULL", @start_time)
# Company breakdown for top traffic sources - use DuckDB if available
@top_companies = with_duckdb_fallback { EventDdb.top_companies(@start_time, 20) } ||
Event.where("timestamp >= ? AND company IS NOT NULL", @start_time)
.group(:company)
.select("company, COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips, COUNT(DISTINCT network_range_id) as network_count")
.order("event_count DESC")
.limit(20)
# ASN breakdown (using denormalized asn columns)
@top_asns = Event.where("timestamp >= ? AND asn IS NOT NULL", @start_time)
# ASN breakdown - use DuckDB if available
@top_asns = with_duckdb_fallback { EventDdb.top_asns(@start_time, 15) } ||
Event.where("timestamp >= ? AND asn IS NOT NULL", @start_time)
.group(:asn, :asn_org)
.select("asn, asn_org, COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips, COUNT(DISTINCT network_range_id) as network_count")
.order("event_count DESC")
.limit(15)
# Geographic breakdown (using denormalized country column)
@top_countries = Event.where("timestamp >= ? AND country IS NOT NULL", @start_time)
# Geographic breakdown - use DuckDB if available
@top_countries = with_duckdb_fallback { EventDdb.top_countries_with_stats(@start_time, 15) } ||
Event.where("timestamp >= ? AND country IS NOT NULL", @start_time)
.group(:country)
.select("country, COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips")
.order("event_count DESC")
@@ -191,12 +246,15 @@ class AnalyticsController < ApplicationController
# Historical hours are cached for full TTL, current hour cached briefly for freshness
# Cache historical hours (1-23 hours ago) - these are complete and won't change
# No expiration - will stick around until evicted by cache store
# No expiration - will stick around until evicted by cache store (uses DuckDB if available)
historical_timeline = Rails.cache.fetch("#{cache_key_base}/chart_historical") do
historical_start = 23.hours.ago.beginning_of_hour
events_by_hour = Event.where("timestamp >= ? AND timestamp < ?", historical_start, Time.current.beginning_of_hour)
.group("DATE_TRUNC('hour', timestamp)")
.count
current_hour_start = Time.current.beginning_of_hour
events_by_hour = with_duckdb_fallback { EventDdb.hourly_timeline(historical_start, current_hour_start) } ||
Event.where("timestamp >= ? AND timestamp < ?", historical_start, current_hour_start)
.group("DATE_TRUNC('hour', timestamp)")
.count
(1..23).map do |hour_ago|
hour_time = hour_ago.hours.ago.beginning_of_hour
@@ -209,6 +267,7 @@ class AnalyticsController < ApplicationController
end
# Current hour (0 hours ago) - cache very briefly since it's actively accumulating
# ALWAYS use PostgreSQL for current hour to get real-time data (DuckDB syncs every minute)
current_hour_data = Rails.cache.fetch("#{cache_key_base}/chart_current_hour", expires_in: 1.minute) do
hour_time = Time.current.beginning_of_hour
count = Event.where("timestamp >= ?", hour_time).count
@@ -290,6 +349,12 @@ class AnalyticsController < ApplicationController
end
def calculate_network_type_stats(start_time)
# Try DuckDB first, fallback to PostgreSQL
duckdb_stats = with_duckdb_fallback { EventDdb.network_type_stats(start_time) }
return duckdb_stats if duckdb_stats
# PostgreSQL fallback
# Get all network types with their traffic statistics using denormalized columns
network_types = [
{ type: 'datacenter', label: 'Datacenter', column: :is_datacenter },
@@ -333,6 +398,12 @@ class AnalyticsController < ApplicationController
end
def calculate_suspicious_patterns(start_time)
# Try DuckDB first, fallback to PostgreSQL
duckdb_patterns = with_duckdb_fallback { EventDdb.suspicious_patterns(start_time) }
return duckdb_patterns if duckdb_patterns
# PostgreSQL fallback
patterns = {}
# High volume networks (top 1% by request count) - using denormalized network_range_id
@@ -358,9 +429,9 @@ class AnalyticsController < ApplicationController
high_deny_networks = Event.where("timestamp >= ? AND network_range_id IS NOT NULL", start_time)
.group(:network_range_id)
.select("network_range_id,
COUNT(CASE WHEN waf_action = 1 THEN 1 END) as denied_count,
COUNT(CASE WHEN waf_action = 0 THEN 1 END) as denied_count,
COUNT(*) as total_count")
.having("COUNT(CASE WHEN waf_action = 1 THEN 1 END)::float / COUNT(*) > 0.5")
.having("COUNT(CASE WHEN waf_action = 0 THEN 1 END)::float / COUNT(*) > 0.5")
.having("COUNT(*) >= 10") # minimum threshold
patterns[:high_deny_rate] = {
@@ -392,12 +463,14 @@ class AnalyticsController < ApplicationController
{
id: network.id,
cidr: network.cidr,
company: network.company,
company: network.display_company,
asn: network.asn,
country: network.country,
country: network.display_country,
network_type: network.network_type,
event_count: network.event_count,
unique_ips: network.unique_ips
unique_ips: network.unique_ips,
has_inherited_data: network.has_inherited_data?,
inherited_from: network.inherited_from
}
},
network_breakdown: @network_breakdown,
@@ -449,4 +522,27 @@ class AnalyticsController < ApplicationController
}
end
end
# Helper method to try DuckDB first, fall back to PostgreSQL
def with_duckdb_fallback(&block)
result = yield
result.nil? ? nil : result # Return result or nil to trigger fallback
rescue StandardError => e
Rails.logger.warn "[Analytics] DuckDB query failed, falling back to PostgreSQL: #{e.message}"
nil # Return nil to trigger fallback
end
# Check if DuckDB has recent data (within last 2 minutes)
# Returns true if DuckDB is up-to-date, false if potentially stale
def duckdb_is_fresh?
newest = AnalyticsDuckdbService.instance.newest_event_timestamp
return false if newest.nil?
# Consider fresh if newest event is within 2 minutes
# (sync job runs every 1 minute, so 2 minutes allows for some lag)
newest >= 2.minutes.ago
rescue StandardError => e
Rails.logger.warn "[Analytics] Error checking DuckDB freshness: #{e.message}"
false
end
end

View File

@@ -0,0 +1,126 @@
# frozen_string_literal: true
class BotNetworkRangesController < ApplicationController
before_action :authenticate_user!
before_action :require_admin
def index
@bot_sources = BotNetworkRangeImporter::BOT_SOURCES
@recent_imports = DataImport.where(import_type: 'bot_network_ranges').order(created_at: :desc).limit(10)
@bot_network_ranges = NetworkRange.where("source LIKE 'bot_import_%'").order(created_at: :desc).limit(50)
end
def import
source_key = params[:source]
options = import_options
if source_key.present?
# Perform import synchronously for immediate feedback
begin
result = BotNetworkRangeImporter.import_from_source(source_key, options)
# Create a data import record
DataImport.create!(
import_type: 'bot_network_ranges',
source: source_key.to_s,
status: 'completed',
records_processed: result[:imported],
notes: "Imported from #{result[:source]}: #{result[:note] || 'Success'}"
)
flash[:notice] = "Successfully imported #{result[:imported]} ranges from #{result[:source]}"
rescue => e
flash[:alert] = "Failed to import from #{source_key}: #{e.message}"
end
else
flash[:alert] = "Please select a source to import from"
end
redirect_to bot_network_ranges_path
end
def import_async
source_key = params[:source]
options = import_options
if source_key.present?
# Create a data import record for tracking
data_import = DataImport.create!(
import_type: 'bot_network_ranges',
source: source_key.to_s,
status: 'pending',
records_processed: 0,
notes: "Import job queued for #{source_key}"
)
# Queue the background job
ImportBotNetworkRangesJob.perform_later(source_key, options.merge(data_import_id: data_import.id))
flash[:notice] = "Import job queued for #{source_key}. You'll be notified when it's complete."
else
flash[:alert] = "Please select a source to import from"
end
redirect_to bot_network_ranges_path
end
def import_all
options = import_options
# Create a data import record for batch import
data_import = DataImport.create!(
import_type: 'bot_network_ranges',
source: 'all_sources',
status: 'pending',
records_processed: 0,
notes: "Batch import job queued for all available sources"
)
# Queue the batch import job
ImportAllBotNetworkRangesJob.perform_later(options.merge(data_import_id: data_import.id))
flash[:notice] = "Batch import job queued for all sources. This may take several minutes."
redirect_to bot_network_ranges_path
end
def show
@network_ranges = NetworkRange.where("source LIKE 'bot_import_#{params[:source]}%'")
.order(created_at: :desc)
.page(params[:page])
.per(50)
@source_name = BotNetworkRangeImporter::BOT_SOURCES[params[:source].to_sym]&.dig(:name) || params[:source]
@import_stats = NetworkRange.where("source LIKE 'bot_import_#{params[:source]}%'")
.group(:source)
.count
end
def destroy
source = params[:source]
deleted_count = NetworkRange.where("source LIKE 'bot_import_#{source}%'").delete_all
flash[:notice] = "Deleted #{deleted_count} network ranges from #{source}"
redirect_to bot_network_ranges_path
end
private
def require_admin
redirect_to root_path, alert: 'Admin access required' unless current_user&.admin?
end
def import_options
options = {}
# AWS-specific options
if params[:aws_services].present?
options[:aws_services] = params[:aws_services].split(',').map(&:strip)
end
# Batch size control
options[:batch_size] = params[:batch_size].to_i if params[:batch_size].present?
options[:batch_size] = 1000 if options[:batch_size].zero?
options
end
end

View File

@@ -46,8 +46,10 @@ class NetworkRangesController < ApplicationController
authorize @network_range
if @network_range.persisted?
# Real network - use direct IP containment for consistency with stats
events_scope = Event.where("ip_address <<= ?", @network_range.cidr).recent
# Real network - use indexed network_range_id for much better performance
# Include child network ranges to capture all traffic within this network block
network_ids = [@network_range.id] + @network_range.child_ranges.pluck(:id)
events_scope = Event.where(network_range_id: network_ids).recent
else
# Virtual network - find events by IP range containment
events_scope = Event.where("ip_address <<= ?::inet", @network_range.to_s).recent
@@ -58,22 +60,24 @@ class NetworkRangesController < ApplicationController
@child_ranges = @network_range.child_ranges.limit(20)
@parent_ranges = @network_range.parent_ranges.limit(10)
@associated_rules = @network_range.persisted? ? @network_range.rules.includes(:user).order(created_at: :desc) : []
@associated_rules = @network_range.persisted? ? @network_range.rules.includes(:user, :network_range, :waf_policy).order(created_at: :desc) : []
# Load rules from supernets and subnets
@supernet_rules = @network_range.persisted? ? @network_range.supernet_rules.includes(:network_range, :user).limit(10) : []
@subnet_rules = @network_range.persisted? ? @network_range.child_rules.includes(:network_range, :user).limit(20) : []
@supernet_rules = @network_range.persisted? ? @network_range.supernet_rules.includes(:network_range, :user, :waf_policy).limit(10) : []
@subnet_rules = @network_range.persisted? ? @network_range.child_rules.includes(:network_range, :user, :waf_policy).limit(20) : []
# Traffic analytics (if we have events)
@traffic_stats = calculate_traffic_stats(@network_range)
# Check if we have IPAPI data (or if parent has it)
# Check if we have IPAPI data (or if parent has it) - cache expensive parent lookup
@has_ipapi_data = @network_range.has_network_data_from?(:ipapi)
@parent_with_ipapi = nil
unless @has_ipapi_data
# Check if parent has IPAPI data
parent = @network_range.parent_with_intelligence
# Cache expensive parent intelligence lookup
parent = Rails.cache.fetch("network_parent_intel:#{@network_range.cache_key}", expires_in: 1.hour) do
@network_range.parent_with_intelligence
end
if parent&.has_network_data_from?(:ipapi)
@parent_with_ipapi = parent
@has_ipapi_data = true
@@ -194,6 +198,15 @@ class NetworkRangesController < ApplicationController
private
# Helper method to try DuckDB first, fall back to PostgreSQL
def with_duckdb_fallback(&block)
result = yield
result.nil? ? nil : result # Return result or nil to trigger fallback
rescue StandardError => e
Rails.logger.warn "[NetworkRanges] DuckDB query failed, falling back to PostgreSQL: #{e.message}"
nil # Return nil to trigger fallback
end
def set_network_range
# Handle CIDR slugs (e.g., "40.77.167.100_32" -> "40.77.167.100/32")
cidr = params[:id].gsub('_', '/')
@@ -248,27 +261,37 @@ class NetworkRangesController < ApplicationController
# Use indexed network_range_id for much better performance instead of expensive CIDR operator
# Include child network ranges to capture all traffic within this network block
network_ids = [network_range.id] + network_range.child_ranges.pluck(:id)
base_query = Event.where(network_range_id: network_ids)
# Use separate queries: one for grouping (without ordering), one for recent activity (with ordering)
events_for_grouping = base_query.limit(1000)
events_for_activity = base_query.recent.limit(20)
# Try DuckDB first for stats (much faster)
duckdb_stats = with_duckdb_fallback { EventDdb.network_traffic_stats(network_ids) }
duckdb_top_paths = with_duckdb_fallback { EventDdb.network_top_paths(network_ids, 10) }
duckdb_top_agents = with_duckdb_fallback { EventDdb.network_top_user_agents(network_ids, 5) }
# Calculate counts properly - use consistent base_query for all counts
total_requests = base_query.count
unique_ips = base_query.except(:order).distinct.count(:ip_address)
blocked_requests = base_query.blocked.count
allowed_requests = base_query.allowed.count
if duckdb_stats
# DuckDB success - use fast aggregated stats
stats = duckdb_stats.merge(
top_paths: duckdb_top_paths&.to_h || {},
top_user_agents: duckdb_top_agents&.to_h || {},
recent_activity: Event.where(network_range_id: network_ids).recent.limit(20)
)
else
# PostgreSQL fallback
base_query = Event.where(network_range_id: network_ids)
events_for_grouping = base_query.limit(1000)
events_for_activity = base_query.recent.limit(20)
{
total_requests: total_requests,
unique_ips: unique_ips,
blocked_requests: blocked_requests,
allowed_requests: allowed_requests,
top_paths: events_for_grouping.group(:request_path).count.sort_by { |_, count| -count }.first(10),
top_user_agents: events_for_grouping.group(:user_agent).count.sort_by { |_, count| -count }.first(5),
recent_activity: events_for_activity
}
stats = {
total_requests: base_query.count,
unique_ips: base_query.except(:order).distinct.count(:ip_address),
blocked_requests: base_query.blocked.count,
allowed_requests: base_query.allowed.count,
top_paths: events_for_grouping.group(:request_path).count.sort_by { |_, count| -count }.first(10).to_h,
top_user_agents: events_for_grouping.group(:user_agent).count.sort_by { |_, count| -count }.first(5).to_h,
recent_activity: events_for_activity
}
end
stats
else
# No events - return empty stats
{
@@ -296,8 +319,8 @@ class NetworkRangesController < ApplicationController
unique_ips: base_query.except(:order).distinct.count(:ip_address),
blocked_requests: base_query.blocked.count,
allowed_requests: base_query.allowed.count,
top_paths: events_for_grouping.group(:request_path).count.sort_by { |_, count| -count }.first(10),
top_user_agents: events_for_grouping.group(:user_agent).count.sort_by { |_, count| -count }.first(5),
top_paths: events_for_grouping.group(:request_path).count.sort_by { |_, count| -count }.first(10).to_h,
top_user_agents: events_for_grouping.group(:user_agent).count.sort_by { |_, count| -count }.first(5).to_h,
recent_activity: events_for_activity
}
else

View File

@@ -46,12 +46,9 @@ class RulesController < ApplicationController
process_quick_create_parameters
# Handle network range creation if CIDR is provided
if params[:cidr].present? && @rule.network_rule?
network_range = NetworkRange.find_or_create_by(cidr: params[:cidr]) do |range|
range.user = Current.user
range.source = 'manual'
range.creation_reason = "Created for rule ##{@rule.id}"
end
cidr_param = params[:new_cidr].presence || params[:cidr].presence
if cidr_param.present? && @rule.network_rule?
network_range = NetworkRange.find_or_create_by_cidr(cidr_param, user: Current.user, source: 'manual')
@rule.network_range = network_range
end
@@ -132,7 +129,9 @@ class RulesController < ApplicationController
:expires_at,
:enabled,
:source,
:network_range_id
:network_range_id,
:header_name,
:header_value
]
# Only include conditions for non-network rules
@@ -250,15 +249,24 @@ def process_quick_create_parameters
})
end
# Parse metadata if it's a string that looks like JSON
if @rule.metadata.is_a?(String) && @rule.metadata.starts_with?('{')
# Parse metadata textarea first if it's JSON
if @rule.metadata.is_a?(String) && @rule.metadata.present? && @rule.metadata.starts_with?('{')
begin
@rule.metadata = JSON.parse(@rule.metadata)
rescue JSON::ParserError
# Keep as string if not valid JSON
# Keep as string if not valid JSON - will be caught by validation
end
end
# Ensure metadata is a hash
@rule.metadata = {} unless @rule.metadata.is_a?(Hash)
# Handle add_header fields - use provided params or existing metadata values
if @rule.add_header_action? && (params[:header_name].present? || params[:header_value].present?)
@rule.metadata['header_name'] = params[:header_name].presence || @rule.metadata['header_name'] || 'X-Bot-Agent'
@rule.metadata['header_value'] = params[:header_value].presence || @rule.metadata['header_value'] || 'Unknown'
end
# Handle expires_at parsing for text input
if params.dig(:rule, :expires_at).present?
expires_at_str = params[:rule][:expires_at].strip